From: Wolfgang Bangerth Date: Mon, 10 Mar 2025 17:14:05 +0000 (-0600) Subject: Upgrade to Kokkos 4.5.1. X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d12ae0fd6abb0877dfe1766d24491e339efa4d1b;p=dealii.git Upgrade to Kokkos 4.5.1. --- diff --git a/bundled/kokkos-3.7.00/CMakeLists.txt b/bundled/kokkos-3.7.00/CMakeLists.txt deleted file mode 100644 index b5502cec20..0000000000 --- a/bundled/kokkos-3.7.00/CMakeLists.txt +++ /dev/null @@ -1,79 +0,0 @@ -## ------------------------------------------------------------------------ -## -## SPDX-License-Identifier: LGPL-2.1-or-later -## Copyright (C) 2022 - 2023 by the deal.II authors -## -## This file is part of the deal.II library. -## -## Part of the source code is dual licensed under Apache-2.0 WITH -## LLVM-exception OR LGPL-2.1-or-later. Detailed license information -## governing the source code and code contributions can be found in -## LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. -## -## ------------------------------------------------------------------------ - -set(src_kokkos - #tpls/desul/src/Lock_Array_CUDA.cpp - #tpls/desul/src/Lock_Array_HIP.cpp - core/src/impl/Kokkos_Profiling.cpp - core/src/impl/Kokkos_NumericTraits.cpp - core/src/impl/Kokkos_HostSpace_deepcopy.cpp - core/src/impl/Kokkos_Error.cpp - core/src/impl/Kokkos_Spinwait.cpp - core/src/impl/Kokkos_MemoryPool.cpp - core/src/impl/Kokkos_HostThreadTeam.cpp - core/src/impl/Kokkos_hwloc.cpp - core/src/impl/Kokkos_MemorySpace.cpp - core/src/impl/Kokkos_HBWSpace.cpp - core/src/impl/Kokkos_HostBarrier.cpp - core/src/impl/Kokkos_Core.cpp - core/src/impl/Kokkos_HostSpace.cpp - core/src/impl/Kokkos_CPUDiscovery.cpp - core/src/impl/Kokkos_Stacktrace.cpp - core/src/impl/Kokkos_SharedAlloc.cpp - core/src/impl/Kokkos_Command_Line_Parsing.cpp - core/src/impl/Kokkos_ExecPolicy.cpp - #core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp - #core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp - #core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp - #core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp - #core/src/OpenACC/Kokkos_OpenACCSpace.cpp - #core/src/OpenACC/Kokkos_OpenACC.cpp - #core/src/OpenACC/Kokkos_OpenACC_Instance.cpp - #core/src/Cuda/Kokkos_Cuda_Instance.cpp - #core/src/Cuda/Kokkos_Cuda_Task.cpp - #core/src/Cuda/Kokkos_CudaSpace.cpp - #core/src/Cuda/Kokkos_Cuda_Locks.cpp - #core/src/HPX/Kokkos_HPX_Task.cpp - #core/src/HPX/Kokkos_HPX.cpp - core/src/dummy.cpp - #core/src/Threads/Kokkos_ThreadsExec.cpp - #core/src/HIP/Kokkos_HIP_Instance.cpp - #core/src/HIP/Kokkos_HIP_Space.cpp - #core/src/HIP/Kokkos_HIP_Locks.cpp - #core/src/SYCL/Kokkos_SYCL_Space.cpp - #core/src/SYCL/Kokkos_SYCL.cpp - #core/src/SYCL/Kokkos_SYCL_Instance.cpp - core/src/Serial/Kokkos_Serial.cpp - core/src/Serial/Kokkos_Serial_Task.cpp - #core/src/OpenMP/Kokkos_OpenMP_Instance.cpp - #core/src/OpenMP/Kokkos_OpenMP_Task.cpp - algorithms/src/KokkosAlgorithms_dummy.cpp - simd/src/Kokkos_SIMD_dummy.cpp - containers/src/impl/Kokkos_UnorderedMap_impl.cpp -) - -enable_if_supported(DEAL_II_WARNING_FLAGS -Wno-float-conversion) -enable_if_supported(DEAL_II_WARNING_FLAGS -Wno-missing-field-initializers) -enable_if_supported(DEAL_II_WARNING_FLAGS -Wno-suggest-override) -enable_if_supported(DEAL_II_WARNING_FLAGS -Wno-unused-but-set-parameter) - -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR}/algorithms/src - ${CMAKE_CURRENT_SOURCE_DIR}/containers/src - ${CMAKE_CURRENT_SOURCE_DIR}/core/src - ${CMAKE_CURRENT_SOURCE_DIR}/simd/src - ${CMAKE_CURRENT_SOURCE_DIR}/tpls/desul/include - ) - -define_object_library(bundled_kokkos OBJECT ${src_kokkos}) diff --git a/bundled/kokkos-3.7.00/Copyright.txt b/bundled/kokkos-3.7.00/Copyright.txt deleted file mode 100644 index 5e2f8d8647..0000000000 --- a/bundled/kokkos-3.7.00/Copyright.txt +++ /dev/null @@ -1,41 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER diff --git a/bundled/kokkos-3.7.00/LICENSE b/bundled/kokkos-3.7.00/LICENSE deleted file mode 100644 index c6f17087d5..0000000000 --- a/bundled/kokkos-3.7.00/LICENSE +++ /dev/null @@ -1,43 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Kokkos is licensed under 3-clause BSD terms of use: -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER diff --git a/bundled/kokkos-3.7.00/algorithms/src/KokkosAlgorithms_dummy.cpp b/bundled/kokkos-3.7.00/algorithms/src/KokkosAlgorithms_dummy.cpp deleted file mode 100644 index 9c08a088b0..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/KokkosAlgorithms_dummy.cpp +++ /dev/null @@ -1 +0,0 @@ -void KOKKOS_ALGORITHMS_SRC_DUMMY_PREVENT_LINK_ERROR() {} diff --git a/bundled/kokkos-3.7.00/algorithms/src/Kokkos_Sort.hpp b/bundled/kokkos-3.7.00/algorithms/src/Kokkos_Sort.hpp deleted file mode 100644 index ad0c2d47b6..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/Kokkos_Sort.hpp +++ /dev/null @@ -1,696 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SORT_HPP_ -#define KOKKOS_SORT_HPP_ -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_SORT -#endif - -#include - -#include - -namespace Kokkos { - -namespace Impl { - -template -struct CopyOp; - -template -struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, - size_t i_src) { - dst(i_dst) = src(i_src); - } -}; - -template -struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, - size_t i_src) { - for (int j = 0; j < (int)dst.extent(1); j++) dst(i_dst, j) = src(i_src, j); - } -}; - -template -struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, - size_t i_src) { - for (int j = 0; j < dst.extent(1); j++) - for (int k = 0; k < dst.extent(2); k++) - dst(i_dst, j, k) = src(i_src, j, k); - } -}; -} // namespace Impl - -//---------------------------------------------------------------------------- - -template -class BinSort { - public: - template - struct copy_functor { - using src_view_type = typename SrcViewType::const_type; - - using copy_op = Impl::CopyOp; - - DstViewType dst_values; - src_view_type src_values; - int dst_offset; - - copy_functor(DstViewType const& dst_values_, int const& dst_offset_, - SrcViewType const& src_values_) - : dst_values(dst_values_), - src_values(src_values_), - dst_offset(dst_offset_) {} - - KOKKOS_INLINE_FUNCTION - void operator()(const int& i) const { - copy_op::copy(dst_values, i + dst_offset, src_values, i); - } - }; - - template - struct copy_permute_functor { - // If a Kokkos::View then can generate constant random access - // otherwise can only use the constant type. - - using src_view_type = std::conditional_t< - Kokkos::is_view::value, - Kokkos::View >, - typename SrcViewType::const_type>; - - using perm_view_type = typename PermuteViewType::const_type; - - using copy_op = Impl::CopyOp; - - DstViewType dst_values; - perm_view_type sort_order; - src_view_type src_values; - int src_offset; - - copy_permute_functor(DstViewType const& dst_values_, - PermuteViewType const& sort_order_, - SrcViewType const& src_values_, int const& src_offset_) - : dst_values(dst_values_), - sort_order(sort_order_), - src_values(src_values_), - src_offset(src_offset_) {} - - KOKKOS_INLINE_FUNCTION - void operator()(const int& i) const { - copy_op::copy(dst_values, i, src_values, src_offset + sort_order(i)); - } - }; - - // Naming this alias "execution_space" would be problematic since it would be - // considered as execution space for the various functors which might use - // another execution space through sort() or create_permute_vector(). - using exec_space = typename Space::execution_space; - using bin_op_type = BinSortOp; - - struct bin_count_tag {}; - struct bin_offset_tag {}; - struct bin_binning_tag {}; - struct bin_sort_bins_tag {}; - - public: - using size_type = SizeType; - using value_type = size_type; - - using offset_type = Kokkos::View; - using bin_count_type = Kokkos::View; - - using const_key_view_type = typename KeyViewType::const_type; - - // If a Kokkos::View then can generate constant random access - // otherwise can only use the constant type. - - using const_rnd_key_view_type = std::conditional_t< - Kokkos::is_view::value, - Kokkos::View >, - const_key_view_type>; - - using non_const_key_scalar = typename KeyViewType::non_const_value_type; - using const_key_scalar = typename KeyViewType::const_value_type; - - using bin_count_atomic_type = - Kokkos::View >; - - private: - const_key_view_type keys; - const_rnd_key_view_type keys_rnd; - - public: - BinSortOp bin_op; - offset_type bin_offsets; - bin_count_atomic_type bin_count_atomic; - bin_count_type bin_count_const; - offset_type sort_order; - - int range_begin; - int range_end; - bool sort_within_bins; - - public: - BinSort() = default; - - //---------------------------------------- - // Constructor: takes the keys, the binning_operator and optionally whether to - // sort within bins (default false) - template - BinSort(const ExecutionSpace& exec, const_key_view_type keys_, - int range_begin_, int range_end_, BinSortOp bin_op_, - bool sort_within_bins_ = false) - : keys(keys_), - keys_rnd(keys_), - bin_op(bin_op_), - bin_offsets(), - bin_count_atomic(), - bin_count_const(), - sort_order(), - range_begin(range_begin_), - range_end(range_end_), - sort_within_bins(sort_within_bins_) { - static_assert( - Kokkos::SpaceAccessibility::accessible, - "The provided execution space must be able to access the memory space " - "BinSort was initialized with!"); - if (bin_op.max_bins() <= 0) - Kokkos::abort( - "The number of bins in the BinSortOp object must be greater than 0!"); - bin_count_atomic = Kokkos::View( - "Kokkos::SortImpl::BinSortFunctor::bin_count", bin_op.max_bins()); - bin_count_const = bin_count_atomic; - bin_offsets = - offset_type(view_alloc(exec, WithoutInitializing, - "Kokkos::SortImpl::BinSortFunctor::bin_offsets"), - bin_op.max_bins()); - sort_order = - offset_type(view_alloc(exec, WithoutInitializing, - "Kokkos::SortImpl::BinSortFunctor::sort_order"), - range_end - range_begin); - } - - BinSort(const_key_view_type keys_, int range_begin_, int range_end_, - BinSortOp bin_op_, bool sort_within_bins_ = false) - : BinSort(exec_space{}, keys_, range_begin_, range_end_, bin_op_, - sort_within_bins_) {} - - template - BinSort(const ExecutionSpace& exec, const_key_view_type keys_, - BinSortOp bin_op_, bool sort_within_bins_ = false) - : BinSort(exec, keys_, 0, keys_.extent(0), bin_op_, sort_within_bins_) {} - - BinSort(const_key_view_type keys_, BinSortOp bin_op_, - bool sort_within_bins_ = false) - : BinSort(exec_space{}, keys_, bin_op_, sort_within_bins_) {} - - //---------------------------------------- - // Create the permutation vector, the bin_offset array and the bin_count - // array. Can be called again if keys changed - template - void create_permute_vector(const ExecutionSpace& exec = exec_space{}) { - static_assert( - Kokkos::SpaceAccessibility::accessible, - "The provided execution space must be able to access the memory space " - "BinSort was initialized with!"); - - const size_t len = range_end - range_begin; - Kokkos::parallel_for( - "Kokkos::Sort::BinCount", - Kokkos::RangePolicy(exec, 0, len), - *this); - Kokkos::parallel_scan("Kokkos::Sort::BinOffset", - Kokkos::RangePolicy( - exec, 0, bin_op.max_bins()), - *this); - - Kokkos::deep_copy(exec, bin_count_atomic, 0); - Kokkos::parallel_for( - "Kokkos::Sort::BinBinning", - Kokkos::RangePolicy(exec, 0, len), - *this); - - if (sort_within_bins) - Kokkos::parallel_for( - "Kokkos::Sort::BinSort", - Kokkos::RangePolicy( - exec, 0, bin_op.max_bins()), - *this); - } - - // Sort a subset of a view with respect to the first dimension using the - // permutation array - template - void sort(const ExecutionSpace& exec, ValuesViewType const& values, - int values_range_begin, int values_range_end) const { - static_assert( - Kokkos::SpaceAccessibility::accessible, - "The provided execution space must be able to access the memory space " - "BinSort was initialized with!"); - static_assert( - Kokkos::SpaceAccessibility< - ExecutionSpace, typename ValuesViewType::memory_space>::accessible, - "The provided execution space must be able to access the memory space " - "of the View argument!"); - - using scratch_view_type = - Kokkos::View; - - const size_t len = range_end - range_begin; - const size_t values_len = values_range_end - values_range_begin; - if (len != values_len) { - Kokkos::abort( - "BinSort::sort: values range length != permutation vector length"); - } - - scratch_view_type sorted_values( - view_alloc(exec, WithoutInitializing, - "Kokkos::SortImpl::BinSortFunctor::sorted_values"), - values.rank_dynamic > 0 ? len : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 1 ? values.extent(1) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 2 ? values.extent(2) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 3 ? values.extent(3) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 4 ? values.extent(4) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 5 ? values.extent(5) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 6 ? values.extent(6) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 7 ? values.extent(7) - : KOKKOS_IMPL_CTOR_DEFAULT_ARG); - - { - copy_permute_functor - functor(sorted_values, sort_order, values, - values_range_begin - range_begin); - - parallel_for("Kokkos::Sort::CopyPermute", - Kokkos::RangePolicy(exec, 0, len), functor); - } - - { - copy_functor functor( - values, range_begin, sorted_values); - - parallel_for("Kokkos::Sort::Copy", - Kokkos::RangePolicy(exec, 0, len), functor); - } - } - - // Sort a subset of a view with respect to the first dimension using the - // permutation array - template - void sort(ValuesViewType const& values, int values_range_begin, - int values_range_end) const { - exec_space exec; - sort(exec, values, values_range_begin, values_range_end); - exec.fence("Kokkos::Sort: fence after sorting"); - } - - template - void sort(ExecutionSpace const& exec, ValuesViewType const& values) const { - this->sort(exec, values, 0, /*values.extent(0)*/ range_end - range_begin); - } - - template - void sort(ValuesViewType const& values) const { - this->sort(values, 0, /*values.extent(0)*/ range_end - range_begin); - } - - // Get the permutation vector - KOKKOS_INLINE_FUNCTION - offset_type get_permute_vector() const { return sort_order; } - - // Get the start offsets for each bin - KOKKOS_INLINE_FUNCTION - offset_type get_bin_offsets() const { return bin_offsets; } - - // Get the count for each bin - KOKKOS_INLINE_FUNCTION - bin_count_type get_bin_count() const { return bin_count_const; } - - public: - KOKKOS_INLINE_FUNCTION - void operator()(const bin_count_tag& /*tag*/, const int i) const { - const int j = range_begin + i; - bin_count_atomic(bin_op.bin(keys, j))++; - } - - KOKKOS_INLINE_FUNCTION - void operator()(const bin_offset_tag& /*tag*/, const int i, - value_type& offset, const bool& final) const { - if (final) { - bin_offsets(i) = offset; - } - offset += bin_count_const(i); - } - - KOKKOS_INLINE_FUNCTION - void operator()(const bin_binning_tag& /*tag*/, const int i) const { - const int j = range_begin + i; - const int bin = bin_op.bin(keys, j); - const int count = bin_count_atomic(bin)++; - - sort_order(bin_offsets(bin) + count) = j; - } - - KOKKOS_INLINE_FUNCTION - void operator()(const bin_sort_bins_tag& /*tag*/, const int i) const { - auto bin_size = bin_count_const(i); - if (bin_size <= 1) return; - int upper_bound = bin_offsets(i) + bin_size; - bool sorted = false; - while (!sorted) { - sorted = true; - int old_idx = sort_order(bin_offsets(i)); - int new_idx = 0; - for (int k = bin_offsets(i) + 1; k < upper_bound; k++) { - new_idx = sort_order(k); - - if (!bin_op(keys_rnd, old_idx, new_idx)) { - sort_order(k - 1) = new_idx; - sort_order(k) = old_idx; - sorted = false; - } else { - old_idx = new_idx; - } - } - upper_bound--; - } - } -}; - -//---------------------------------------------------------------------------- - -template -struct BinOp1D { - int max_bins_ = {}; - double mul_ = {}; - double min_ = {}; - - BinOp1D() = default; - - // Construct BinOp with number of bins, minimum value and maximum value - BinOp1D(int max_bins__, typename KeyViewType::const_value_type min, - typename KeyViewType::const_value_type max) - : max_bins_(max_bins__ + 1), - // Cast to double to avoid possible overflow when using integer - mul_(static_cast(max_bins__) / - (static_cast(max) - static_cast(min))), - min_(static_cast(min)) { - // For integral types the number of bins may be larger than the range - // in which case we can exactly have one unique value per bin - // and then don't need to sort bins. - if (std::is_integral::value && - (static_cast(max) - static_cast(min)) <= - static_cast(max_bins__)) { - mul_ = 1.; - } - } - - // Determine bin index from key value - template - KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { - return static_cast(mul_ * (static_cast(keys(i)) - min_)); - } - - // Return maximum bin index + 1 - KOKKOS_INLINE_FUNCTION - int max_bins() const { return max_bins_; } - - // Compare to keys within a bin if true new_val will be put before old_val - template - KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, - iType2& i2) const { - return keys(i1) < keys(i2); - } -}; - -template -struct BinOp3D { - int max_bins_[3] = {}; - double mul_[3] = {}; - double min_[3] = {}; - - BinOp3D() = default; - - BinOp3D(int max_bins__[], typename KeyViewType::const_value_type min[], - typename KeyViewType::const_value_type max[]) { - max_bins_[0] = max_bins__[0]; - max_bins_[1] = max_bins__[1]; - max_bins_[2] = max_bins__[2]; - mul_[0] = static_cast(max_bins__[0]) / - (static_cast(max[0]) - static_cast(min[0])); - mul_[1] = static_cast(max_bins__[1]) / - (static_cast(max[1]) - static_cast(min[1])); - mul_[2] = static_cast(max_bins__[2]) / - (static_cast(max[2]) - static_cast(min[2])); - min_[0] = static_cast(min[0]); - min_[1] = static_cast(min[1]); - min_[2] = static_cast(min[2]); - } - - template - KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { - return int((((int(mul_[0] * (keys(i, 0) - min_[0])) * max_bins_[1]) + - int(mul_[1] * (keys(i, 1) - min_[1]))) * - max_bins_[2]) + - int(mul_[2] * (keys(i, 2) - min_[2]))); - } - - KOKKOS_INLINE_FUNCTION - int max_bins() const { return max_bins_[0] * max_bins_[1] * max_bins_[2]; } - - template - KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, - iType2& i2) const { - if (keys(i1, 0) > keys(i2, 0)) - return true; - else if (keys(i1, 0) == keys(i2, 0)) { - if (keys(i1, 1) > keys(i2, 1)) - return true; - else if (keys(i1, 1) == keys(i2, 1)) { - if (keys(i1, 2) > keys(i2, 2)) return true; - } - } - return false; - } -}; - -namespace Impl { - -template -bool try_std_sort(ViewType view, const ExecutionSpace& exec) { - bool possible = true; - size_t stride[8] = {view.stride_0(), view.stride_1(), view.stride_2(), - view.stride_3(), view.stride_4(), view.stride_5(), - view.stride_6(), view.stride_7()}; - possible = possible && - SpaceAccessibility::accessible; - possible = possible && (ViewType::Rank == 1); - possible = possible && (stride[0] == 1); - if (possible) { - exec.fence("Kokkos::sort: Fence before sorting on the host"); - std::sort(view.data(), view.data() + view.extent(0)); - } - return possible; -} - -template -struct min_max_functor { - using minmax_scalar = - Kokkos::MinMaxScalar; - - ViewType view; - min_max_functor(const ViewType& view_) : view(view_) {} - - KOKKOS_INLINE_FUNCTION - void operator()(const size_t& i, minmax_scalar& minmax) const { - if (view(i) < minmax.min_val) minmax.min_val = view(i); - if (view(i) > minmax.max_val) minmax.max_val = view(i); - } -}; - -} // namespace Impl - -template -std::enable_if_t::value> sort( - const ExecutionSpace& exec, ViewType const& view) { - using CompType = BinOp1D; - - Kokkos::MinMaxScalar result; - Kokkos::MinMax reducer(result); - parallel_reduce("Kokkos::Sort::FindExtent", - Kokkos::RangePolicy( - exec, 0, view.extent(0)), - Impl::min_max_functor(view), reducer); - if (result.min_val == result.max_val) return; - // For integral types the number of bins may be larger than the range - // in which case we can exactly have one unique value per bin - // and then don't need to sort bins. - bool sort_in_bins = true; - // TODO: figure out better max_bins then this ... - int64_t max_bins = view.extent(0) / 2; - if (std::is_integral::value) { - // Cast to double to avoid possible overflow when using integer - auto const max_val = static_cast(result.max_val); - auto const min_val = static_cast(result.min_val); - // using 10M as the cutoff for special behavior (roughly 40MB for the count - // array) - if ((max_val - min_val) < 10000000) { - max_bins = max_val - min_val + 1; - sort_in_bins = false; - } - } - if (std::is_floating_point::value) { - KOKKOS_ASSERT(std::isfinite(static_cast(result.max_val) - - static_cast(result.min_val))); - } - - BinSort bin_sort( - view, CompType(max_bins, result.min_val, result.max_val), sort_in_bins); - bin_sort.create_permute_vector(exec); - bin_sort.sort(exec, view); -} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -template -KOKKOS_DEPRECATED_WITH_COMMENT( - "Use the overload not taking bool always_use_kokkos_sort") -std::enable_if_t::value> sort( - const ExecutionSpace& exec, ViewType const& view, - bool const always_use_kokkos_sort) { - if (!always_use_kokkos_sort && Impl::try_std_sort(view, exec)) { - return; - } else { - sort(exec, view); - } -} -#endif - -template -void sort(ViewType const& view) { - typename ViewType::execution_space exec; - sort(exec, view); - exec.fence("Kokkos::Sort: fence after sorting"); -} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -template -KOKKOS_DEPRECATED_WITH_COMMENT( - "Use the overload not taking bool always_use_kokkos_sort") -void sort(ViewType const& view, bool const always_use_kokkos_sort) { - typename ViewType::execution_space exec; - sort(exec, view, always_use_kokkos_sort); - exec.fence("Kokkos::Sort: fence after sorting"); -} -#endif - -template -std::enable_if_t::value> sort( - const ExecutionSpace& exec, ViewType view, size_t const begin, - size_t const end) { - using range_policy = Kokkos::RangePolicy; - using CompType = BinOp1D; - - Kokkos::MinMaxScalar result; - Kokkos::MinMax reducer(result); - - parallel_reduce("Kokkos::Sort::FindExtent", range_policy(exec, begin, end), - Impl::min_max_functor(view), reducer); - - if (result.min_val == result.max_val) return; - - BinSort bin_sort( - exec, view, begin, end, - CompType((end - begin) / 2, result.min_val, result.max_val), true); - - bin_sort.create_permute_vector(exec); - bin_sort.sort(exec, view, begin, end); -} - -template -void sort(ViewType view, size_t const begin, size_t const end) { - typename ViewType::execution_space exec; - sort(exec, view, begin, end); - exec.fence("Kokkos::Sort: fence after sorting"); -} - -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_SORT -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_SORT -#endif -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AdjacentDifference.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AdjacentDifference.hpp deleted file mode 100644 index 0a7cf06f5b..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AdjacentDifference.hpp +++ /dev/null @@ -1,188 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_HPP -#define KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_HPP - -#include "impl/Kokkos_AdjacentDifference.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -std::enable_if_t::value, - OutputIteratorType> -adjacent_difference(const ExecutionSpace& ex, InputIteratorType first_from, - InputIteratorType last_from, - OutputIteratorType first_dest) { - using value_type1 = typename InputIteratorType::value_type; - using value_type2 = typename OutputIteratorType::value_type; - using binary_op = - Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; - return Impl::adjacent_difference_impl( - "Kokkos::adjacent_difference_iterator_api", ex, first_from, last_from, - first_dest, binary_op()); -} - -template -std::enable_if_t::value, - OutputIteratorType> -adjacent_difference(const ExecutionSpace& ex, InputIteratorType first_from, - InputIteratorType last_from, OutputIteratorType first_dest, - BinaryOp bin_op) { - return Impl::adjacent_difference_impl( - "Kokkos::adjacent_difference_iterator_api", ex, first_from, last_from, - first_dest, bin_op); -} - -template -std::enable_if_t::value, - OutputIteratorType> -adjacent_difference(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first_from, InputIteratorType last_from, - OutputIteratorType first_dest) { - using value_type1 = typename InputIteratorType::value_type; - using value_type2 = typename OutputIteratorType::value_type; - using binary_op = - Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; - return Impl::adjacent_difference_impl(label, ex, first_from, last_from, - first_dest, binary_op()); -} - -template -std::enable_if_t::value, - OutputIteratorType> -adjacent_difference(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first_from, InputIteratorType last_from, - OutputIteratorType first_dest, BinaryOp bin_op) { - return Impl::adjacent_difference_impl(label, ex, first_from, last_from, - first_dest, bin_op); -} - -template -auto adjacent_difference( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - - using view_type1 = ::Kokkos::View; - using view_type2 = ::Kokkos::View; - using value_type1 = typename view_type1::value_type; - using value_type2 = typename view_type2::value_type; - using binary_op = - Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; - return Impl::adjacent_difference_impl( - "Kokkos::adjacent_difference_view_api", ex, KE::cbegin(view_from), - KE::cend(view_from), KE::begin(view_dest), binary_op()); -} - -template -auto adjacent_difference( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOp bin_op) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - return Impl::adjacent_difference_impl( - "Kokkos::adjacent_difference_view_api", ex, KE::cbegin(view_from), - KE::cend(view_from), KE::begin(view_dest), bin_op); -} - -template -auto adjacent_difference( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - - using view_type1 = ::Kokkos::View; - using view_type2 = ::Kokkos::View; - using value_type1 = typename view_type1::value_type; - using value_type2 = typename view_type2::value_type; - using binary_op = - Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; - - return Impl::adjacent_difference_impl(label, ex, KE::cbegin(view_from), - KE::cend(view_from), - KE::begin(view_dest), binary_op()); -} - -template -auto adjacent_difference( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOp bin_op) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - return Impl::adjacent_difference_impl(label, ex, KE::cbegin(view_from), - KE::cend(view_from), - KE::begin(view_dest), bin_op); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AdjacentFind.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AdjacentFind.hpp deleted file mode 100644 index 332f9dd369..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AdjacentFind.hpp +++ /dev/null @@ -1,124 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_HPP -#define KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_HPP - -#include "impl/Kokkos_AdjacentFind.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set1 -template -IteratorType adjacent_find(const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::adjacent_find_impl("Kokkos::adjacent_find_iterator_api_default", - ex, first, last); -} - -template -IteratorType adjacent_find(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { - return Impl::adjacent_find_impl(label, ex, first, last); -} - -template -auto adjacent_find(const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - namespace KE = ::Kokkos::Experimental; - return Impl::adjacent_find_impl("Kokkos::adjacent_find_view_api_default", ex, - KE::begin(v), KE::end(v)); -} - -template -auto adjacent_find(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - namespace KE = ::Kokkos::Experimental; - return Impl::adjacent_find_impl(label, ex, KE::begin(v), KE::end(v)); -} - -// overload set2 -template -IteratorType adjacent_find(const ExecutionSpace& ex, IteratorType first, - IteratorType last, BinaryPredicateType pred) { - return Impl::adjacent_find_impl("Kokkos::adjacent_find_iterator_api_default", - ex, first, last, pred); -} - -template -IteratorType adjacent_find(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - BinaryPredicateType pred) { - return Impl::adjacent_find_impl(label, ex, first, last, pred); -} - -template -auto adjacent_find(const ExecutionSpace& ex, - const ::Kokkos::View& v, - BinaryPredicateType pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - namespace KE = ::Kokkos::Experimental; - return Impl::adjacent_find_impl("Kokkos::adjacent_find_view_api_default", ex, - KE::begin(v), KE::end(v), pred); -} - -template -auto adjacent_find(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - BinaryPredicateType pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - namespace KE = ::Kokkos::Experimental; - return Impl::adjacent_find_impl(label, ex, KE::begin(v), KE::end(v), pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AllOf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AllOf.hpp deleted file mode 100644 index 66a49541f3..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AllOf.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ALL_OF_HPP -#define KOKKOS_STD_ALGORITHMS_ALL_OF_HPP - -#include "impl/Kokkos_AllOfAnyOfNoneOf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -bool all_of(const ExecutionSpace& ex, InputIterator first, InputIterator last, - Predicate predicate) { - return Impl::all_of_impl("Kokkos::all_of_iterator_api_default", ex, first, - last, predicate); -} - -template -bool all_of(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, Predicate predicate) { - return Impl::all_of_impl(label, ex, first, last, predicate); -} - -template -bool all_of(const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::all_of_impl("Kokkos::all_of_view_api_default", ex, KE::cbegin(v), - KE::cend(v), std::move(predicate)); -} - -template -bool all_of(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::all_of_impl(label, ex, KE::cbegin(v), KE::cend(v), - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AnyOf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AnyOf.hpp deleted file mode 100644 index e50e90f6da..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_AnyOf.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ANY_OF_HPP -#define KOKKOS_STD_ALGORITHMS_ANY_OF_HPP - -#include "impl/Kokkos_AllOfAnyOfNoneOf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -bool any_of(const ExecutionSpace& ex, InputIterator first, InputIterator last, - Predicate predicate) { - return Impl::any_of_impl("Kokkos::any_of_view_api_default", ex, first, last, - predicate); -} - -template -bool any_of(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, Predicate predicate) { - return Impl::any_of_impl(label, ex, first, last, predicate); -} - -template -bool any_of(const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::any_of_impl("Kokkos::any_of_view_api_default", ex, KE::cbegin(v), - KE::cend(v), std::move(predicate)); -} - -template -bool any_of(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::any_of_impl(label, ex, KE::cbegin(v), KE::cend(v), - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Copy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Copy.hpp deleted file mode 100644 index b3237041b7..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Copy.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_COPY_HPP - -#include "impl/Kokkos_CopyCopyN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator copy(const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first) { - return Impl::copy_impl("Kokkos::copy_iterator_api_default", ex, first, last, - d_first); -} - -template -OutputIterator copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first) { - return Impl::copy_impl(label, ex, first, last, d_first); -} - -template -auto copy(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - namespace KE = ::Kokkos::Experimental; - return Impl::copy_impl("Kokkos::copy_view_api_default", ex, - KE::cbegin(source), KE::cend(source), KE::begin(dest)); -} - -template -auto copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - namespace KE = ::Kokkos::Experimental; - return Impl::copy_impl(label, ex, KE::cbegin(source), KE::cend(source), - KE::begin(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyBackward.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyBackward.hpp deleted file mode 100644 index 83efd96672..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyBackward.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_HPP -#define KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_HPP - -#include "impl/Kokkos_CopyBackward.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType2 copy_backward(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 d_last) { - return Impl::copy_backward_impl("Kokkos::copy_backward_iterator_api_default", - ex, first, last, d_last); -} - -template -IteratorType2 copy_backward(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 d_last) { - return Impl::copy_backward_impl(label, ex, first, last, d_last); -} - -template -auto copy_backward(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::copy_backward_impl("Kokkos::copy_backward_view_api_default", ex, - cbegin(source), cend(source), end(dest)); -} - -template -auto copy_backward(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::copy_backward_impl(label, ex, cbegin(source), cend(source), - end(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyIf.hpp deleted file mode 100644 index c83cc29886..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyIf.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COPY_IF_HPP -#define KOKKOS_STD_ALGORITHMS_COPY_IF_HPP - -#include "impl/Kokkos_CopyIf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator copy_if(const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first, - Predicate pred) { - return Impl::copy_if_impl("Kokkos::copy_if_iterator_api_default", ex, first, - last, d_first, std::move(pred)); -} - -template -OutputIterator copy_if(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first, Predicate pred) { - return Impl::copy_if_impl(label, ex, first, last, d_first, std::move(pred)); -} - -template -auto copy_if(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest, Predicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::copy_if_impl("Kokkos::copy_if_view_api_default", ex, - cbegin(source), cend(source), begin(dest), - std::move(pred)); -} - -template -auto copy_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest, Predicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::copy_if_impl(label, ex, cbegin(source), cend(source), - begin(dest), std::move(pred)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyN.hpp deleted file mode 100644 index 7f3b9374c7..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CopyN.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COPY_N_HPP -#define KOKKOS_STD_ALGORITHMS_COPY_N_HPP - -#include "impl/Kokkos_CopyCopyN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator copy_n(const ExecutionSpace& ex, InputIterator first, Size count, - OutputIterator result) { - return Impl::copy_n_impl("Kokkos::copy_n_iterator_api_default", ex, first, - count, result); -} - -template -OutputIterator copy_n(const std::string& label, const ExecutionSpace& ex, - InputIterator first, Size count, OutputIterator result) { - return Impl::copy_n_impl(label, ex, first, count, result); -} - -template -auto copy_n(const ExecutionSpace& ex, - const ::Kokkos::View& source, Size count, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - namespace KE = ::Kokkos::Experimental; - return Impl::copy_n_impl("Kokkos::copy_n_view_api_default", ex, - KE::cbegin(source), count, KE::begin(dest)); -} - -template -auto copy_n(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, Size count, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - namespace KE = ::Kokkos::Experimental; - return Impl::copy_n_impl(label, ex, KE::cbegin(source), count, - KE::begin(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Count.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Count.hpp deleted file mode 100644 index a885ee4ad2..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Count.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COUNT_HPP -#define KOKKOS_STD_ALGORITHMS_COUNT_HPP - -#include "impl/Kokkos_CountCountIf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -typename IteratorType::difference_type count(const ExecutionSpace& ex, - IteratorType first, - IteratorType last, - const T& value) { - return Impl::count_impl("Kokkos::count_iterator_api_default", ex, first, last, - value); -} - -template -typename IteratorType::difference_type count(const std::string& label, - const ExecutionSpace& ex, - IteratorType first, - IteratorType last, - const T& value) { - return Impl::count_impl(label, ex, first, last, value); -} - -template -auto count(const ExecutionSpace& ex, - const ::Kokkos::View& v, const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::count_impl("Kokkos::count_view_api_default", ex, KE::cbegin(v), - KE::cend(v), value); -} - -template -auto count(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::count_impl(label, ex, KE::cbegin(v), KE::cend(v), value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CountIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CountIf.hpp deleted file mode 100644 index 98b9d74c4c..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_CountIf.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COUNT_IF_HPP -#define KOKKOS_STD_ALGORITHMS_COUNT_IF_HPP - -#include "impl/Kokkos_CountCountIf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -typename IteratorType::difference_type count_if(const ExecutionSpace& ex, - IteratorType first, - IteratorType last, - Predicate predicate) { - return Impl::count_if_impl("Kokkos::count_if_iterator_api_default", ex, first, - last, std::move(predicate)); -} - -template -typename IteratorType::difference_type count_if(const std::string& label, - const ExecutionSpace& ex, - IteratorType first, - IteratorType last, - Predicate predicate) { - return Impl::count_if_impl(label, ex, first, last, std::move(predicate)); -} - -template -auto count_if(const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::count_if_impl("Kokkos::count_if_view_api_default", ex, - KE::cbegin(v), KE::cend(v), std::move(predicate)); -} - -template -auto count_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::count_if_impl(label, ex, KE::cbegin(v), KE::cend(v), - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Distance.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Distance.hpp deleted file mode 100644 index 4e148642b1..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Distance.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_DISTANCE_HPP -#define KOKKOS_STD_ALGORITHMS_DISTANCE_HPP - -#include "impl/Kokkos_Constraints.hpp" -#include "impl/Kokkos_RandomAccessIterator.hpp" - -namespace Kokkos { -namespace Experimental { - -template -KOKKOS_INLINE_FUNCTION constexpr typename IteratorType::difference_type -distance(IteratorType first, IteratorType last) { - static_assert( - ::Kokkos::Experimental::Impl::are_random_access_iterators< - IteratorType>::value, - "Kokkos::Experimental::distance: only implemented for random access " - "iterators."); - - return last - first; -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Equal.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Equal.hpp deleted file mode 100644 index 8634019fad..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Equal.hpp +++ /dev/null @@ -1,198 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_EQUAL_HPP -#define KOKKOS_STD_ALGORITHMS_EQUAL_HPP - -#include "impl/Kokkos_Equal.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2) { - return Impl::equal_impl("Kokkos::equal_iterator_api_default", ex, first1, - last1, first2); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2) { - return Impl::equal_impl(label, ex, first1, last1, first2); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, BinaryPredicateType predicate) { - return Impl::equal_impl("Kokkos::equal_iterator_api_default", ex, first1, - last1, first2, std::move(predicate)); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, - BinaryPredicateType predicate) { - return Impl::equal_impl(label, ex, first1, last1, first2, - std::move(predicate)); -} - -template -bool equal(const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::equal_impl("Kokkos::equal_view_api_default", ex, - KE::cbegin(view1), KE::cend(view1), - KE::cbegin(view2)); -} - -template -bool equal(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::equal_impl(label, ex, KE::cbegin(view1), KE::cend(view1), - KE::cbegin(view2)); -} - -template -bool equal(const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2, - BinaryPredicateType predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::equal_impl("Kokkos::equal_view_api_default", ex, - KE::cbegin(view1), KE::cend(view1), KE::cbegin(view2), - std::move(predicate)); -} - -template -bool equal(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2, - BinaryPredicateType predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::equal_impl(label, ex, KE::cbegin(view1), KE::cend(view1), - KE::cbegin(view2), std::move(predicate)); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2) { - return Impl::equal_impl("Kokkos::equal_iterator_api_default", ex, first1, - last1, first2, last2); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, IteratorType2 last2) { - return Impl::equal_impl(label, ex, first1, last1, first2, last2); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2, - BinaryPredicateType predicate) { - return Impl::equal_impl("Kokkos::equal_iterator_api_default", ex, first1, - last1, first2, last2, std::move(predicate)); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - IteratorType1, IteratorType2>::value, - bool> -equal(const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, IteratorType2 last2, - BinaryPredicateType predicate) { - return Impl::equal_impl(label, ex, first1, last1, first2, last2, - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ExclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ExclusiveScan.hpp deleted file mode 100644 index b97710f24f..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ExclusiveScan.hpp +++ /dev/null @@ -1,190 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_HPP -#define KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_HPP - -#include "impl/Kokkos_ExclusiveScan.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1 -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -exclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - ValueType init_value) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - return Impl::exclusive_scan_default_op_impl( - "Kokkos::exclusive_scan_default_functors_iterator_api", ex, first, last, - first_dest, init_value); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -exclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, ValueType init_value) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - return Impl::exclusive_scan_default_op_impl(label, ex, first, last, - first_dest, init_value); -} - -template -auto exclusive_scan(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - ValueType init_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - namespace KE = ::Kokkos::Experimental; - return Impl::exclusive_scan_default_op_impl( - "Kokkos::exclusive_scan_default_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - init_value); -} - -template -auto exclusive_scan(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - ValueType init_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - namespace KE = ::Kokkos::Experimental; - return Impl::exclusive_scan_default_op_impl(label, ex, KE::cbegin(view_from), - KE::cend(view_from), - KE::begin(view_dest), init_value); -} - -// overload set 2 -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -exclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - ValueType init_value, BinaryOpType bop) { - Impl::static_assert_is_not_openmptarget(ex); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - return Impl::exclusive_scan_custom_op_impl( - "Kokkos::exclusive_scan_custom_functors_iterator_api", ex, first, last, - first_dest, init_value, bop); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -exclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, ValueType init_value, - BinaryOpType bop) { - Impl::static_assert_is_not_openmptarget(ex); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - return Impl::exclusive_scan_custom_op_impl(label, ex, first, last, first_dest, - init_value, bop); -} - -template -auto exclusive_scan(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - ValueType init_value, BinaryOpType bop) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - namespace KE = ::Kokkos::Experimental; - return Impl::exclusive_scan_custom_op_impl( - "Kokkos::exclusive_scan_custom_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - init_value, bop); -} - -template -auto exclusive_scan(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - ValueType init_value, BinaryOpType bop) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - namespace KE = ::Kokkos::Experimental; - return Impl::exclusive_scan_custom_op_impl( - label, ex, KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), init_value, bop); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Fill.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Fill.hpp deleted file mode 100644 index 200e03b9dc..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Fill.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FILL_HPP -#define KOKKOS_STD_ALGORITHMS_FILL_HPP - -#include "impl/Kokkos_FillFillN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -void fill(const ExecutionSpace& ex, IteratorType first, IteratorType last, - const T& value) { - Impl::fill_impl("Kokkos::fill_iterator_api_default", ex, first, last, value); -} - -template -void fill(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, const T& value) { - Impl::fill_impl(label, ex, first, last, value); -} - -template -void fill(const ExecutionSpace& ex, - const ::Kokkos::View& view, const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - Impl::fill_impl("Kokkos::fill_view_api_default", ex, begin(view), end(view), - value); -} - -template -void fill(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - Impl::fill_impl(label, ex, begin(view), end(view), value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FillN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FillN.hpp deleted file mode 100644 index 2e814dc55f..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FillN.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FILL_N_HPP -#define KOKKOS_STD_ALGORITHMS_FILL_N_HPP - -#include "impl/Kokkos_FillFillN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType fill_n(const ExecutionSpace& ex, IteratorType first, SizeType n, - const T& value) { - return Impl::fill_n_impl("Kokkos::fill_n_iterator_api_default", ex, first, n, - value); -} - -template -IteratorType fill_n(const std::string& label, const ExecutionSpace& ex, - IteratorType first, SizeType n, const T& value) { - return Impl::fill_n_impl(label, ex, first, n, value); -} - -template -auto fill_n(const ExecutionSpace& ex, - const ::Kokkos::View& view, SizeType n, - const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::fill_n_impl("Kokkos::fill_n_view_api_default", ex, begin(view), - n, value); -} - -template -auto fill_n(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, SizeType n, - const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::fill_n_impl(label, ex, begin(view), n, value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Find.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Find.hpp deleted file mode 100644 index 6758f00ce4..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Find.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FIND_HPP -#define KOKKOS_STD_ALGORITHMS_FIND_HPP - -#include "impl/Kokkos_FindIfOrNot.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -InputIterator find(const ExecutionSpace& ex, InputIterator first, - InputIterator last, const T& value) { - return Impl::find_impl("Kokkos::find_iterator_api_default", ex, first, last, - value); -} - -template -InputIterator find(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, const T& value) { - return Impl::find_impl(label, ex, first, last, value); -} - -template -auto find(const ExecutionSpace& ex, - const ::Kokkos::View& view, const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_impl("Kokkos::find_view_api_default", ex, KE::begin(view), - KE::end(view), value); -} - -template -auto find(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, const T& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_impl(label, ex, KE::begin(view), KE::end(view), value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindEnd.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindEnd.hpp deleted file mode 100644 index 61b54c8225..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindEnd.hpp +++ /dev/null @@ -1,149 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FIND_END_HPP -#define KOKKOS_STD_ALGORITHMS_FIND_END_HPP - -#include "impl/Kokkos_FindEnd.hpp" -#include "Kokkos_Equal.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1: no binary predicate passed -template -IteratorType1 find_end(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last) { - return Impl::find_end_impl("Kokkos::find_end_iterator_api_default", ex, first, - last, s_first, s_last); -} - -template -IteratorType1 find_end(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last) { - return Impl::find_end_impl(label, ex, first, last, s_first, s_last); -} - -template -auto find_end(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_end_impl("Kokkos::find_end_view_api_default", ex, - KE::begin(view), KE::end(view), KE::begin(s_view), - KE::end(s_view)); -} - -template -auto find_end(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_end_impl(label, ex, KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view)); -} - -// overload set 2: binary predicate passed -template -IteratorType1 find_end(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last, const BinaryPredicateType& pred) { - return Impl::find_end_impl("Kokkos::find_end_iterator_api_default", ex, first, - last, s_first, s_last, pred); -} - -template -IteratorType1 find_end(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last, - const BinaryPredicateType& pred) { - return Impl::find_end_impl(label, ex, first, last, s_first, s_last, pred); -} - -template -auto find_end(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_end_impl("Kokkos::find_end_view_api_default", ex, - KE::begin(view), KE::end(view), KE::begin(s_view), - KE::end(s_view), pred); -} - -template -auto find_end(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_end_impl(label, ex, KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view), pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindFirstOf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindFirstOf.hpp deleted file mode 100644 index b8c27cb272..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindFirstOf.hpp +++ /dev/null @@ -1,150 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FIND_FIRST_OF_HPP -#define KOKKOS_STD_ALGORITHMS_FIND_FIRST_OF_HPP - -#include "impl/Kokkos_FindFirstOf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1: no binary predicate passed -template -IteratorType1 find_first_of(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last) { - return Impl::find_first_of_impl("Kokkos::find_first_of_iterator_api_default", - ex, first, last, s_first, s_last); -} - -template -IteratorType1 find_first_of(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last) { - return Impl::find_first_of_impl(label, ex, first, last, s_first, s_last); -} - -template -auto find_first_of(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_first_of_impl("Kokkos::find_first_of_view_api_default", ex, - KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view)); -} - -template -auto find_first_of(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_first_of_impl(label, ex, KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view)); -} - -// overload set 2: binary predicate passed -template -IteratorType1 find_first_of(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last, - const BinaryPredicateType& pred) { - return Impl::find_first_of_impl("Kokkos::find_first_of_iterator_api_default", - ex, first, last, s_first, s_last, pred); -} - -template -IteratorType1 find_first_of(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last, - const BinaryPredicateType& pred) { - return Impl::find_first_of_impl(label, ex, first, last, s_first, s_last, - pred); -} - -template -auto find_first_of(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_first_of_impl("Kokkos::find_first_of_view_api_default", ex, - KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view), pred); -} - -template -auto find_first_of(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_first_of_impl(label, ex, KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view), pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindIf.hpp deleted file mode 100644 index 54896da117..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindIf.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FIND_IF_HPP -#define KOKKOS_STD_ALGORITHMS_FIND_IF_HPP - -#include "impl/Kokkos_FindIfOrNot.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType find_if(const ExecutionSpace& ex, IteratorType first, - IteratorType last, PredicateType predicate) { - return Impl::find_if_or_not_impl("Kokkos::find_if_iterator_api_default", - ex, first, last, std::move(predicate)); -} - -template -IteratorType find_if(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - PredicateType predicate) { - return Impl::find_if_or_not_impl(label, ex, first, last, - std::move(predicate)); -} - -template -auto find_if(const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - namespace KE = ::Kokkos::Experimental; - return Impl::find_if_or_not_impl("Kokkos::find_if_view_api_default", ex, - KE::begin(v), KE::end(v), - std::move(predicate)); -} - -template -auto find_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - namespace KE = ::Kokkos::Experimental; - return Impl::find_if_or_not_impl(label, ex, KE::begin(v), KE::end(v), - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindIfNot.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindIfNot.hpp deleted file mode 100644 index cfe6bb84d8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_FindIfNot.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FIND_IF_NOT_HPP -#define KOKKOS_STD_ALGORITHMS_FIND_IF_NOT_HPP - -#include "impl/Kokkos_FindIfOrNot.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType find_if_not(const ExecutionSpace& ex, IteratorType first, - IteratorType last, Predicate predicate) { - return Impl::find_if_or_not_impl( - "Kokkos::find_if_not_iterator_api_default", ex, first, last, - std::move(predicate)); -} - -template -IteratorType find_if_not(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - Predicate predicate) { - return Impl::find_if_or_not_impl(label, ex, first, last, - std::move(predicate)); -} - -template -auto find_if_not(const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_if_or_not_impl( - "Kokkos::find_if_not_view_api_default", ex, KE::begin(v), KE::end(v), - std::move(predicate)); -} - -template -auto find_if_not(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::find_if_or_not_impl(label, ex, KE::begin(v), KE::end(v), - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ForEach.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ForEach.hpp deleted file mode 100644 index 8a2f90e82b..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ForEach.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FOR_EACH_HPP -#define KOKKOS_STD_ALGORITHMS_FOR_EACH_HPP - -#include "impl/Kokkos_ForEachForEachN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -UnaryFunctorType for_each(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - UnaryFunctorType functor) { - return Impl::for_each_impl(label, ex, first, last, std::move(functor)); -} - -template -UnaryFunctorType for_each(const ExecutionSpace& ex, IteratorType first, - IteratorType last, UnaryFunctorType functor) { - return Impl::for_each_impl("Kokkos::for_each_iterator_api_default", ex, first, - last, std::move(functor)); -} - -template -UnaryFunctorType for_each(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - UnaryFunctorType functor) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::for_each_impl(label, ex, KE::begin(v), KE::end(v), - std::move(functor)); -} - -template -UnaryFunctorType for_each(const ExecutionSpace& ex, - const ::Kokkos::View& v, - UnaryFunctorType functor) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::for_each_impl("Kokkos::for_each_view_api_default", ex, - KE::begin(v), KE::end(v), std::move(functor)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ForEachN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ForEachN.hpp deleted file mode 100644 index dd917a33e8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ForEachN.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FOR_EACH_N_HPP -#define KOKKOS_STD_ALGORITHMS_FOR_EACH_N_HPP - -#include "impl/Kokkos_ForEachForEachN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType for_each_n(const std::string& label, const ExecutionSpace& ex, - IteratorType first, SizeType n, - UnaryFunctorType functor) { - return Impl::for_each_n_impl(label, ex, first, n, std::move(functor)); -} - -template -IteratorType for_each_n(const ExecutionSpace& ex, IteratorType first, - SizeType n, UnaryFunctorType functor) { - return Impl::for_each_n_impl("Kokkos::for_each_n_iterator_api_default", ex, - first, n, std::move(functor)); -} - -template -auto for_each_n(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, SizeType n, - UnaryFunctorType functor) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::for_each_n_impl(label, ex, KE::begin(v), n, std::move(functor)); -} - -template -auto for_each_n(const ExecutionSpace& ex, - const ::Kokkos::View& v, SizeType n, - UnaryFunctorType functor) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::for_each_n_impl("Kokkos::for_each_n_view_api_default", ex, - KE::begin(v), n, std::move(functor)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Generate.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Generate.hpp deleted file mode 100644 index 955cb42d4b..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Generate.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_GENERATE_HPP -#define KOKKOS_STD_ALGORITHMS_GENERATE_HPP - -#include "impl/Kokkos_GenerateGenerateN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -void generate(const ExecutionSpace& ex, IteratorType first, IteratorType last, - Generator g) { - Impl::generate_impl("Kokkos::generate_iterator_api_default", ex, first, last, - std::move(g)); -} - -template -void generate(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, Generator g) { - Impl::generate_impl(label, ex, first, last, std::move(g)); -} - -template -void generate(const ExecutionSpace& ex, - const ::Kokkos::View& view, - Generator g) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - Impl::generate_impl("Kokkos::generate_view_api_default", ex, begin(view), - end(view), std::move(g)); -} - -template -void generate(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - Generator g) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - Impl::generate_impl(label, ex, begin(view), end(view), std::move(g)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_GenerateN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_GenerateN.hpp deleted file mode 100644 index 470edb1596..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_GenerateN.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_GENERATE_N_HPP -#define KOKKOS_STD_ALGORITHMS_GENERATE_N_HPP - -#include "impl/Kokkos_GenerateGenerateN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType generate_n(const ExecutionSpace& ex, IteratorType first, - Size count, Generator g) { - Impl::generate_n_impl("Kokkos::generate_n_iterator_api_default", ex, first, - count, std::move(g)); - return first + count; -} - -template -IteratorType generate_n(const std::string& label, const ExecutionSpace& ex, - IteratorType first, Size count, Generator g) { - Impl::generate_n_impl(label, ex, first, count, std::move(g)); - return first + count; -} - -template -auto generate_n(const ExecutionSpace& ex, - const ::Kokkos::View& view, Size count, - Generator g) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::generate_n_impl("Kokkos::generate_n_view_api_default", ex, - begin(view), count, std::move(g)); -} - -template -auto generate_n(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, Size count, - Generator g) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::generate_n_impl(label, ex, begin(view), count, std::move(g)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_InclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_InclusiveScan.hpp deleted file mode 100644 index c34b5f43c9..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_InclusiveScan.hpp +++ /dev/null @@ -1,223 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_INCLUSIVE_SCAN_HPP -#define KOKKOS_STD_ALGORITHMS_INCLUSIVE_SCAN_HPP - -#include "impl/Kokkos_InclusiveScan.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1 -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -inclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest) { - return Impl::inclusive_scan_default_op_impl( - "Kokkos::inclusive_scan_default_functors_iterator_api", ex, first, last, - first_dest); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -inclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest) { - return Impl::inclusive_scan_default_op_impl(label, ex, first, last, - first_dest); -} - -template -auto inclusive_scan( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::inclusive_scan_default_op_impl( - "Kokkos::inclusive_scan_default_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest)); -} - -template -auto inclusive_scan( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::inclusive_scan_default_op_impl(label, ex, KE::cbegin(view_from), - KE::cend(view_from), - KE::begin(view_dest)); -} - -// overload set 2 (accepting custom binary op) -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -inclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - BinaryOp binary_op) { - return Impl::inclusive_scan_custom_binary_op_impl( - "Kokkos::inclusive_scan_custom_functors_iterator_api", ex, first, last, - first_dest, binary_op); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -inclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, BinaryOp binary_op) { - return Impl::inclusive_scan_custom_binary_op_impl(label, ex, first, last, - first_dest, binary_op); -} - -template -auto inclusive_scan(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOp binary_op) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::inclusive_scan_custom_binary_op_impl( - "Kokkos::inclusive_scan_custom_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - binary_op); -} - -template -auto inclusive_scan(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOp binary_op) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::inclusive_scan_custom_binary_op_impl( - label, ex, KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), binary_op); -} - -// overload set 3 -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -inclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - BinaryOp binary_op, ValueType init_value) { - return Impl::inclusive_scan_custom_binary_op_impl( - "Kokkos::inclusive_scan_custom_functors_iterator_api", ex, first, last, - first_dest, binary_op, init_value); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -inclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, BinaryOp binary_op, - ValueType init_value) { - return Impl::inclusive_scan_custom_binary_op_impl( - label, ex, first, last, first_dest, binary_op, init_value); -} - -template -auto inclusive_scan(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOp binary_op, ValueType init_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::inclusive_scan_custom_binary_op_impl( - "Kokkos::inclusive_scan_custom_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - binary_op, init_value); -} - -template -auto inclusive_scan(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOp binary_op, ValueType init_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::inclusive_scan_custom_binary_op_impl( - label, ex, KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), binary_op, init_value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsPartitioned.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsPartitioned.hpp deleted file mode 100644 index 8a2ca207ae..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsPartitioned.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_IS_PARTITIONED_HPP -#define KOKKOS_STD_ALGORITHMS_IS_PARTITIONED_HPP - -#include "impl/Kokkos_IsPartitioned.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -bool is_partitioned(const ExecutionSpace& ex, IteratorType first, - IteratorType last, PredicateType p) { - return Impl::is_partitioned_impl( - "Kokkos::is_partitioned_iterator_api_default", ex, first, last, - std::move(p)); -} - -template -bool is_partitioned(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, PredicateType p) { - return Impl::is_partitioned_impl(label, ex, first, last, std::move(p)); -} - -template -bool is_partitioned(const ExecutionSpace& ex, - const ::Kokkos::View& v, - PredicateType p) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::is_partitioned_impl("Kokkos::is_partitioned_view_api_default", - ex, cbegin(v), cend(v), std::move(p)); -} - -template -bool is_partitioned(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - PredicateType p) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::is_partitioned_impl(label, ex, cbegin(v), cend(v), std::move(p)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsSorted.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsSorted.hpp deleted file mode 100644 index 0ab466f338..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsSorted.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_HPP -#define KOKKOS_STD_ALGORITHMS_IS_SORTED_HPP - -#include "impl/Kokkos_IsSorted.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -bool is_sorted(const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::is_sorted_impl("Kokkos::is_sorted_iterator_api_default", ex, - first, last); -} - -template -bool is_sorted(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { - return Impl::is_sorted_impl(label, ex, first, last); -} - -template -bool is_sorted(const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_impl("Kokkos::is_sorted_view_api_default", ex, - KE::cbegin(view), KE::cend(view)); -} - -template -bool is_sorted(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_impl(label, ex, KE::cbegin(view), KE::cend(view)); -} - -template -bool is_sorted(const ExecutionSpace& ex, IteratorType first, IteratorType last, - ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - return Impl::is_sorted_impl("Kokkos::is_sorted_iterator_api_default", ex, - first, last, std::move(comp)); -} - -template -bool is_sorted(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - return Impl::is_sorted_impl(label, ex, first, last, std::move(comp)); -} - -template -bool is_sorted(const ExecutionSpace& ex, - const ::Kokkos::View& view, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_not_openmptarget(ex); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_impl("Kokkos::is_sorted_view_api_default", ex, - KE::cbegin(view), KE::cend(view), - std::move(comp)); -} - -template -bool is_sorted(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_not_openmptarget(ex); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_impl(label, ex, KE::cbegin(view), KE::cend(view), - std::move(comp)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsSortedUntil.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsSortedUntil.hpp deleted file mode 100644 index c480d9ee5a..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IsSortedUntil.hpp +++ /dev/null @@ -1,134 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_HPP -#define KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_HPP - -#include "impl/Kokkos_IsSortedUntil.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType is_sorted_until(const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::is_sorted_until_impl( - "Kokkos::is_sorted_until_iterator_api_default", ex, first, last); -} - -template -IteratorType is_sorted_until(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { - return Impl::is_sorted_until_impl(label, ex, first, last); -} - -template -auto is_sorted_until(const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_until_impl("Kokkos::is_sorted_until_view_api_default", - ex, KE::begin(view), KE::end(view)); -} - -template -auto is_sorted_until(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_until_impl(label, ex, KE::begin(view), KE::end(view)); -} - -template -IteratorType is_sorted_until(const ExecutionSpace& ex, IteratorType first, - IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - return Impl::is_sorted_until_impl( - "Kokkos::is_sorted_until_iterator_api_default", ex, first, last, - std::move(comp)); -} - -template -IteratorType is_sorted_until(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::is_sorted_until_impl(label, ex, first, last, std::move(comp)); -} - -template -auto is_sorted_until(const ExecutionSpace& ex, - const ::Kokkos::View& view, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_not_openmptarget(ex); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_until_impl("Kokkos::is_sorted_until_view_api_default", - ex, KE::begin(view), KE::end(view), - std::move(comp)); -} - -template -auto is_sorted_until(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_not_openmptarget(ex); - - namespace KE = ::Kokkos::Experimental; - return Impl::is_sorted_until_impl(label, ex, KE::begin(view), KE::end(view), - std::move(comp)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IterSwap.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IterSwap.hpp deleted file mode 100644 index 1174740a5b..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_IterSwap.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ITER_SWAP_HPP -#define KOKKOS_STD_ALGORITHMS_ITER_SWAP_HPP - -#include -#include "impl/Kokkos_Constraints.hpp" -#include "Kokkos_Swap.hpp" - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdIterSwapFunctor { - IteratorType1 m_a; - IteratorType2 m_b; - - KOKKOS_FUNCTION - void operator()(int i) const { - (void)i; - ::Kokkos::Experimental::swap(*m_a, *m_b); - } - - KOKKOS_FUNCTION - StdIterSwapFunctor(IteratorType1 _a, IteratorType2 _b) - : m_a(std::move(_a)), m_b(std::move(_b)) {} -}; - -template -void iter_swap_impl(IteratorType1 a, IteratorType2 b) { - // is there a better way to do this maybe? - ::Kokkos::parallel_for( - 1, StdIterSwapFunctor(a, b)); - Kokkos::DefaultExecutionSpace().fence( - "Kokkos::iter_swap: fence after operation"); -} -} // namespace Impl -//---------------------------------------------------------------------------- - -// iter_swap -template -void iter_swap(IteratorType1 a, IteratorType2 b) { - Impl::iter_swap_impl(a, b); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_LexicographicalCompare.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_LexicographicalCompare.hpp deleted file mode 100644 index 4704a9ec56..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_LexicographicalCompare.hpp +++ /dev/null @@ -1,154 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_LEXICOGRAPHICAL_COMPARE_HPP -#define KOKKOS_STD_ALGORITHMS_LEXICOGRAPHICAL_COMPARE_HPP - -#include "impl/Kokkos_LexicographicalCompare.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -bool lexicographical_compare(const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, - IteratorType2 last2) { - return Impl::lexicographical_compare_impl( - "Kokkos::lexicographical_compare_iterator_api_default", ex, first1, last1, - first2, last2); -} - -template -bool lexicographical_compare(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2) { - return Impl::lexicographical_compare_impl(label, ex, first1, last1, first2, - last2); -} - -template -bool lexicographical_compare( - const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::lexicographical_compare_impl( - "Kokkos::lexicographical_compare_view_api_default", ex, KE::cbegin(view1), - KE::cend(view1), KE::cbegin(view2), KE::cend(view2)); -} - -template -bool lexicographical_compare( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::lexicographical_compare_impl(label, ex, KE::cbegin(view1), - KE::cend(view1), KE::cbegin(view2), - KE::cend(view2)); -} - -template -bool lexicographical_compare(const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, - IteratorType2 last2, ComparatorType comp) { - return Impl::lexicographical_compare_impl( - "Kokkos::lexicographical_compare_iterator_api_default", ex, first1, last1, - first2, last2, comp); -} - -template -bool lexicographical_compare(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2, - ComparatorType comp) { - return Impl::lexicographical_compare_impl(label, ex, first1, last1, first2, - last2, comp); -} - -template -bool lexicographical_compare( - const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2, ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::lexicographical_compare_impl( - "Kokkos::lexicographical_compare_view_api_default", ex, KE::cbegin(view1), - KE::cend(view1), KE::cbegin(view2), KE::cend(view2), comp); -} - -template -bool lexicographical_compare( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view1, - ::Kokkos::View& view2, ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::lexicographical_compare_impl(label, ex, KE::cbegin(view1), - KE::cend(view1), KE::cbegin(view2), - KE::cend(view2), comp); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MaxElement.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MaxElement.hpp deleted file mode 100644 index 5f6e5cbf62..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MaxElement.hpp +++ /dev/null @@ -1,132 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MAX_ELEMENT_HPP -#define KOKKOS_STD_ALGORITHMS_MAX_ELEMENT_HPP - -#include "impl/Kokkos_MinMaxMinmaxElement.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -auto max_element(const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::min_or_max_element_impl( - "Kokkos::max_element_iterator_api_default", ex, first, last); -} - -template -auto max_element(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { - return Impl::min_or_max_element_impl(label, ex, first, last); -} - -template -auto max_element(const ExecutionSpace& ex, IteratorType first, - IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - "Kokkos::max_element_iterator_api_default", ex, first, last, - std::move(comp)); -} - -template -auto max_element(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - label, ex, first, last, std::move(comp)); -} - -template -auto max_element(const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::min_or_max_element_impl( - "Kokkos::max_element_view_api_default", ex, begin(v), end(v)); -} - -template -auto max_element(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::min_or_max_element_impl(label, ex, begin(v), - end(v)); -} - -template -auto max_element(const ExecutionSpace& ex, - const ::Kokkos::View& v, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - "Kokkos::max_element_view_api_default", ex, begin(v), end(v), - std::move(comp)); -} - -template -auto max_element(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - label, ex, begin(v), end(v), std::move(comp)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MinElement.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MinElement.hpp deleted file mode 100644 index 63cc548960..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MinElement.hpp +++ /dev/null @@ -1,132 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MIN_ELEMENT_HPP -#define KOKKOS_STD_ALGORITHMS_MIN_ELEMENT_HPP - -#include "impl/Kokkos_MinMaxMinmaxElement.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -auto min_element(const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::min_or_max_element_impl( - "Kokkos::min_element_iterator_api_default", ex, first, last); -} - -template -auto min_element(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { - return Impl::min_or_max_element_impl(label, ex, first, last); -} - -template -auto min_element(const ExecutionSpace& ex, IteratorType first, - IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - "Kokkos::min_element_iterator_api_default", ex, first, last, - std::move(comp)); -} - -template -auto min_element(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - label, ex, first, last, std::move(comp)); -} - -template -auto min_element(const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::min_or_max_element_impl( - "Kokkos::min_element_view_api_default", ex, begin(v), end(v)); -} - -template -auto min_element(const ExecutionSpace& ex, - const ::Kokkos::View& v, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - "Kokkos::min_element_view_api_default", ex, begin(v), end(v), - std::move(comp)); -} - -template -auto min_element(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::min_or_max_element_impl(label, ex, begin(v), - end(v)); -} - -template -auto min_element(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::min_or_max_element_impl( - label, ex, begin(v), end(v), std::move(comp)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MinMaxElement.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MinMaxElement.hpp deleted file mode 100644 index 07cdefcc05..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MinMaxElement.hpp +++ /dev/null @@ -1,133 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MINMAX_ELEMENT_HPP -#define KOKKOS_STD_ALGORITHMS_MINMAX_ELEMENT_HPP - -#include "impl/Kokkos_MinMaxMinmaxElement.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -auto minmax_element(const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::minmax_element_impl( - "Kokkos::minmax_element_iterator_api_default", ex, first, last); -} - -template -auto minmax_element(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { - return Impl::minmax_element_impl(label, ex, first, last); -} - -template -auto minmax_element(const ExecutionSpace& ex, IteratorType first, - IteratorType last, ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::minmax_element_impl( - "Kokkos::minmax_element_iterator_api_default", ex, first, last, - std::move(comp)); -} - -template -auto minmax_element(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - ComparatorType comp) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::minmax_element_impl( - label, ex, first, last, std::move(comp)); -} - -template -auto minmax_element(const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::minmax_element_impl( - "Kokkos::minmax_element_view_api_default", ex, begin(v), end(v)); -} - -template -auto minmax_element(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - return Impl::minmax_element_impl(label, ex, begin(v), - end(v)); -} - -template -auto minmax_element(const ExecutionSpace& ex, - const ::Kokkos::View& v, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::minmax_element_impl( - "Kokkos::minmax_element_view_api_default", ex, begin(v), end(v), - std::move(comp)); -} - -template -auto minmax_element(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - ComparatorType comp) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::minmax_element_impl( - label, ex, begin(v), end(v), std::move(comp)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Mismatch.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Mismatch.hpp deleted file mode 100644 index 3418e048a9..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Mismatch.hpp +++ /dev/null @@ -1,160 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MISMATCH_HPP -#define KOKKOS_STD_ALGORITHMS_MISMATCH_HPP - -#include "impl/Kokkos_Mismatch.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// FIXME: add mismatch overloads accepting 3 iterators. -// An overload consistent with other algorithms: -// -// auto mismatch(const ExecSpace& ex, It1 first1, It1 last1, It2 first2) {...} -// -// makes API ambiguous (with the overload accepting views). - -template -::Kokkos::pair mismatch(const ExecutionSpace& ex, - IteratorType1 first1, - IteratorType1 last1, - IteratorType2 first2, - IteratorType2 last2) { - return Impl::mismatch_impl("Kokkos::mismatch_iterator_api_default", ex, - first1, last1, first2, last2); -} - -template -::Kokkos::pair mismatch( - const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2, - BinaryPredicateType&& predicate) { - return Impl::mismatch_impl("Kokkos::mismatch_iterator_api_default", ex, - first1, last1, first2, last2, - std::forward(predicate)); -} - -template -::Kokkos::pair mismatch( - const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, IteratorType2 last2) { - return Impl::mismatch_impl(label, ex, first1, last1, first2, last2); -} - -template -::Kokkos::pair mismatch( - const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, IteratorType2 last2, - BinaryPredicateType&& predicate) { - return Impl::mismatch_impl(label, ex, first1, last1, first2, last2, - std::forward(predicate)); -} - -template -auto mismatch(const ExecutionSpace& ex, - const ::Kokkos::View& view1, - const ::Kokkos::View& view2) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::mismatch_impl("Kokkos::mismatch_view_api_default", ex, - KE::begin(view1), KE::end(view1), KE::begin(view2), - KE::end(view2)); -} - -template -auto mismatch(const ExecutionSpace& ex, - const ::Kokkos::View& view1, - const ::Kokkos::View& view2, - BinaryPredicateType&& predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::mismatch_impl("Kokkos::mismatch_view_api_default", ex, - KE::begin(view1), KE::end(view1), KE::begin(view2), - KE::end(view2), - std::forward(predicate)); -} - -template -auto mismatch(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view1, - const ::Kokkos::View& view2) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::mismatch_impl(label, ex, KE::begin(view1), KE::end(view1), - KE::begin(view2), KE::end(view2)); -} - -template -auto mismatch(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view1, - const ::Kokkos::View& view2, - BinaryPredicateType&& predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); - - namespace KE = ::Kokkos::Experimental; - return Impl::mismatch_impl(label, ex, KE::begin(view1), KE::end(view1), - KE::begin(view2), KE::end(view2), - std::forward(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Move.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Move.hpp deleted file mode 100644 index c2ce4655fa..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Move.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MOVE_HPP -#define KOKKOS_STD_ALGORITHMS_MOVE_HPP - -#include "impl/Kokkos_Move.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator move(const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first) { - return Impl::move_impl("Kokkos::move_iterator_api_default", ex, first, last, - d_first); -} - -template -OutputIterator move(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first) { - return Impl::move_impl(label, ex, first, last, d_first); -} - -template -auto move(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::move_impl("Kokkos::move_view_api_default", ex, begin(source), - end(source), begin(dest)); -} - -template -auto move(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::move_impl(label, ex, begin(source), end(source), begin(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MoveBackward.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MoveBackward.hpp deleted file mode 100644 index f7462d52d8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_MoveBackward.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MOVE_BACKWARD_HPP -#define KOKKOS_STD_ALGORITHMS_MOVE_BACKWARD_HPP - -#include "impl/Kokkos_MoveBackward.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType2 move_backward(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 d_last) { - return Impl::move_backward_impl("Kokkos::move_backward_iterator_api_default", - ex, first, last, d_last); -} - -template -auto move_backward(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::move_backward_impl("Kokkos::move_backward_view_api_default", ex, - begin(source), end(source), end(dest)); -} - -template -IteratorType2 move_backward(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 d_last) { - return Impl::move_backward_impl(label, ex, first, last, d_last); -} - -template -auto move_backward(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::move_backward_impl(label, ex, begin(source), end(source), - end(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_NoneOf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_NoneOf.hpp deleted file mode 100644 index 30ffb52442..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_NoneOf.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_NONE_OF_HPP -#define KOKKOS_STD_ALGORITHMS_NONE_OF_HPP - -#include "impl/Kokkos_AllOfAnyOfNoneOf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -bool none_of(const ExecutionSpace& ex, IteratorType first, IteratorType last, - Predicate predicate) { - return Impl::none_of_impl("Kokkos::none_of_iterator_api_default", ex, first, - last, predicate); -} - -template -bool none_of(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, Predicate predicate) { - return Impl::none_of_impl(label, ex, first, last, predicate); -} - -template -bool none_of(const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::none_of_impl("Kokkos::none_of_view_api_default", ex, - KE::cbegin(v), KE::cend(v), std::move(predicate)); -} - -template -bool none_of(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - Predicate predicate) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - - namespace KE = ::Kokkos::Experimental; - return Impl::none_of_impl(label, ex, KE::cbegin(v), KE::cend(v), - std::move(predicate)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_PartitionCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_PartitionCopy.hpp deleted file mode 100644 index 5b00669fd1..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_PartitionCopy.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_PARTITION_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_PARTITION_COPY_HPP - -#include "impl/Kokkos_PartitionCopy.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -::Kokkos::pair partition_copy( - const ExecutionSpace& ex, InputIteratorType from_first, - InputIteratorType from_last, OutputIteratorTrueType to_first_true, - OutputIteratorFalseType to_first_false, PredicateType p) { - return Impl::partition_copy_impl( - "Kokkos::partition_copy_iterator_api_default", ex, from_first, from_last, - to_first_true, to_first_false, std::move(p)); -} - -template -::Kokkos::pair partition_copy( - const std::string& label, const ExecutionSpace& ex, - InputIteratorType from_first, InputIteratorType from_last, - OutputIteratorTrueType to_first_true, - OutputIteratorFalseType to_first_false, PredicateType p) { - return Impl::partition_copy_impl(label, ex, from_first, from_last, - to_first_true, to_first_false, std::move(p)); -} - -template -auto partition_copy( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest_true, - const ::Kokkos::View& view_dest_false, - PredicateType p) { - return Impl::partition_copy_impl("Kokkos::partition_copy_view_api_default", - ex, cbegin(view_from), cend(view_from), - begin(view_dest_true), - begin(view_dest_false), std::move(p)); -} - -template -auto partition_copy( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest_true, - const ::Kokkos::View& view_dest_false, - PredicateType p) { - return Impl::partition_copy_impl(label, ex, cbegin(view_from), - cend(view_from), begin(view_dest_true), - begin(view_dest_false), std::move(p)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_PartitionPoint.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_PartitionPoint.hpp deleted file mode 100644 index b714d5a271..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_PartitionPoint.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_PARTITION_POINT_HPP -#define KOKKOS_STD_ALGORITHMS_PARTITION_POINT_HPP - -#include "impl/Kokkos_PartitionPoint.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType partition_point(const ExecutionSpace& ex, IteratorType first, - IteratorType last, UnaryPredicate p) { - return Impl::partition_point_impl( - "Kokkos::partitioned_point_iterator_api_default", ex, first, last, - std::move(p)); -} - -template -IteratorType partition_point(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - UnaryPredicate p) { - return Impl::partition_point_impl(label, ex, first, last, std::move(p)); -} - -template -auto partition_point(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& v, - UnaryPredicate p) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - return Impl::partition_point_impl(label, ex, begin(v), end(v), std::move(p)); -} - -template -auto partition_point(const ExecutionSpace& ex, - const ::Kokkos::View& v, - UnaryPredicate p) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); - return Impl::partition_point_impl("Kokkos::partition_point_view_api_default", - ex, begin(v), end(v), std::move(p)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Reduce.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Reduce.hpp deleted file mode 100644 index 3cf9153202..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Reduce.hpp +++ /dev/null @@ -1,221 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REDUCE_HPP -#define KOKKOS_STD_ALGORITHMS_REDUCE_HPP - -#include "impl/Kokkos_Reduce.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// -// overload set 1 -// -template -typename IteratorType::value_type reduce(const ExecutionSpace& ex, - IteratorType first, - IteratorType last) { - return Impl::reduce_default_functors_impl( - "Kokkos::reduce_default_functors_iterator_api", ex, first, last, - typename IteratorType::value_type()); -} - -template -typename IteratorType::value_type reduce(const std::string& label, - const ExecutionSpace& ex, - IteratorType first, - IteratorType last) { - return Impl::reduce_default_functors_impl( - label, ex, first, last, typename IteratorType::value_type()); -} - -template -auto reduce(const ExecutionSpace& ex, - const ::Kokkos::View& view) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - using view_type = ::Kokkos::View; - using value_type = typename view_type::value_type; - - return Impl::reduce_default_functors_impl( - "Kokkos::reduce_default_functors_view_api", ex, KE::cbegin(view), - KE::cend(view), value_type()); -} - -template -auto reduce(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - using view_type = ::Kokkos::View; - using value_type = typename view_type::value_type; - - return Impl::reduce_default_functors_impl(label, ex, KE::cbegin(view), - KE::cend(view), value_type()); -} - -// -// overload set2: -// -template -ValueType reduce(const ExecutionSpace& ex, IteratorType first, - IteratorType last, ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::reduce_default_functors_impl( - "Kokkos::reduce_default_functors_iterator_api", ex, first, last, - init_reduction_value); -} - -template -ValueType reduce(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::reduce_default_functors_impl(label, ex, first, last, - init_reduction_value); -} - -template -ValueType reduce(const ExecutionSpace& ex, - const ::Kokkos::View& view, - ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::reduce_default_functors_impl( - "Kokkos::reduce_default_functors_view_api", ex, KE::cbegin(view), - KE::cend(view), init_reduction_value); -} - -template -ValueType reduce(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::reduce_default_functors_impl( - label, ex, KE::cbegin(view), KE::cend(view), init_reduction_value); -} - -// -// overload set 3 -// -template -ValueType reduce(const ExecutionSpace& ex, IteratorType first, - IteratorType last, ValueType init_reduction_value, - BinaryOp joiner) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::reduce_custom_functors_impl( - "Kokkos::reduce_default_functors_iterator_api", ex, first, last, - init_reduction_value, joiner); -} - -template -ValueType reduce(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::reduce_custom_functors_impl(label, ex, first, last, - init_reduction_value, joiner); -} - -template -ValueType reduce(const ExecutionSpace& ex, - const ::Kokkos::View& view, - ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::reduce_custom_functors_impl( - "Kokkos::reduce_custom_functors_view_api", ex, KE::cbegin(view), - KE::cend(view), init_reduction_value, joiner); -} - -template -ValueType reduce(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::reduce_custom_functors_impl(label, ex, KE::cbegin(view), - KE::cend(view), init_reduction_value, - joiner); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Remove.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Remove.hpp deleted file mode 100644 index d8d7c999b6..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Remove.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_HPP -#define KOKKOS_STD_ALGORITHMS_REMOVE_HPP - -#include "impl/Kokkos_RemoveAllVariants.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -Iterator remove(const ExecutionSpace& ex, Iterator first, Iterator last, - const ValueType& value) { - return Impl::remove_impl("Kokkos::remove_iterator_api_default", ex, first, - last, value); -} - -template -Iterator remove(const std::string& label, const ExecutionSpace& ex, - Iterator first, Iterator last, const ValueType& value) { - return Impl::remove_impl(label, ex, first, last, value); -} - -template -auto remove(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ValueType& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::remove_impl("Kokkos::remove_iterator_api_default", ex, - ::Kokkos::Experimental::begin(view), - ::Kokkos::Experimental::end(view), value); -} - -template -auto remove(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ValueType& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::remove_impl(label, ex, ::Kokkos::Experimental::begin(view), - ::Kokkos::Experimental::end(view), value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveCopy.hpp deleted file mode 100644 index 7d5c163af9..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveCopy.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_REMOVE_COPY_HPP - -#include "impl/Kokkos_RemoveAllVariants.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator remove_copy(const ExecutionSpace& ex, InputIterator first_from, - InputIterator last_from, OutputIterator first_dest, - const ValueType& value) { - return Impl::remove_copy_impl("Kokkos::remove_copy_iterator_api_default", ex, - first_from, last_from, first_dest, value); -} - -template -OutputIterator remove_copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first_from, InputIterator last_from, - OutputIterator first_dest, const ValueType& value) { - return Impl::remove_copy_impl(label, ex, first_from, last_from, first_dest, - value); -} - -template -auto remove_copy(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - const ValueType& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - - return Impl::remove_copy_impl("Kokkos::remove_copy_iterator_api_default", ex, - ::Kokkos::Experimental::cbegin(view_from), - ::Kokkos::Experimental::cend(view_from), - ::Kokkos::Experimental::begin(view_dest), - value); -} - -template -auto remove_copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - const ValueType& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - - return Impl::remove_copy_impl( - label, ex, ::Kokkos::Experimental::cbegin(view_from), - ::Kokkos::Experimental::cend(view_from), - ::Kokkos::Experimental::begin(view_dest), value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveCopyIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveCopyIf.hpp deleted file mode 100644 index 8a9a3e4c14..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveCopyIf.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_COPY_IF_HPP -#define KOKKOS_STD_ALGORITHMS_REMOVE_COPY_IF_HPP - -#include "impl/Kokkos_RemoveAllVariants.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator remove_copy_if(const ExecutionSpace& ex, - InputIterator first_from, InputIterator last_from, - OutputIterator first_dest, - const UnaryPredicate& pred) { - return Impl::remove_copy_if_impl( - "Kokkos::remove_copy_if_iterator_api_default", ex, first_from, last_from, - first_dest, pred); -} - -template -OutputIterator remove_copy_if(const std::string& label, - const ExecutionSpace& ex, - InputIterator first_from, InputIterator last_from, - OutputIterator first_dest, - const UnaryPredicate& pred) { - return Impl::remove_copy_if_impl(label, ex, first_from, last_from, first_dest, - pred); -} - -template -auto remove_copy_if(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - const UnaryPredicate& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - - return Impl::remove_copy_if_impl( - "Kokkos::remove_copy_if_iterator_api_default", ex, - ::Kokkos::Experimental::cbegin(view_from), - ::Kokkos::Experimental::cend(view_from), - ::Kokkos::Experimental::begin(view_dest), pred); -} - -template -auto remove_copy_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - const UnaryPredicate& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - - return Impl::remove_copy_if_impl( - label, ex, ::Kokkos::Experimental::cbegin(view_from), - ::Kokkos::Experimental::cend(view_from), - ::Kokkos::Experimental::begin(view_dest), pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveIf.hpp deleted file mode 100644 index e4171ca917..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RemoveIf.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_IF_HPP -#define KOKKOS_STD_ALGORITHMS_REMOVE_IF_HPP - -#include "impl/Kokkos_RemoveAllVariants.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -Iterator remove_if(const ExecutionSpace& ex, Iterator first, Iterator last, - UnaryPredicate pred) { - return Impl::remove_if_impl("Kokkos::remove_if_iterator_api_default", ex, - first, last, pred); -} - -template -Iterator remove_if(const std::string& label, const ExecutionSpace& ex, - Iterator first, Iterator last, UnaryPredicate pred) { - return Impl::remove_if_impl(label, ex, first, last, pred); -} - -template -auto remove_if(const ExecutionSpace& ex, - const ::Kokkos::View& view, - UnaryPredicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::remove_if_impl("Kokkos::remove_if_iterator_api_default", ex, - ::Kokkos::Experimental::begin(view), - ::Kokkos::Experimental::end(view), pred); -} - -template -auto remove_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - UnaryPredicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::remove_if_impl(label, ex, ::Kokkos::Experimental::begin(view), - ::Kokkos::Experimental::end(view), pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Replace.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Replace.hpp deleted file mode 100644 index 10ca46af25..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Replace.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_HPP - -#include "impl/Kokkos_Replace.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -void replace(const ExecutionSpace& ex, Iterator first, Iterator last, - const ValueType& old_value, const ValueType& new_value) { - return Impl::replace_impl("Kokkos::replace_iterator_api", ex, first, last, - old_value, new_value); -} - -template -void replace(const std::string& label, const ExecutionSpace& ex, Iterator first, - Iterator last, const ValueType& old_value, - const ValueType& new_value) { - return Impl::replace_impl(label, ex, first, last, old_value, new_value); -} - -template -void replace(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ValueType& old_value, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_impl("Kokkos::replace_view_api", ex, KE::begin(view), - KE::end(view), old_value, new_value); -} - -template -void replace(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ValueType& old_value, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_impl(label, ex, KE::begin(view), KE::end(view), - old_value, new_value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceCopy.hpp deleted file mode 100644 index f5136eb438..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceCopy.hpp +++ /dev/null @@ -1,107 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_COPY_HPP - -#include "impl/Kokkos_ReplaceCopy.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator replace_copy(const ExecutionSpace& ex, InputIterator first_from, - InputIterator last_from, OutputIterator first_dest, - const ValueType& old_value, - const ValueType& new_value) { - return Impl::replace_copy_impl("Kokkos::replace_copy_iterator_api", ex, - first_from, last_from, first_dest, old_value, - new_value); -} - -template -OutputIterator replace_copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first_from, InputIterator last_from, - OutputIterator first_dest, - const ValueType& old_value, - const ValueType& new_value) { - return Impl::replace_copy_impl(label, ex, first_from, last_from, first_dest, - old_value, new_value); -} - -template -auto replace_copy(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - const ValueType& old_value, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_copy_impl("Kokkos::replace_copy_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), old_value, new_value); -} - -template -auto replace_copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - const ValueType& old_value, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_copy_impl(label, ex, KE::cbegin(view_from), - KE::cend(view_from), KE::begin(view_dest), - old_value, new_value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceCopyIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceCopyIf.hpp deleted file mode 100644 index a3f3fe69ab..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceCopyIf.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IF_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IF_HPP - -#include "impl/Kokkos_ReplaceCopyIf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator replace_copy_if(const ExecutionSpace& ex, - InputIterator first_from, - InputIterator last_from, - OutputIterator first_dest, PredicateType pred, - const ValueType& new_value) { - return Impl::replace_copy_if_impl("Kokkos::replace_copy_if_iterator_api", ex, - first_from, last_from, first_dest, pred, - new_value); -} - -template -OutputIterator replace_copy_if(const std::string& label, - const ExecutionSpace& ex, - InputIterator first_from, - InputIterator last_from, - OutputIterator first_dest, PredicateType pred, - const ValueType& new_value) { - return Impl::replace_copy_if_impl(label, ex, first_from, last_from, - first_dest, pred, new_value); -} - -template -auto replace_copy_if(const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - PredicateType pred, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_copy_if_impl("Kokkos::replace_copy_if_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), pred, new_value); -} - -template -auto replace_copy_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - PredicateType pred, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_copy_if_impl(label, ex, KE::cbegin(view_from), - KE::cend(view_from), KE::begin(view_dest), - pred, new_value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceIf.hpp deleted file mode 100644 index bdb59f28af..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReplaceIf.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_IF_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_IF_HPP - -#include "impl/Kokkos_ReplaceIf.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -void replace_if(const ExecutionSpace& ex, InputIterator first, - InputIterator last, Predicate pred, - const ValueType& new_value) { - return Impl::replace_if_impl("Kokkos::replace_if_iterator_api", ex, first, - last, pred, new_value); -} - -template -void replace_if(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, Predicate pred, - const ValueType& new_value) { - return Impl::replace_if_impl(label, ex, first, last, pred, new_value); -} - -template -void replace_if(const ExecutionSpace& ex, - const ::Kokkos::View& view, - Predicate pred, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_if_impl("Kokkos::replace_if_view_api", ex, - KE::begin(view), KE::end(view), pred, new_value); -} - -template -void replace_if(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - Predicate pred, const ValueType& new_value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - namespace KE = ::Kokkos::Experimental; - return Impl::replace_if_impl(label, ex, KE::begin(view), KE::end(view), pred, - new_value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Reverse.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Reverse.hpp deleted file mode 100644 index 4848b20f66..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Reverse.hpp +++ /dev/null @@ -1,87 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REVERSE_HPP -#define KOKKOS_STD_ALGORITHMS_REVERSE_HPP - -#include "impl/Kokkos_Reverse.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -void reverse(const ExecutionSpace& ex, InputIterator first, - InputIterator last) { - return Impl::reverse_impl("Kokkos::reverse_iterator_api_default", ex, first, - last); -} - -template -void reverse(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last) { - return Impl::reverse_impl(label, ex, first, last); -} - -template -void reverse(const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - namespace KE = ::Kokkos::Experimental; - return Impl::reverse_impl("Kokkos::reverse_view_api_default", ex, - KE::begin(view), KE::end(view)); -} - -template -void reverse(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - namespace KE = ::Kokkos::Experimental; - return Impl::reverse_impl(label, ex, KE::begin(view), KE::end(view)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReverseCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReverseCopy.hpp deleted file mode 100644 index bb4462bf41..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ReverseCopy.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REVERSE_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_REVERSE_COPY_HPP - -#include "impl/Kokkos_ReverseCopy.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator reverse_copy(const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first) { - return Impl::reverse_copy_impl("Kokkos::reverse_copy_iterator_api_default", - ex, first, last, d_first); -} - -template -OutputIterator reverse_copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first) { - return Impl::reverse_copy_impl(label, ex, first, last, d_first); -} - -template -auto reverse_copy(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::reverse_copy_impl("Kokkos::reverse_copy_view_api_default", ex, - cbegin(source), cend(source), begin(dest)); -} - -template -auto reverse_copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::reverse_copy_impl(label, ex, cbegin(source), cend(source), - begin(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Rotate.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Rotate.hpp deleted file mode 100644 index 39975811a4..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Rotate.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ROTATE_HPP -#define KOKKOS_STD_ALGORITHMS_ROTATE_HPP - -#include "impl/Kokkos_Rotate.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType rotate(const ExecutionSpace& ex, IteratorType first, - IteratorType n_first, IteratorType last) { - return Impl::rotate_impl("Kokkos::rotate_iterator_api_default", ex, first, - n_first, last); -} - -template -IteratorType rotate(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType n_first, - IteratorType last) { - return Impl::rotate_impl(label, ex, first, n_first, last); -} - -template -auto rotate(const ExecutionSpace& ex, - const ::Kokkos::View& view, - std::size_t n_location) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::rotate_impl("Kokkos::rotate_view_api_default", ex, begin(view), - begin(view) + n_location, end(view)); -} - -template -auto rotate(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - std::size_t n_location) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::rotate_impl(label, ex, begin(view), begin(view) + n_location, - end(view)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RotateCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RotateCopy.hpp deleted file mode 100644 index f98686ab63..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_RotateCopy.hpp +++ /dev/null @@ -1,100 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ROTATE_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_ROTATE_COPY_HPP - -#include "impl/Kokkos_RotateCopy.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -OutputIterator rotate_copy(const ExecutionSpace& ex, InputIterator first, - InputIterator n_first, InputIterator last, - OutputIterator d_first) { - return Impl::rotate_copy_impl("Kokkos::rotate_copy_iterator_api_default", ex, - first, n_first, last, d_first); -} - -template -OutputIterator rotate_copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator n_first, - InputIterator last, OutputIterator d_first) { - return Impl::rotate_copy_impl(label, ex, first, n_first, last, d_first); -} - -template -auto rotate_copy(const ExecutionSpace& ex, - const ::Kokkos::View& source, - std::size_t n_location, - const ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::rotate_copy_impl("Kokkos::rotate_copy_view_api_default", ex, - cbegin(source), cbegin(source) + n_location, - cend(source), begin(dest)); -} - -template -auto rotate_copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - std::size_t n_location, - const ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::rotate_copy_impl(label, ex, cbegin(source), - cbegin(source) + n_location, cend(source), - begin(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Search.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Search.hpp deleted file mode 100644 index ce656da31c..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Search.hpp +++ /dev/null @@ -1,148 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SEARCH_HPP -#define KOKKOS_STD_ALGORITHMS_SEARCH_HPP - -#include "impl/Kokkos_Search.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1: no binary predicate passed -template -IteratorType1 search(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last) { - return Impl::search_impl("Kokkos::search_iterator_api_default", ex, first, - last, s_first, s_last); -} - -template -IteratorType1 search(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last) { - return Impl::search_impl(label, ex, first, last, s_first, s_last); -} - -template -auto search(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_impl("Kokkos::search_view_api_default", ex, - KE::begin(view), KE::end(view), KE::begin(s_view), - KE::end(s_view)); -} - -template -auto search(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_impl(label, ex, KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view)); -} - -// overload set 2: binary predicate passed -template -IteratorType1 search(const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last, const BinaryPredicateType& pred) { - return Impl::search_impl("Kokkos::search_iterator_api_default", ex, first, - last, s_first, s_last, pred); -} - -template -IteratorType1 search(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last, - const BinaryPredicateType& pred) { - return Impl::search_impl(label, ex, first, last, s_first, s_last, pred); -} - -template -auto search(const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_impl("Kokkos::search_view_api_default", ex, - KE::begin(view), KE::end(view), KE::begin(s_view), - KE::end(s_view), pred); -} - -template -auto search(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - const ::Kokkos::View& s_view, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_impl(label, ex, KE::begin(view), KE::end(view), - KE::begin(s_view), KE::end(s_view), pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_SearchN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_SearchN.hpp deleted file mode 100644 index 854d911e76..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_SearchN.hpp +++ /dev/null @@ -1,144 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SEARCH_N_HPP -#define KOKKOS_STD_ALGORITHMS_SEARCH_N_HPP - -#include "impl/Kokkos_SearchN.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1: no binary predicate passed -template -IteratorType search_n(const ExecutionSpace& ex, IteratorType first, - IteratorType last, SizeType count, - const ValueType& value) { - return Impl::search_n_impl("Kokkos::search_n_iterator_api_default", ex, first, - last, count, value); -} - -template -IteratorType search_n(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, SizeType count, - const ValueType& value) { - return Impl::search_n_impl(label, ex, first, last, count, value); -} - -template -auto search_n(const ExecutionSpace& ex, - const ::Kokkos::View& view, - SizeType count, const ValueType& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_n_impl("Kokkos::search_n_view_api_default", ex, - KE::begin(view), KE::end(view), count, value); -} - -template -auto search_n(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - SizeType count, const ValueType& value) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_n_impl(label, ex, KE::begin(view), KE::end(view), count, - value); -} - -// overload set 2: binary predicate passed -template -IteratorType search_n(const ExecutionSpace& ex, IteratorType first, - IteratorType last, SizeType count, const ValueType& value, - const BinaryPredicateType& pred) { - return Impl::search_n_impl("Kokkos::search_n_iterator_api_default", ex, first, - last, count, value, pred); -} - -template -IteratorType search_n(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, SizeType count, - const ValueType& value, const BinaryPredicateType& pred) { - return Impl::search_n_impl(label, ex, first, last, count, value, pred); -} - -template -auto search_n(const ExecutionSpace& ex, - const ::Kokkos::View& view, - SizeType count, const ValueType& value, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_n_impl("Kokkos::search_n_view_api_default", ex, - KE::begin(view), KE::end(view), count, value, - pred); -} - -template -auto search_n(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - SizeType count, const ValueType& value, - const BinaryPredicateType& pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - namespace KE = ::Kokkos::Experimental; - return Impl::search_n_impl(label, ex, KE::begin(view), KE::end(view), count, - value, pred); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ShiftLeft.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ShiftLeft.hpp deleted file mode 100644 index cee111af98..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ShiftLeft.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SHIFT_LEFT_HPP -#define KOKKOS_STD_ALGORITHMS_SHIFT_LEFT_HPP - -#include "impl/Kokkos_ShiftLeft.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType shift_left(const ExecutionSpace& ex, IteratorType first, - IteratorType last, - typename IteratorType::difference_type n) { - return Impl::shift_left_impl("Kokkos::shift_left_iterator_api_default", ex, - first, last, n); -} - -template -IteratorType shift_left(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - typename IteratorType::difference_type n) { - return Impl::shift_left_impl(label, ex, first, last, n); -} - -template -auto shift_left(const ExecutionSpace& ex, - const ::Kokkos::View& view, - typename decltype(begin(view))::difference_type n) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::shift_left_impl("Kokkos::shift_left_view_api_default", ex, - begin(view), end(view), n); -} - -template -auto shift_left(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - typename decltype(begin(view))::difference_type n) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::shift_left_impl(label, ex, begin(view), end(view), n); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ShiftRight.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ShiftRight.hpp deleted file mode 100644 index f104d2bd7a..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_ShiftRight.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SHIFT_RIGHT_HPP -#define KOKKOS_STD_ALGORITHMS_SHIFT_RIGHT_HPP - -#include "impl/Kokkos_ShiftRight.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType shift_right(const ExecutionSpace& ex, IteratorType first, - IteratorType last, - typename IteratorType::difference_type n) { - return Impl::shift_right_impl("Kokkos::shift_right_iterator_api_default", ex, - first, last, n); -} - -template -IteratorType shift_right(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - typename IteratorType::difference_type n) { - return Impl::shift_right_impl(label, ex, first, last, n); -} - -template -auto shift_right(const ExecutionSpace& ex, - const ::Kokkos::View& view, - typename decltype(begin(view))::difference_type n) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::shift_right_impl("Kokkos::shift_right_view_api_default", ex, - begin(view), end(view), n); -} - -template -auto shift_right(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - typename decltype(begin(view))::difference_type n) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::shift_right_impl(label, ex, begin(view), end(view), n); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Swap.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Swap.hpp deleted file mode 100644 index 9006aa9167..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Swap.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SWAP_HPP -#define KOKKOS_STD_ALGORITHMS_SWAP_HPP - -#include - -namespace Kokkos { -namespace Experimental { - -// swap -template -KOKKOS_INLINE_FUNCTION void swap(T& a, T& b) noexcept { - static_assert( - std::is_move_assignable::value && std::is_move_constructible::value, - "Kokkos::Experimental::swap arguments must be move assignable " - "and move constructible"); - - T tmp = std::move(a); - a = std::move(b); - b = std::move(tmp); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_SwapRanges.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_SwapRanges.hpp deleted file mode 100644 index 2997cdab4b..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_SwapRanges.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SWAP_RANGES_HPP -#define KOKKOS_STD_ALGORITHMS_SWAP_RANGES_HPP - -#include "impl/Kokkos_SwapRanges.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -IteratorType2 swap_ranges(const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2) { - return Impl::swap_ranges_impl("Kokkos::swap_ranges_iterator_api_default", ex, - first1, last1, first2); -} - -template -auto swap_ranges(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - assert(source.extent(0) == dest.extent(0)); - return Impl::swap_ranges_impl("Kokkos::swap_ranges_view_api_default", ex, - begin(source), end(source), begin(dest)); -} - -template -IteratorType2 swap_ranges(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2) { - return Impl::swap_ranges_impl(label, ex, first1, last1, first2); -} - -template -auto swap_ranges(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - assert(source.extent(0) == dest.extent(0)); - return Impl::swap_ranges_impl(label, ex, begin(source), end(source), - begin(dest)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Transform.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Transform.hpp deleted file mode 100644 index 6dfb83a8c0..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Transform.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_HPP -#define KOKKOS_STD_ALGORITHMS_TRANSFORM_HPP - -#include "impl/Kokkos_Transform.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - InputIterator, OutputIterator>::value, - OutputIterator> -transform(const ExecutionSpace& ex, InputIterator first1, InputIterator last1, - OutputIterator d_first, UnaryOperation unary_op) { - return Impl::transform_impl("Kokkos::transform_iterator_api_default", ex, - first1, last1, d_first, std::move(unary_op)); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - InputIterator, OutputIterator>::value, - OutputIterator> -transform(const std::string& label, const ExecutionSpace& ex, - InputIterator first1, InputIterator last1, OutputIterator d_first, - UnaryOperation unary_op) { - return Impl::transform_impl(label, ex, first1, last1, d_first, - std::move(unary_op)); -} - -template -auto transform(const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest, - UnaryOperation unary_op) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::transform_impl("Kokkos::transform_view_api_default", ex, - begin(source), end(source), begin(dest), - std::move(unary_op)); -} - -template -auto transform(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - ::Kokkos::View& dest, - UnaryOperation unary_op) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::transform_impl(label, ex, begin(source), end(source), - begin(dest), std::move(unary_op)); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - InputIterator1, InputIterator2, OutputIterator>::value, - OutputIterator> -transform(const ExecutionSpace& ex, InputIterator1 first1, InputIterator1 last1, - InputIterator2 first2, OutputIterator d_first, - BinaryOperation binary_op) { - return Impl::transform_impl("Kokkos::transform_iterator_api_default", ex, - first1, last1, first2, d_first, - std::move(binary_op)); -} - -template -std::enable_if_t< ::Kokkos::Experimental::Impl::are_iterators< - InputIterator1, InputIterator2, OutputIterator>::value, - OutputIterator> -transform(const std::string& label, const ExecutionSpace& ex, - InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, - OutputIterator d_first, BinaryOperation binary_op) { - return Impl::transform_impl(label, ex, first1, last1, first2, d_first, - std::move(binary_op)); -} - -template -auto transform(const ExecutionSpace& ex, - const ::Kokkos::View& source1, - const ::Kokkos::View& source2, - ::Kokkos::View& dest, - BinaryOperation binary_op) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source2); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::transform_impl("Kokkos::transform_view_api_default", ex, - begin(source1), end(source1), begin(source2), - begin(dest), std::move(binary_op)); -} - -template -auto transform(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source1, - const ::Kokkos::View& source2, - ::Kokkos::View& dest, - BinaryOperation binary_op) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source1); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source2); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::transform_impl(label, ex, begin(source1), end(source1), - begin(source2), begin(dest), - std::move(binary_op)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformExclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformExclusiveScan.hpp deleted file mode 100644 index d0073599b0..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformExclusiveScan.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRASFORM_EXCLUSIVE_SCAN_HPP -#define KOKKOS_STD_ALGORITHMS_TRASFORM_EXCLUSIVE_SCAN_HPP - -#include "impl/Kokkos_TransformExclusiveScan.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -transform_exclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - ValueType init_value, BinaryOpType binary_op, - UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - return Impl::transform_exclusive_scan_impl( - "Kokkos::transform_exclusive_scan_custom_functors_iterator_api", ex, - first, last, first_dest, init_value, binary_op, unary_op); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -transform_exclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, ValueType init_value, - BinaryOpType binary_op, UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - return Impl::transform_exclusive_scan_impl(label, ex, first, last, first_dest, - init_value, binary_op, unary_op); -} - -template -auto transform_exclusive_scan( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - ValueType init_value, BinaryOpType binary_op, UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - namespace KE = ::Kokkos::Experimental; - return Impl::transform_exclusive_scan_impl( - "Kokkos::transform_exclusive_scan_custom_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - init_value, binary_op, unary_op); -} - -template -auto transform_exclusive_scan( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - ValueType init_value, BinaryOpType binary_op, UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - namespace KE = ::Kokkos::Experimental; - return Impl::transform_exclusive_scan_impl( - label, ex, KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), init_value, binary_op, unary_op); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformInclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformInclusiveScan.hpp deleted file mode 100644 index 088e162adb..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformInclusiveScan.hpp +++ /dev/null @@ -1,190 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_INCLUSIVE_SCAN_HPP -#define KOKKOS_STD_ALGORITHMS_TRANSFORM_INCLUSIVE_SCAN_HPP - -#include "impl/Kokkos_TransformInclusiveScan.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set 1 (no init value) -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -transform_inclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - BinaryOpType binary_op, UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::transform_inclusive_scan_impl( - "Kokkos::transform_inclusive_scan_custom_functors_iterator_api", ex, - first, last, first_dest, binary_op, unary_op); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -transform_inclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, BinaryOpType binary_op, - UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - - return Impl::transform_inclusive_scan_impl(label, ex, first, last, first_dest, - binary_op, unary_op); -} - -template -auto transform_inclusive_scan( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOpType binary_op, UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::transform_inclusive_scan_impl( - "Kokkos::transform_inclusive_scan_custom_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - binary_op, unary_op); -} - -template -auto transform_inclusive_scan( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOpType binary_op, UnaryOpType unary_op) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::transform_inclusive_scan_impl( - label, ex, KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), binary_op, unary_op); -} - -// overload set 2 (init value) -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -transform_inclusive_scan(const ExecutionSpace& ex, InputIteratorType first, - InputIteratorType last, OutputIteratorType first_dest, - BinaryOpType binary_op, UnaryOpType unary_op, - ValueType init_value) { - Impl::static_assert_is_not_openmptarget(ex); - return Impl::transform_inclusive_scan_impl( - "Kokkos::transform_inclusive_scan_custom_functors_iterator_api", ex, - first, last, first_dest, binary_op, unary_op, init_value); -} - -template -std::enable_if_t<::Kokkos::Experimental::Impl::are_iterators< - InputIteratorType, OutputIteratorType>::value, - OutputIteratorType> -transform_inclusive_scan(const std::string& label, const ExecutionSpace& ex, - InputIteratorType first, InputIteratorType last, - OutputIteratorType first_dest, BinaryOpType binary_op, - UnaryOpType unary_op, ValueType init_value) { - Impl::static_assert_is_not_openmptarget(ex); - return Impl::transform_inclusive_scan_impl(label, ex, first, last, first_dest, - binary_op, unary_op, init_value); -} - -template -auto transform_inclusive_scan( - const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::transform_inclusive_scan_impl( - "Kokkos::transform_inclusive_scan_custom_functors_view_api", ex, - KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), - binary_op, unary_op, init_value); -} - -template -auto transform_inclusive_scan( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view_from, - const ::Kokkos::View& view_dest, - BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { - Impl::static_assert_is_not_openmptarget(ex); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); - namespace KE = ::Kokkos::Experimental; - return Impl::transform_inclusive_scan_impl( - label, ex, KE::cbegin(view_from), KE::cend(view_from), - KE::begin(view_dest), binary_op, unary_op, init_value); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp deleted file mode 100644 index 5caced5917..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp +++ /dev/null @@ -1,283 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_REDUCE_HPP -#define KOKKOS_STD_ALGORITHMS_TRANSFORM_REDUCE_HPP - -#include "impl/Kokkos_TransformReduce.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// ---------------------------- -// overload set1: -// no custom functors passed, so equivalent to -// transform_reduce(first1, last1, first2, init, plus<>(), multiplies<>()); -// ---------------------------- -template -ValueType transform_reduce(const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, - ValueType init_reduction_value) { - return Impl::transform_reduce_default_functors_impl( - "Kokkos::transform_reduce_default_functors_iterator_api", ex, first1, - last1, first2, std::move(init_reduction_value)); -} - -template -ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, - ValueType init_reduction_value) { - return Impl::transform_reduce_default_functors_impl( - label, ex, first1, last1, first2, std::move(init_reduction_value)); -} - -// overload1 accepting views -template -ValueType transform_reduce( - const ExecutionSpace& ex, - const ::Kokkos::View& first_view, - const ::Kokkos::View& second_view, - ValueType init_reduction_value) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); - - return Impl::transform_reduce_default_functors_impl( - "Kokkos::transform_reduce_default_functors_iterator_api", ex, - KE::cbegin(first_view), KE::cend(first_view), KE::cbegin(second_view), - std::move(init_reduction_value)); -} - -template -ValueType transform_reduce( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& first_view, - const ::Kokkos::View& second_view, - ValueType init_reduction_value) { - namespace KE = ::Kokkos::Experimental; - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); - - return Impl::transform_reduce_default_functors_impl( - label, ex, KE::cbegin(first_view), KE::cend(first_view), - KE::cbegin(second_view), std::move(init_reduction_value)); -} - -// -// overload set2: -// accepts a custom transform and joiner functor -// - -// Note the std refers to the arg BinaryReductionOp -// but in the Kokkos naming convention, it corresponds -// to a "joiner" that knows how to join two values -// NOTE: "joiner/transformer" need to be commutative. - -// https://en.cppreference.com/w/cpp/algorithm/transform_reduce - -// api accepting iterators -template -ValueType transform_reduce(const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, - ValueType init_reduction_value, - BinaryJoinerType joiner, - BinaryTransform transformer) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::transform_reduce_custom_functors_impl( - "Kokkos::transform_reduce_custom_functors_iterator_api", ex, first1, - last1, first2, std::move(init_reduction_value), std::move(joiner), - std::move(transformer)); -} - -template -ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, ValueType init_reduction_value, - BinaryJoinerType joiner, - BinaryTransform transformer) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::transform_reduce_custom_functors_impl( - label, ex, first1, last1, first2, std::move(init_reduction_value), - std::move(joiner), std::move(transformer)); -} - -// accepting views -template -ValueType transform_reduce( - const ExecutionSpace& ex, - const ::Kokkos::View& first_view, - const ::Kokkos::View& second_view, - ValueType init_reduction_value, BinaryJoinerType joiner, - BinaryTransform transformer) { - namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); - - return Impl::transform_reduce_custom_functors_impl( - "Kokkos::transform_reduce_custom_functors_view_api", ex, - KE::cbegin(first_view), KE::cend(first_view), KE::cbegin(second_view), - std::move(init_reduction_value), std::move(joiner), - std::move(transformer)); -} - -template -ValueType transform_reduce( - const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& first_view, - const ::Kokkos::View& second_view, - ValueType init_reduction_value, BinaryJoinerType joiner, - BinaryTransform transformer) { - namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); - - return Impl::transform_reduce_custom_functors_impl( - label, ex, KE::cbegin(first_view), KE::cend(first_view), - KE::cbegin(second_view), std::move(init_reduction_value), - std::move(joiner), std::move(transformer)); -} - -// -// overload set3: -// -// accepting iterators -template -// need this to avoid ambiguous call -std::enable_if_t< - ::Kokkos::Experimental::Impl::are_iterators::value, ValueType> -transform_reduce(const ExecutionSpace& ex, IteratorType first1, - IteratorType last1, ValueType init_reduction_value, - BinaryJoinerType joiner, UnaryTransform transformer) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::transform_reduce_custom_functors_impl( - "Kokkos::transform_reduce_custom_functors_iterator_api", ex, first1, - last1, std::move(init_reduction_value), std::move(joiner), - std::move(transformer)); -} - -template -// need this to avoid ambiguous call -std::enable_if_t< - ::Kokkos::Experimental::Impl::are_iterators::value, ValueType> -transform_reduce(const std::string& label, const ExecutionSpace& ex, - IteratorType first1, IteratorType last1, - ValueType init_reduction_value, BinaryJoinerType joiner, - UnaryTransform transformer) { - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - return Impl::transform_reduce_custom_functors_impl( - label, ex, first1, last1, std::move(init_reduction_value), - std::move(joiner), std::move(transformer)); -} - -// accepting views -template -ValueType transform_reduce(const ExecutionSpace& ex, - const ::Kokkos::View& view, - ValueType init_reduction_value, - BinaryJoinerType joiner, - UnaryTransform transformer) { - namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::transform_reduce_custom_functors_impl( - "Kokkos::transform_reduce_custom_functors_view_api", ex, KE::cbegin(view), - KE::cend(view), std::move(init_reduction_value), std::move(joiner), - std::move(transformer)); -} - -template -ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - ValueType init_reduction_value, - BinaryJoinerType joiner, - UnaryTransform transformer) { - namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, - "ValueType must be move constructible."); - - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - - return Impl::transform_reduce_custom_functors_impl( - label, ex, KE::cbegin(view), KE::cend(view), - std::move(init_reduction_value), std::move(joiner), - std::move(transformer)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Unique.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Unique.hpp deleted file mode 100644 index aeb54a6ffe..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_Unique.hpp +++ /dev/null @@ -1,124 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_UNIQUE_HPP -#define KOKKOS_STD_ALGORITHMS_UNIQUE_HPP - -#include "impl/Kokkos_Unique.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// note: the enable_if below is to avoid "call to ... is ambiguous" -// for example in the unit test when using a variadic function - -// overload set1 -template -std::enable_if_t::value, IteratorType> unique( - const ExecutionSpace& ex, IteratorType first, IteratorType last) { - return Impl::unique_impl("Kokkos::unique_iterator_api_default", ex, first, - last); -} - -template -std::enable_if_t::value, IteratorType> unique( - const std::string& label, const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - return Impl::unique_impl(label, ex, first, last); -} - -template -auto unique(const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return ::Kokkos::Experimental::unique("Kokkos::unique_view_api_default", ex, - begin(view), end(view)); -} - -template -auto unique(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return ::Kokkos::Experimental::unique(label, ex, begin(view), end(view)); -} - -// overload set2 -template -IteratorType unique(const ExecutionSpace& ex, IteratorType first, - IteratorType last, BinaryPredicate pred) { - return Impl::unique_impl("Kokkos::unique_iterator_api_default", ex, first, - last, pred); -} - -template -IteratorType unique(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - BinaryPredicate pred) { - return Impl::unique_impl(label, ex, first, last, pred); -} - -template -auto unique(const ExecutionSpace& ex, - const ::Kokkos::View& view, - BinaryPredicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::unique_impl("Kokkos::unique_view_api_default", ex, begin(view), - end(view), std::move(pred)); -} - -template -auto unique(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& view, - BinaryPredicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); - return Impl::unique_impl(label, ex, begin(view), end(view), std::move(pred)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_UniqueCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_UniqueCopy.hpp deleted file mode 100644 index 632b560fa8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_UniqueCopy.hpp +++ /dev/null @@ -1,143 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_UNIQUE_COPY_HPP -#define KOKKOS_STD_ALGORITHMS_UNIQUE_COPY_HPP - -#include "impl/Kokkos_UniqueCopy.hpp" -#include "Kokkos_BeginEnd.hpp" - -namespace Kokkos { -namespace Experimental { - -// overload set1 -template -std::enable_if_t::value, OutputIterator> -unique_copy(const ExecutionSpace& ex, InputIterator first, InputIterator last, - OutputIterator d_first) { - return Impl::unique_copy_impl("Kokkos::unique_copy_iterator_api_default", ex, - first, last, d_first); -} - -template -std::enable_if_t::value, OutputIterator> -unique_copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, OutputIterator d_first) { - return Impl::unique_copy_impl(label, ex, first, last, d_first); -} - -template -auto unique_copy(const ExecutionSpace& ex, - const ::Kokkos::View& source, - const ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return ::Kokkos::Experimental::unique_copy( - "Kokkos::unique_copy_view_api_default", ex, cbegin(source), cend(source), - begin(dest)); -} - -template -auto unique_copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - const ::Kokkos::View& dest) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return ::Kokkos::Experimental::unique_copy(label, ex, cbegin(source), - cend(source), begin(dest)); -} - -// overload set2 -template -OutputIterator unique_copy(const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first, - BinaryPredicate pred) { - return Impl::unique_copy_impl("Kokkos::unique_copy_iterator_api_default", ex, - first, last, d_first, pred); -} - -template -OutputIterator unique_copy(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first, BinaryPredicate pred) { - return Impl::unique_copy_impl(label, ex, first, last, d_first, pred); -} - -template -auto unique_copy(const ExecutionSpace& ex, - const ::Kokkos::View& source, - const ::Kokkos::View& dest, - BinaryPredicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::unique_copy_impl("Kokkos::unique_copy_view_api_default", ex, - cbegin(source), cend(source), begin(dest), - std::move(pred)); -} - -template -auto unique_copy(const std::string& label, const ExecutionSpace& ex, - const ::Kokkos::View& source, - const ::Kokkos::View& dest, - BinaryPredicate pred) { - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); - Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); - - return Impl::unique_copy_impl(label, ex, cbegin(source), cend(source), - begin(dest), std::move(pred)); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AdjacentDifference.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AdjacentDifference.hpp deleted file mode 100644 index 35c78b86bf..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AdjacentDifference.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdAdjacentDifferenceDefaultBinaryOpFunctor { - KOKKOS_FUNCTION - constexpr RetType operator()(const ValueType1& a, const ValueType2& b) const { - return a - b; - } -}; - -template -struct StdAdjacentDiffFunctor { - using index_type = typename InputIteratorType::difference_type; - - const InputIteratorType m_first_from; - const OutputIteratorType m_first_dest; - BinaryOperator m_op; - - KOKKOS_FUNCTION - void operator()(const index_type i) const { - const auto& my_value = m_first_from[i]; - if (i == 0) { - m_first_dest[i] = my_value; - } else { - const auto& left_value = m_first_from[i - 1]; - m_first_dest[i] = m_op(my_value, left_value); - } - } - - KOKKOS_FUNCTION - StdAdjacentDiffFunctor(InputIteratorType first_from, - OutputIteratorType first_dest, BinaryOperator op) - : m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)), - m_op(std::move(op)) {} -}; - -template -OutputIteratorType adjacent_difference_impl(const std::string& label, - const ExecutionSpace& ex, - InputIteratorType first_from, - InputIteratorType last_from, - OutputIteratorType first_dest, - BinaryOp bin_op) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - if (first_from == last_from) { - return first_dest; - } - - // aliases - using value_type = typename OutputIteratorType::value_type; - using aux_view_type = ::Kokkos::View; - using functor_t = - StdAdjacentDiffFunctor; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - aux_view_type aux_view("aux_view", num_elements); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - functor_t(first_from, first_dest, bin_op)); - ex.fence("Kokkos::adjacent_difference: fence after operation"); - - // return - return first_dest + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AdjacentFind.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AdjacentFind.hpp deleted file mode 100644 index 155f6c7bb8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AdjacentFind.hpp +++ /dev/null @@ -1,140 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdAdjacentFindFunctor { - using red_value_type = typename ReducerType::value_type; - - IteratorType m_first; - ReducerType m_reducer; - PredicateType m_p; - - KOKKOS_FUNCTION - void operator()(const IndexType i, red_value_type& red_value) const { - const auto& my_value = m_first[i]; - const auto& next_value = m_first[i + 1]; - const bool are_equal = m_p(my_value, next_value); - - auto rv = - are_equal - ? red_value_type{i} - : red_value_type{::Kokkos::reduction_identity::min()}; - - m_reducer.join(red_value, rv); - } - - KOKKOS_FUNCTION - StdAdjacentFindFunctor(IteratorType first, ReducerType reducer, - PredicateType p) - : m_first(std::move(first)), - m_reducer(std::move(reducer)), - m_p(std::move(p)) {} -}; - -template -IteratorType adjacent_find_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType first, - IteratorType last, PredicateType pred) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - const auto num_elements = Kokkos::Experimental::distance(first, last); - - if (num_elements <= 1) { - return last; - } - - using index_type = typename IteratorType::difference_type; - using reducer_type = FirstLoc; - using reduction_value_type = typename reducer_type::value_type; - using func_t = StdAdjacentFindFunctor; - - reduction_value_type red_result; - reducer_type reducer(red_result); - - // note that we use below num_elements-1 because - // each index i in the reduction checks i and (i+1). - ::Kokkos::parallel_reduce( - label, RangePolicy(ex, 0, num_elements - 1), - func_t(first, reducer, pred), reducer); - - // fence not needed because reducing into scalar - if (red_result.min_loc_true == - ::Kokkos::reduction_identity::min()) { - return last; - } else { - return first + red_result.min_loc_true; - } -} - -template -IteratorType adjacent_find_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - using value_type = typename IteratorType::value_type; - using default_pred_t = StdAlgoEqualBinaryPredicate; - return adjacent_find_impl(label, ex, first, last, default_pred_t()); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AllOfAnyOfNoneOf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AllOfAnyOfNoneOf.hpp deleted file mode 100644 index dd8ae4f5b4..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_AllOfAnyOfNoneOf.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_ALL_OF_ANY_OF_NONE_OF_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_ALL_OF_ANY_OF_NONE_OF_IMPL_HPP - -#include "Kokkos_FindIfOrNot.hpp" - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -bool all_of_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, Predicate predicate) { - return (find_if_or_not_impl(label, ex, first, last, predicate) == - last); -} - -template -bool any_of_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, Predicate predicate) { - return (find_if_or_not_impl(label, ex, first, last, predicate) != last); -} - -template -bool none_of_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, Predicate predicate) { - return (find_if_or_not_impl(label, ex, first, last, predicate) == last); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyBackward.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyBackward.hpp deleted file mode 100644 index 18f614094c..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyBackward.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdCopyBackwardFunctor { - static_assert(std::is_signed::value, - "Kokkos: StdCopyBackwardFunctor requires signed index type"); - - IteratorType1 m_last; - IteratorType2 m_dest_last; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { m_dest_last[-i - 1] = m_last[-i - 1]; } - - KOKKOS_FUNCTION - StdCopyBackwardFunctor(IteratorType1 _last, IteratorType2 _dest_last) - : m_last(std::move(_last)), m_dest_last(std::move(_dest_last)) {} -}; - -template -IteratorType2 copy_backward_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 d_last) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_last); - Impl::static_assert_iterators_have_matching_difference_type(first, d_last); - Impl::expect_valid_range(first, last); - - // aliases - using index_type = typename IteratorType1::difference_type; - using func_t = - StdCopyBackwardFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(last, d_last)); - ex.fence("Kokkos::copy_backward: fence after operation"); - - // return - return d_last - num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyCopyN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyCopyN.hpp deleted file mode 100644 index 03b6fc6eca..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyCopyN.hpp +++ /dev/null @@ -1,116 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COPY_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_COPY_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdCopyFunctor { - InputIterator m_first; - OutputIterator m_dest_first; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { m_dest_first[i] = m_first[i]; } - - KOKKOS_FUNCTION - StdCopyFunctor(InputIterator _first, OutputIterator _dest_first) - : m_first(std::move(_first)), m_dest_first(std::move(_dest_first)) {} -}; - -template -OutputIterator copy_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first, d_first); - Impl::expect_valid_range(first, last); - - // aliases - using index_type = typename InputIterator::difference_type; - using func_t = StdCopyFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first, d_first)); - ex.fence("Kokkos::copy: fence after operation"); - - // return - return d_first + num_elements; -} - -template -OutputIterator copy_n_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first_from, Size count, - OutputIterator first_dest) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - - if (count > 0) { - return copy_impl(label, ex, first_from, first_from + count, first_dest); - } else { - return first_dest; - } -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CountCountIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CountCountIf.hpp deleted file mode 100644 index 982ac40464..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CountCountIf.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_COUNT_IF_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_COUNT_IF_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdCountIfFunctor { - using index_type = typename IteratorType::difference_type; - IteratorType m_first; - Predicate m_predicate; - - KOKKOS_FUNCTION - void operator()(index_type i, index_type& lsum) const { - if (m_predicate(m_first[i])) { - lsum++; - } - } - - KOKKOS_FUNCTION - StdCountIfFunctor(IteratorType _first, Predicate _predicate) - : m_first(std::move(_first)), m_predicate(std::move(_predicate)) {} -}; - -template -typename IteratorType::difference_type count_if_impl(const std::string& label, - const ExecutionSpace& ex, - IteratorType first, - IteratorType last, - Predicate predicate) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // aliases - using func_t = StdCountIfFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - typename IteratorType::difference_type count = 0; - ::Kokkos::parallel_reduce(label, - RangePolicy(ex, 0, num_elements), - func_t(first, predicate), count); - ex.fence("Kokkos::count_if: fence after operation"); - - return count; -} - -template -auto count_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, const T& value) { - return count_if_impl( - label, ex, first, last, - ::Kokkos::Experimental::Impl::StdAlgoEqualsValUnaryPredicate(value)); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Equal.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Equal.hpp deleted file mode 100644 index 9482917abb..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Equal.hpp +++ /dev/null @@ -1,147 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_EQUAL_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_EQUAL_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdEqualFunctor { - IteratorType1 m_first1; - IteratorType2 m_first2; - BinaryPredicateType m_predicate; - - KOKKOS_FUNCTION - void operator()(IndexType i, std::size_t& lsum) const { - if (!m_predicate(m_first1[i], m_first2[i])) { - lsum = 1; - } - } - - KOKKOS_FUNCTION - StdEqualFunctor(IteratorType1 _first1, IteratorType2 _first2, - BinaryPredicateType _predicate) - : m_first1(std::move(_first1)), - m_first2(std::move(_first2)), - m_predicate(std::move(_predicate)) {} -}; - -template -bool equal_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, - BinaryPredicateType predicate) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first1, first2); - Impl::static_assert_iterators_have_matching_difference_type(first1, first2); - Impl::expect_valid_range(first1, last1); - - // aliases - using index_type = typename IteratorType1::difference_type; - using func_t = StdEqualFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first1, last1); - std::size_t different = 0; - ::Kokkos::parallel_reduce(label, - RangePolicy(ex, 0, num_elements), - func_t(first1, first2, predicate), different); - ex.fence("Kokkos::equal: fence after operation"); - - return !different; -} - -template -bool equal_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2) { - using value_type1 = typename IteratorType1::value_type; - using value_type2 = typename IteratorType2::value_type; - using pred_t = StdAlgoEqualBinaryPredicate; - return equal_impl(label, ex, first1, last1, first2, pred_t()); -} - -template -bool equal_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, - IteratorType2 last2, BinaryPredicateType predicate) { - const auto d1 = ::Kokkos::Experimental::distance(first1, last1); - const auto d2 = ::Kokkos::Experimental::distance(first2, last2); - if (d1 != d2) { - return false; - } - - return equal_impl(label, ex, first1, last1, first2, predicate); -} - -template -bool equal_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, - IteratorType2 last2) { - Impl::expect_valid_range(first1, last1); - Impl::expect_valid_range(first2, last2); - - using value_type1 = typename IteratorType1::value_type; - using value_type2 = typename IteratorType2::value_type; - using pred_t = StdAlgoEqualBinaryPredicate; - return equal_impl(label, ex, first1, last1, first2, last2, pred_t()); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ExclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ExclusiveScan.hpp deleted file mode 100644 index 0ae4651c6a..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ExclusiveScan.hpp +++ /dev/null @@ -1,232 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include "Kokkos_ValueWrapperForNoNeutralElement.hpp" -#include "Kokkos_IdentityReferenceUnaryFunctor.hpp" -#include -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct ExclusiveScanDefaultFunctorForKnownNeutralElement { - using execution_space = ExeSpace; - - ValueType m_init_value; - FirstFrom m_first_from; - FirstDest m_first_dest; - - KOKKOS_FUNCTION - ExclusiveScanDefaultFunctorForKnownNeutralElement(ValueType init, - FirstFrom first_from, - FirstDest first_dest) - : m_init_value(std::move(init)), - m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)) {} - - KOKKOS_FUNCTION - void operator()(const IndexType i, ValueType& update, - const bool final_pass) const { - if (final_pass) m_first_dest[i] = update + m_init_value; - update += m_first_from[i]; - } -}; - -template -struct ExclusiveScanDefaultFunctor { - using execution_space = ExeSpace; - using value_type = - ::Kokkos::Experimental::Impl::ValueWrapperForNoNeutralElement; - - ValueType m_init_value; - FirstFrom m_first_from; - FirstDest m_first_dest; - - KOKKOS_FUNCTION - ExclusiveScanDefaultFunctor(ValueType init, FirstFrom first_from, - FirstDest first_dest) - : m_init_value(std::move(init)), - m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)) {} - - KOKKOS_FUNCTION - void operator()(const IndexType i, value_type& update, - const bool final_pass) const { - if (final_pass) { - if (i == 0) { - m_first_dest[i] = m_init_value; - } else { - m_first_dest[i] = update.val + m_init_value; - } - } - - const auto tmp = value_type{m_first_from[i], false}; - this->join(update, tmp); - } - - KOKKOS_FUNCTION - void init(value_type& update) const { - update.val = {}; - update.is_initial = true; - } - - KOKKOS_FUNCTION - void join(value_type& update, const value_type& input) const { - if (update.is_initial) { - update.val = input.val; - update.is_initial = false; - } else { - update.val = update.val + input.val; - } - } -}; - -template -OutputIteratorType exclusive_scan_custom_op_impl( - const std::string& label, const ExecutionSpace& ex, - InputIteratorType first_from, InputIteratorType last_from, - OutputIteratorType first_dest, ValueType init_value, BinaryOpType bop) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // aliases - using index_type = typename InputIteratorType::difference_type; - using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; - using func_type = - TransformExclusiveScanFunctor; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_scan( - label, RangePolicy(ex, 0, num_elements), - func_type(init_value, first_from, first_dest, bop, unary_op_type())); - ex.fence("Kokkos::exclusive_scan_custom_op: fence after operation"); - - // return - return first_dest + num_elements; -} - -template -using ex_scan_has_reduction_identity_sum_t = - decltype(Kokkos::reduction_identity::sum()); - -template -OutputIteratorType exclusive_scan_default_op_impl(const std::string& label, - const ExecutionSpace& ex, - InputIteratorType first_from, - InputIteratorType last_from, - OutputIteratorType first_dest, - ValueType init_value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // does it make sense to do this static_assert too? - // using input_iterator_value_type = typename InputIteratorType::value_type; - // static_assert - // (std::is_convertible, - // ValueType>::value, - // "exclusive_scan: InputIteratorType::value_type not convertible to - // ValueType"); - - // we are unnecessarily duplicating code, but this is on purpose - // so that we can use the default_op for OpenMPTarget. - // Originally, I had this implemented as: - // ''' - // using bop_type = StdExclusiveScanDefaultJoinFunctor; - // call exclusive_scan_custom_op_impl(..., bop_type()); - // ''' - // which avoids duplicating the functors, but for OpenMPTarget - // I cannot use a custom binary op. - // This is the same problem that occurs for reductions. - - // aliases - using index_type = typename InputIteratorType::difference_type; - using func_type = std::conditional_t< - ::Kokkos::is_detected::value, - ExclusiveScanDefaultFunctorForKnownNeutralElement< - ExecutionSpace, index_type, ValueType, InputIteratorType, - OutputIteratorType>, - ExclusiveScanDefaultFunctor>; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_scan(label, - RangePolicy(ex, 0, num_elements), - func_type(init_value, first_from, first_dest)); - - ex.fence("Kokkos::exclusive_scan_default_op: fence after operation"); - - return first_dest + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FillFillN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FillFillN.hpp deleted file mode 100644 index 843771b6b1..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FillFillN.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FILL_AND_FILL_N_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_FILL_AND_FILL_N_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdFillFunctor { - using index_type = typename InputIterator::difference_type; - InputIterator m_first; - T m_value; - - KOKKOS_FUNCTION - void operator()(index_type i) const { m_first[i] = m_value; } - - KOKKOS_FUNCTION - StdFillFunctor(InputIterator _first, T _value) - : m_first(std::move(_first)), m_value(std::move(_value)) {} -}; - -template -void fill_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, const T& value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - StdFillFunctor(first, value)); - ex.fence("Kokkos::fill: fence after operation"); -} - -template -IteratorType fill_n_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, SizeType n, const T& value) { - auto last = first + n; - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - if (n <= 0) { - return first; - } - - fill_impl(label, ex, first, last, value); - return last; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ForEachForEachN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ForEachForEachN.hpp deleted file mode 100644 index 8bd37b13bc..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ForEachForEachN.hpp +++ /dev/null @@ -1,113 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_FOR_EACH_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_FOR_EACH_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdForEachFunctor { - using index_type = typename IteratorType::difference_type; - IteratorType m_first; - UnaryFunctorType m_functor; - - KOKKOS_FUNCTION - void operator()(index_type i) const { m_functor(m_first[i]); } - - KOKKOS_FUNCTION - StdForEachFunctor(IteratorType _first, UnaryFunctorType _functor) - : m_first(std::move(_first)), m_functor(std::move(_functor)) {} -}; - -template -UnaryFunctorType for_each_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType first, - IteratorType last, UnaryFunctorType functor) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for( - label, RangePolicy(ex, 0, num_elements), - StdForEachFunctor(first, functor)); - ex.fence("Kokkos::for_each: fence after operation"); - - return functor; -} - -template -IteratorType for_each_n_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, SizeType n, - UnaryFunctorType functor) { - auto last = first + n; - Impl::static_assert_random_access_and_accessible(ex, first, last); - Impl::expect_valid_range(first, last); - - if (n == 0) { - return first; - } - - for_each_impl(label, ex, first, last, std::move(functor)); - // no neeed to fence since for_each_impl fences already - - return last; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_GenerateGenerateN.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_GenerateGenerateN.hpp deleted file mode 100644 index f01d9bfb55..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_GenerateGenerateN.hpp +++ /dev/null @@ -1,105 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_GENERATE_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_GENERATE_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdGenerateFunctor { - using index_type = typename IteratorType::difference_type; - IteratorType m_first; - Generator m_generator; - - KOKKOS_FUNCTION - void operator()(index_type i) const { m_first[i] = m_generator(); } - - KOKKOS_FUNCTION - StdGenerateFunctor(IteratorType _first, Generator _g) - : m_first(std::move(_first)), m_generator(std::move(_g)) {} -}; - -template -void generate_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, Generator g) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // aliases - using func_t = StdGenerateFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first, g)); - ex.fence("Kokkos::generate: fence after operation"); -} - -template -IteratorType generate_n_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, Size count, Generator g) { - if (count <= 0) { - return first; - } - - generate_impl(label, ex, first, first + count, g); - return first + count; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IdentityReferenceUnaryFunctor.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IdentityReferenceUnaryFunctor.hpp deleted file mode 100644 index f41e567c9b..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IdentityReferenceUnaryFunctor.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_NUMERIC_IDENTITY_REFERENCE_UNARY_FUNCTOR_HPP -#define KOKKOS_STD_ALGORITHMS_NUMERIC_IDENTITY_REFERENCE_UNARY_FUNCTOR_HPP - -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdNumericScanIdentityReferenceUnaryFunctor { - KOKKOS_FUNCTION - constexpr const ValueType& operator()(const ValueType& a) const { return a; } -}; - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsSortedUntil.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsSortedUntil.hpp deleted file mode 100644 index 4e99c301b2..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsSortedUntil.hpp +++ /dev/null @@ -1,153 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdIsSortedUntilFunctor { - using index_type = typename IteratorType::difference_type; - IteratorType m_first; - IndicatorViewType m_indicator; - ComparatorType m_comparator; - - KOKKOS_FUNCTION - void operator()(const index_type i, int& update, const bool final) const { - const auto& val_i = m_first[i]; - const auto& val_ip1 = m_first[i + 1]; - - if (m_comparator(val_ip1, val_i)) { - ++update; - } - - if (final) { - m_indicator(i) = update; - } - } - - KOKKOS_FUNCTION - StdIsSortedUntilFunctor(IteratorType _first1, IndicatorViewType indicator, - ComparatorType comparator) - : m_first(std::move(_first1)), - m_indicator(std::move(indicator)), - m_comparator(std::move(comparator)) {} -}; - -template -IteratorType is_sorted_until_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType first, - IteratorType last, ComparatorType comp) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - const auto num_elements = Kokkos::Experimental::distance(first, last); - - // trivial case - if (num_elements <= 1) { - return last; - } - - /* - use scan and a helper "indicator" view - such that we scan the data and fill the indicator with - partial sum that is always 0 unless we find a pair that - breaks the sorting, so in that case the indicator will - have a 1 starting at the location where the sorting breaks. - So finding that 1 means finding the location we want. - */ - - // aliases - using indicator_value_type = std::size_t; - using indicator_view_type = - ::Kokkos::View; - using functor_type = - StdIsSortedUntilFunctor; - - // do scan - // use num_elements-1 because each index handles i and i+1 - const auto num_elements_minus_one = num_elements - 1; - indicator_view_type indicator("is_sorted_until_indicator_helper", - num_elements_minus_one); - ::Kokkos::parallel_scan( - label, RangePolicy(ex, 0, num_elements_minus_one), - functor_type(first, indicator, std::move(comp))); - - // try to find the first sentinel value, which indicates - // where the sorting condition breaks - namespace KE = ::Kokkos::Experimental; - constexpr indicator_value_type sentinel_value = 1; - auto r = - KE::find(ex, KE::cbegin(indicator), KE::cend(indicator), sentinel_value); - const auto shift = r - ::Kokkos::Experimental::cbegin(indicator); - - return first + (shift + 1); -} - -template -IteratorType is_sorted_until_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType first, - IteratorType last) { - using value_type = typename IteratorType::value_type; - using pred_t = Impl::StdAlgoLessThanBinaryPredicate; - return is_sorted_until_impl(label, ex, first, last, pred_t()); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Mismatch.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Mismatch.hpp deleted file mode 100644 index 180afe925c..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Mismatch.hpp +++ /dev/null @@ -1,162 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MISMATCH_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_MISMATCH_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdMismatchRedFunctor { - using red_value_type = typename ReducerType::value_type; - - IteratorType1 m_first1; - IteratorType2 m_first2; - ReducerType m_reducer; - BinaryPredicateType m_predicate; - - KOKKOS_FUNCTION - void operator()(const IndexType i, red_value_type& red_value) const { - const auto& my_value1 = m_first1[i]; - const auto& my_value2 = m_first2[i]; - - auto rv = - !m_predicate(my_value1, my_value2) - ? red_value_type{i} - : red_value_type{::Kokkos::reduction_identity::min()}; - - m_reducer.join(red_value, rv); - } - - KOKKOS_FUNCTION - StdMismatchRedFunctor(IteratorType1 first1, IteratorType2 first2, - ReducerType reducer, BinaryPredicateType predicate) - : m_first1(std::move(first1)), - m_first2(std::move(first2)), - m_reducer(std::move(reducer)), - m_predicate(std::move(predicate)) {} -}; - -template -::Kokkos::pair mismatch_impl( - const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, IteratorType2 last2, - BinaryPredicateType predicate) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first1, first2); - Impl::static_assert_iterators_have_matching_difference_type(first1, first2); - Impl::expect_valid_range(first1, last1); - Impl::expect_valid_range(first2, last2); - - // aliases - using return_type = ::Kokkos::pair; - using index_type = typename IteratorType1::difference_type; - using reducer_type = FirstLoc; - using reduction_value_type = typename reducer_type::value_type; - using functor_type = - StdMismatchRedFunctor; - - // trivial case: note that this is important, - // for OpenMPTarget, omitting special handling of - // the trivial case was giving all sorts of strange stuff. - const auto num_e1 = last1 - first1; - const auto num_e2 = last2 - first2; - if (num_e1 == 0 || num_e2 == 0) { - return return_type(first1, first2); - } - - // run - const auto num_elemen_par_reduce = (num_e1 <= num_e2) ? num_e1 : num_e2; - reduction_value_type red_result; - reducer_type reducer(red_result); - ::Kokkos::parallel_reduce( - label, RangePolicy(ex, 0, num_elemen_par_reduce), - functor_type(first1, first2, reducer, std::move(predicate)), reducer); - - // fence not needed because reducing into scalar - - // decide and return - constexpr auto red_min = ::Kokkos::reduction_identity::min(); - if (red_result.min_loc_true == red_min) { - // in here means mismatch has not been found - if (num_e1 == num_e2) { - return return_type(last1, last2); - } else if (num_e1 < num_e2) { - return return_type(last1, first2 + num_e1); - } else { - return return_type(first1 + num_e2, last2); - } - } else { - // in here means mismatch has been found - return return_type(first1 + red_result.min_loc_true, - first2 + red_result.min_loc_true); - } -} - -template -::Kokkos::pair mismatch_impl( - const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2, IteratorType2 last2) { - using value_type1 = typename IteratorType1::value_type; - using value_type2 = typename IteratorType2::value_type; - using pred_t = StdAlgoEqualBinaryPredicate; - return mismatch_impl(label, ex, first1, last1, first2, last2, pred_t()); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Move.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Move.hpp deleted file mode 100644 index 6b1ed1da4c..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Move.hpp +++ /dev/null @@ -1,100 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MOVE_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_MOVE_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdMoveFunctor { - InputIterator m_first; - OutputIterator m_dest_first; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { - m_dest_first[i] = std::move(m_first[i]); - } - - StdMoveFunctor(InputIterator _first, OutputIterator _dest_first) - : m_first(std::move(_first)), m_dest_first(std::move(_dest_first)) {} -}; - -template -OutputIterator move_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first, d_first); - Impl::expect_valid_range(first, last); - - // aliases - using index_type = typename InputIterator::difference_type; - using func_t = StdMoveFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first, d_first)); - ex.fence("Kokkos::move: fence after operation"); - - // return - return d_first + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp deleted file mode 100644 index c34ab679db..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_MOVE_BACKWARD_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_MOVE_BACKWARD_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdMoveBackwardFunctor { - static_assert(std::is_signed::value, - "Kokkos: StdMoveBackwardFunctor requires signed index type"); - - IteratorType1 m_last; - IteratorType2 m_dest_last; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { - m_dest_last[-i - 1] = std::move(m_last[-i - 1]); - } - - StdMoveBackwardFunctor(IteratorType1 _last, IteratorType2 _dest_last) - : m_last(std::move(_last)), m_dest_last(std::move(_dest_last)) {} -}; - -template -IteratorType2 move_backward_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 d_last) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_last); - Impl::static_assert_iterators_have_matching_difference_type(first, d_last); - Impl::expect_valid_range(first, last); - - // aliases - using index_type = typename IteratorType1::difference_type; - using func_t = - StdMoveBackwardFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(last, d_last)); - ex.fence("Kokkos::move_backward: fence after operation"); - - // return - return d_last - num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Replace.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Replace.hpp deleted file mode 100644 index 877ffa276f..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Replace.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdReplaceFunctor { - using index_type = typename InputIterator::difference_type; - InputIterator m_first; - ValueType m_old_value; - ValueType m_new_value; - - KOKKOS_FUNCTION - void operator()(index_type i) const { - if (m_first[i] == m_old_value) { - m_first[i] = m_new_value; - } - } - - KOKKOS_FUNCTION - StdReplaceFunctor(InputIterator first, ValueType old_value, - ValueType new_value) - : m_first(std::move(first)), - m_old_value(std::move(old_value)), - m_new_value(std::move(new_value)) {} -}; - -template -void replace_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - const ValueType& old_value, const ValueType& new_value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // aliases - using func_t = StdReplaceFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first, old_value, new_value)); - ex.fence("Kokkos::replace: fence after operation"); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceCopy.hpp deleted file mode 100644 index b75dde9cd8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceCopy.hpp +++ /dev/null @@ -1,122 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdReplaceCopyFunctor { - using index_type = typename InputIterator::difference_type; - - InputIterator m_first_from; - OutputIterator m_first_dest; - ValueType m_old_value; - ValueType m_new_value; - - KOKKOS_FUNCTION - void operator()(index_type i) const { - const auto& myvalue_from = m_first_from[i]; - - if (myvalue_from == m_old_value) { - m_first_dest[i] = m_new_value; - } else { - m_first_dest[i] = myvalue_from; - } - } - - KOKKOS_FUNCTION - StdReplaceCopyFunctor(InputIterator first_from, OutputIterator first_dest, - ValueType old_value, ValueType new_value) - : m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)), - m_old_value(std::move(old_value)), - m_new_value(std::move(new_value)) {} -}; - -template -OutputIteratorType replace_copy_impl(const std::string& label, - const ExecutionSpace& ex, - InputIteratorType first_from, - InputIteratorType last_from, - OutputIteratorType first_dest, - const ValueType& old_value, - const ValueType& new_value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // aliases - using func_t = - StdReplaceCopyFunctor; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first_from, first_dest, old_value, new_value)); - ex.fence("Kokkos::replace_copy: fence after operation"); - - // return - return first_dest + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceCopyIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceCopyIf.hpp deleted file mode 100644 index 8f7c8140e6..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceCopyIf.hpp +++ /dev/null @@ -1,123 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IF_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IF_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdReplaceIfCopyFunctor { - InputIterator m_first_from; - OutputIterator m_first_dest; - PredicateType m_pred; - ValueType m_new_value; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { - const auto& myvalue_from = m_first_from[i]; - - if (m_pred(myvalue_from)) { - m_first_dest[i] = m_new_value; - } else { - m_first_dest[i] = myvalue_from; - } - } - - KOKKOS_FUNCTION - StdReplaceIfCopyFunctor(InputIterator first_from, OutputIterator first_dest, - PredicateType pred, ValueType new_value) - : m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)), - m_pred(std::move(pred)), - m_new_value(std::move(new_value)) {} -}; - -template -OutputIteratorType replace_copy_if_impl(const std::string& label, - const ExecutionSpace& ex, - InputIteratorType first_from, - InputIteratorType last_from, - OutputIteratorType first_dest, - PredicateType pred, - const ValueType& new_value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // aliases - using index_type = typename InputIteratorType::difference_type; - using func_t = - StdReplaceIfCopyFunctor; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_for( - label, RangePolicy(ex, 0, num_elements), - func_t(first_from, first_dest, std::move(pred), new_value)); - ex.fence("Kokkos::replace_copy_if: fence after operation"); - - // return - return first_dest + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceIf.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceIf.hpp deleted file mode 100644 index 6fe33019c0..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReplaceIf.hpp +++ /dev/null @@ -1,105 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_IF_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_REPLACE_IF_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdReplaceIfFunctor { - using index_type = typename InputIterator::difference_type; - - InputIterator m_first; - PredicateType m_predicate; - NewValueType m_new_value; - - KOKKOS_FUNCTION - void operator()(index_type i) const { - if (m_predicate(m_first[i])) { - m_first[i] = m_new_value; - } - } - - KOKKOS_FUNCTION - StdReplaceIfFunctor(InputIterator first, PredicateType pred, - NewValueType new_value) - : m_first(std::move(first)), - m_predicate(std::move(pred)), - m_new_value(std::move(new_value)) {} -}; - -template -void replace_if_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, PredicateType pred, - const ValueType& new_value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // aliases - using func_t = StdReplaceIfFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first, std::move(pred), new_value)); - ex.fence("Kokkos::replace_if: fence after operation"); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp deleted file mode 100644 index f84eb2c81a..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REVERSE_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_REVERSE_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdReverseFunctor { - using index_type = typename InputIterator::difference_type; - static_assert(std::is_signed::value, - "Kokkos: StdReverseFunctor requires signed index type"); - - InputIterator m_first; - InputIterator m_last; - - KOKKOS_FUNCTION - void operator()(index_type i) const { - // the swap below is doing the same thing, but - // for Intel 18.0.5 does not work. - // But putting the impl directly here, it works. -#ifdef KOKKOS_COMPILER_INTEL - typename InputIterator::value_type tmp = std::move(m_first[i]); - m_first[i] = std::move(m_last[-i - 1]); - m_last[-i - 1] = std::move(tmp); -#else - ::Kokkos::Experimental::swap(m_first[i], m_last[-i - 1]); -#endif - } - - StdReverseFunctor(InputIterator first, InputIterator last) - : m_first(std::move(first)), m_last(std::move(last)) {} -}; - -template -void reverse_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first); - Impl::expect_valid_range(first, last); - - // aliases - using func_t = StdReverseFunctor; - - // run - if (last >= first + 2) { - // only need half - const auto num_elements = Kokkos::Experimental::distance(first, last) / 2; - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first, last)); - ex.fence("Kokkos::reverse: fence after operation"); - } -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp deleted file mode 100644 index 88b6ed16b5..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_REVERSE_COPY_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_REVERSE_COPY_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdReverseCopyFunctor { - static_assert(std::is_signed::value, - "Kokkos: StdReverseCopyFunctor requires signed index type"); - - InputIterator m_last; - OutputIterator m_dest_first; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { m_dest_first[i] = m_last[-1 - i]; } - - StdReverseCopyFunctor(InputIterator _last, OutputIterator _dest_first) - : m_last(std::move(_last)), m_dest_first(std::move(_dest_first)) {} -}; - -template -OutputIterator reverse_copy_impl(const std::string& label, - const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first, d_first); - Impl::expect_valid_range(first, last); - - // aliases - using index_type = typename InputIterator::difference_type; - using func_t = - StdReverseCopyFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first, last); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(last, d_first)); - ex.fence("Kokkos::reverse_copy: fence after operation"); - - // return - return d_first + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_SwapRanges.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_SwapRanges.hpp deleted file mode 100644 index 3e6ca14697..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_SwapRanges.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_SWAP_RANGES_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_SWAP_RANGES_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdSwapRangesFunctor { - IteratorType1 m_first1; - IteratorType2 m_first2; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { - // the swap below is doing the same thing, but - // for Intel 18.0.5 does not work. - // But putting the impl directly here, it works. -#ifdef KOKKOS_COMPILER_INTEL - typename IteratorType1::value_type tmp = std::move(m_first1[i]); - m_first1[i] = std::move(m_first2[i]); - m_first2[i] = std::move(tmp); -#else - ::Kokkos::Experimental::swap(m_first1[i], m_first2[i]); -#endif - } - - KOKKOS_FUNCTION - StdSwapRangesFunctor(IteratorType1 _first1, IteratorType2 _first2) - : m_first1(std::move(_first1)), m_first2(std::move(_first2)) {} -}; - -template -IteratorType2 swap_ranges_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType1 first1, - IteratorType1 last1, IteratorType2 first2) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first1, first2); - Impl::static_assert_iterators_have_matching_difference_type(first1, first2); - Impl::expect_valid_range(first1, last1); - - // aliases - using index_type = typename IteratorType1::difference_type; - using func_t = StdSwapRangesFunctor; - - // run - const auto num_elements_to_swap = - Kokkos::Experimental::distance(first1, last1); - ::Kokkos::parallel_for( - label, RangePolicy(ex, 0, num_elements_to_swap), - func_t(first1, first2)); - ex.fence("Kokkos::swap_ranges: fence after operation"); - - // return - return first2 + num_elements_to_swap; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Transform.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Transform.hpp deleted file mode 100644 index 5d2c0cc982..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Transform.hpp +++ /dev/null @@ -1,158 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_TRANSFORM_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdTransformFunctor { - InputIterator m_first; - OutputIterator m_d_first; - UnaryFunctorType m_unary_op; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { m_d_first[i] = m_unary_op(m_first[i]); } - - KOKKOS_FUNCTION - StdTransformFunctor(InputIterator _first, OutputIterator _m_d_first, - UnaryFunctorType _functor) - : m_first(std::move(_first)), - m_d_first(std::move(_m_d_first)), - m_unary_op(std::move(_functor)) {} -}; - -template -struct StdTransformBinaryFunctor { - InputIterator1 m_first1; - InputIterator2 m_first2; - OutputIterator m_d_first; - BinaryFunctorType m_binary_op; - - KOKKOS_FUNCTION - void operator()(IndexType i) const { - m_d_first[i] = m_binary_op(m_first1[i], m_first2[i]); - } - - KOKKOS_FUNCTION - StdTransformBinaryFunctor(InputIterator1 _first1, InputIterator2 _first2, - OutputIterator _m_d_first, - BinaryFunctorType _functor) - : m_first1(std::move(_first1)), - m_first2(std::move(_first2)), - m_d_first(std::move(_m_d_first)), - m_binary_op(std::move(_functor)) {} -}; - -template -OutputIterator transform_impl(const std::string& label, - const ExecutionSpace& ex, InputIterator first1, - InputIterator last1, OutputIterator d_first, - UnaryOperation unary_op) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first1, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first1, d_first); - Impl::expect_valid_range(first1, last1); - - // aliases - using index_type = typename InputIterator::difference_type; - using func_t = StdTransformFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first1, last1); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first1, d_first, unary_op)); - ex.fence("Kokkos::transform: fence after operation"); - - // return - return d_first + num_elements; -} - -template -OutputIterator transform_impl(const std::string& label, - const ExecutionSpace& ex, InputIterator1 first1, - InputIterator1 last1, InputIterator2 first2, - OutputIterator d_first, - BinaryOperation binary_op) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first1, first2, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first1, first2, - d_first); - Impl::expect_valid_range(first1, last1); - - // aliases - using index_type = typename InputIterator1::difference_type; - using func_t = - StdTransformBinaryFunctor; - - // run - const auto num_elements = Kokkos::Experimental::distance(first1, last1); - ::Kokkos::parallel_for(label, - RangePolicy(ex, 0, num_elements), - func_t(first1, first2, d_first, binary_op)); - ex.fence("Kokkos::transform: fence after operation"); - return d_first + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_TransformExclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_TransformExclusiveScan.hpp deleted file mode 100644 index 9fb8cbcc30..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_TransformExclusiveScan.hpp +++ /dev/null @@ -1,153 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_EXCLUSIVE_SCAN_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_TRANSFORM_EXCLUSIVE_SCAN_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include "Kokkos_ValueWrapperForNoNeutralElement.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct TransformExclusiveScanFunctor { - using execution_space = ExeSpace; - using value_type = - ::Kokkos::Experimental::Impl::ValueWrapperForNoNeutralElement; - - ValueType m_init_value; - FirstFrom m_first_from; - FirstDest m_first_dest; - BinaryOpType m_binary_op; - UnaryOpType m_unary_op; - - KOKKOS_FUNCTION - TransformExclusiveScanFunctor(ValueType init, FirstFrom first_from, - FirstDest first_dest, BinaryOpType bop, - UnaryOpType uop) - : m_init_value(std::move(init)), - m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)), - m_binary_op(std::move(bop)), - m_unary_op(std::move(uop)) {} - - KOKKOS_FUNCTION - void operator()(const IndexType i, value_type& update, - const bool final_pass) const { - if (final_pass) { - if (i == 0) { - // for both ExclusiveScan and TransformExclusiveScan, - // init is unmodified - m_first_dest[i] = m_init_value; - } else { - m_first_dest[i] = m_binary_op(update.val, m_init_value); - } - } - - const auto tmp = value_type{m_unary_op(m_first_from[i]), false}; - this->join(update, tmp); - } - - KOKKOS_FUNCTION - void init(value_type& update) const { - update.val = {}; - update.is_initial = true; - } - - KOKKOS_FUNCTION - void join(value_type& update, const value_type& input) const { - if (update.is_initial) { - update.val = input.val; - } else { - update.val = m_binary_op(update.val, input.val); - } - update.is_initial = false; - } -}; - -template -OutputIteratorType transform_exclusive_scan_impl( - const std::string& label, const ExecutionSpace& ex, - InputIteratorType first_from, InputIteratorType last_from, - OutputIteratorType first_dest, ValueType init_value, BinaryOpType bop, - UnaryOpType uop) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // aliases - using index_type = typename InputIteratorType::difference_type; - using func_type = - TransformExclusiveScanFunctor; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_scan( - label, RangePolicy(ex, 0, num_elements), - func_type(init_value, first_from, first_dest, bop, uop)); - ex.fence("Kokkos::transform_exclusive_scan: fence after operation"); - - // return - return first_dest + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_TransformInclusiveScan.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_TransformInclusiveScan.hpp deleted file mode 100644 index 281eb6f8a1..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_TransformInclusiveScan.hpp +++ /dev/null @@ -1,235 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_INCLUSIVE_SCAN_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_TRANSFORM_INCLUSIVE_SCAN_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include "Kokkos_ValueWrapperForNoNeutralElement.hpp" -#include "Kokkos_IdentityReferenceUnaryFunctor.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct TransformInclusiveScanNoInitValueFunctor { - using execution_space = ExeSpace; - using value_type = ValueWrapperForNoNeutralElement; - - FirstFrom m_first_from; - FirstDest m_first_dest; - BinaryOpType m_binary_op; - UnaryOpType m_unary_op; - - KOKKOS_FUNCTION - TransformInclusiveScanNoInitValueFunctor(FirstFrom first_from, - FirstDest first_dest, - BinaryOpType bop, UnaryOpType uop) - : m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)), - m_binary_op(std::move(bop)), - m_unary_op(std::move(uop)) {} - - KOKKOS_FUNCTION - void operator()(const IndexType i, value_type& update, - const bool final_pass) const { - const auto tmp = value_type{m_unary_op(m_first_from[i]), false}; - this->join(update, tmp); - if (final_pass) { - m_first_dest[i] = update.val; - } - } - - KOKKOS_FUNCTION - void init(value_type& update) const { - update.val = {}; - update.is_initial = true; - } - - KOKKOS_FUNCTION - void join(value_type& update, const value_type& input) const { - if (update.is_initial) { - update.val = input.val; - } else { - update.val = m_binary_op(update.val, input.val); - } - update.is_initial = false; - } -}; - -template -struct TransformInclusiveScanWithInitValueFunctor { - using execution_space = ExeSpace; - using value_type = ValueWrapperForNoNeutralElement; - - FirstFrom m_first_from; - FirstDest m_first_dest; - BinaryOpType m_binary_op; - UnaryOpType m_unary_op; - ValueType m_init; - - KOKKOS_FUNCTION - TransformInclusiveScanWithInitValueFunctor(FirstFrom first_from, - FirstDest first_dest, - BinaryOpType bop, UnaryOpType uop, - ValueType init) - : m_first_from(std::move(first_from)), - m_first_dest(std::move(first_dest)), - m_binary_op(std::move(bop)), - m_unary_op(std::move(uop)), - m_init(std::move(init)) {} - - KOKKOS_FUNCTION - void operator()(const IndexType i, value_type& update, - const bool final_pass) const { - const auto tmp = value_type{m_unary_op(m_first_from[i]), false}; - this->join(update, tmp); - - if (final_pass) { - m_first_dest[i] = m_binary_op(update.val, m_init); - } - } - - KOKKOS_FUNCTION - void init(value_type& update) const { - update.val = {}; - update.is_initial = true; - } - - KOKKOS_FUNCTION - void join(value_type& update, const value_type& input) const { - if (update.is_initial) { - update.val = input.val; - } else { - update.val = m_binary_op(update.val, input.val); - } - update.is_initial = false; - } -}; - -// ------------------------------------------------------------- -// transform_inclusive_scan_impl without init_value -// ------------------------------------------------------------- -template -OutputIteratorType transform_inclusive_scan_impl(const std::string& label, - const ExecutionSpace& ex, - InputIteratorType first_from, - InputIteratorType last_from, - OutputIteratorType first_dest, - BinaryOpType binary_op, - UnaryOpType unary_op) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // aliases - using index_type = typename InputIteratorType::difference_type; - using value_type = - std::remove_const_t; - using func_type = TransformInclusiveScanNoInitValueFunctor< - ExecutionSpace, index_type, value_type, InputIteratorType, - OutputIteratorType, BinaryOpType, UnaryOpType>; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_scan( - label, RangePolicy(ex, 0, num_elements), - func_type(first_from, first_dest, binary_op, unary_op)); - ex.fence("Kokkos::transform_inclusive_scan: fence after operation"); - - // return - return first_dest + num_elements; -} - -// ------------------------------------------------------------- -// transform_inclusive_scan_impl with init_value -// ------------------------------------------------------------- -template -OutputIteratorType transform_inclusive_scan_impl( - const std::string& label, const ExecutionSpace& ex, - InputIteratorType first_from, InputIteratorType last_from, - OutputIteratorType first_dest, BinaryOpType binary_op, UnaryOpType unary_op, - ValueType init_value) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); - Impl::static_assert_iterators_have_matching_difference_type(first_from, - first_dest); - Impl::expect_valid_range(first_from, last_from); - - // aliases - using index_type = typename InputIteratorType::difference_type; - using func_type = TransformInclusiveScanWithInitValueFunctor< - ExecutionSpace, index_type, ValueType, InputIteratorType, - OutputIteratorType, BinaryOpType, UnaryOpType>; - - // run - const auto num_elements = - Kokkos::Experimental::distance(first_from, last_from); - ::Kokkos::parallel_scan( - label, RangePolicy(ex, 0, num_elements), - func_type(first_from, first_dest, binary_op, unary_op, init_value)); - ex.fence("Kokkos::transform_inclusive_scan: fence after operation"); - - // return - return first_dest + num_elements; -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_UniqueCopy.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_UniqueCopy.hpp deleted file mode 100644 index e4fd6f3ed8..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_UniqueCopy.hpp +++ /dev/null @@ -1,156 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_UNIQUE_COPY_IMPL_HPP -#define KOKKOS_STD_ALGORITHMS_UNIQUE_COPY_IMPL_HPP - -#include -#include "Kokkos_Constraints.hpp" -#include "Kokkos_HelperPredicates.hpp" -#include "Kokkos_CopyCopyN.hpp" -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -template -struct StdUniqueCopyFunctor { - InputIt m_first_from; - InputIt m_last_from; - OutputIt m_first_dest; - BinaryPredicateType m_pred; - - KOKKOS_FUNCTION - StdUniqueCopyFunctor(InputIt first_from, InputIt last_from, - OutputIt first_dest, BinaryPredicateType pred) - : m_first_from(std::move(first_from)), - m_last_from(std::move(last_from)), - m_first_dest(std::move(first_dest)), - m_pred(std::move(pred)) {} - - KOKKOS_FUNCTION - void operator()(const IndexType i, IndexType& update, - const bool final_pass) const { - const auto& val_i = m_first_from[i]; - const auto& val_ip1 = m_first_from[i + 1]; - - if (final_pass) { - if (!m_pred(val_i, val_ip1)) { - m_first_dest[update] = val_i; - } - } - - if (!m_pred(val_i, val_ip1)) { - update += 1; - } - } -}; - -template -OutputIterator unique_copy_impl(const std::string& label, - const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first, - PredicateType pred) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first, d_first); - Impl::expect_valid_range(first, last); - - // branch for trivial vs non trivial case - const auto num_elements = Kokkos::Experimental::distance(first, last); - if (num_elements == 0) { - return d_first; - } else if (num_elements == 1) { - return Impl::copy_impl("Kokkos::copy_from_unique_copy", ex, first, last, - d_first); - } else { - // aliases - using index_type = typename InputIterator::difference_type; - using func_type = StdUniqueCopyFunctor; - - // note here that we run scan for num_elements - 1 - // because of the way we implement this, the last element is always needed. - // We avoid performing checks inside functor that we are within limits - // and run a "safe" scan and then copy the last element. - const auto scan_size = num_elements - 1; - index_type count = 0; - ::Kokkos::parallel_scan(label, - RangePolicy(ex, 0, scan_size), - func_type(first, last, d_first, pred), count); - - return Impl::copy_impl("Kokkos::copy_from_unique_copy", ex, - first + scan_size, last, d_first + count); - } -} - -template -OutputIterator unique_copy_impl(const std::string& label, - const ExecutionSpace& ex, InputIterator first, - InputIterator last, OutputIterator d_first) { - // checks - Impl::static_assert_random_access_and_accessible(ex, first, d_first); - Impl::static_assert_iterators_have_matching_difference_type(first, d_first); - Impl::expect_valid_range(first, last); - - // aliases - using value_type1 = typename InputIterator::value_type; - using value_type2 = typename OutputIterator::value_type; - - // default binary predicate uses == - using binary_pred_t = StdAlgoEqualBinaryPredicate; - - // run - return unique_copy_impl(label, ex, first, last, d_first, binary_pred_t()); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ValueWrapperForNoNeutralElement.hpp b/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ValueWrapperForNoNeutralElement.hpp deleted file mode 100644 index da9b6ef9a9..0000000000 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_ValueWrapperForNoNeutralElement.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_STD_ALGORITHMS_VALUE_WRAPPER_FOR_NO_NEUTRAL_ELEMENT_HPP -#define KOKKOS_STD_ALGORITHMS_VALUE_WRAPPER_FOR_NO_NEUTRAL_ELEMENT_HPP - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -// -// scalar wrapper used for reductions and scans -// when we don't have neutral element -// -template -struct ValueWrapperForNoNeutralElement { - Scalar val; - bool is_initial = true; - - KOKKOS_FUNCTION - void operator=(const ValueWrapperForNoNeutralElement& rhs) { - val = rhs.val; - is_initial = rhs.is_initial; - } -}; - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/containers/src/Kokkos_DynRankView.hpp b/bundled/kokkos-3.7.00/containers/src/Kokkos_DynRankView.hpp deleted file mode 100644 index 442f0d8617..0000000000 --- a/bundled/kokkos-3.7.00/containers/src/Kokkos_DynRankView.hpp +++ /dev/null @@ -1,2504 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/// \file Kokkos_DynRankView.hpp -/// \brief Declaration and definition of Kokkos::DynRankView. -/// -/// This header file declares and defines Kokkos::DynRankView and its -/// related nonmember functions. - -#ifndef KOKKOS_DYNRANKVIEW_HPP -#define KOKKOS_DYNRANKVIEW_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNRANKVIEW -#endif - -#include -#include -#include - -namespace Kokkos { - -template -class DynRankView; // forward declare - -namespace Impl { - -template -struct DynRankDimTraits { - enum : size_t { unspecified = KOKKOS_INVALID_INDEX }; - - // Compute the rank of the view from the nonzero dimension arguments. - KOKKOS_INLINE_FUNCTION - static size_t computeRank(const size_t N0, const size_t N1, const size_t N2, - const size_t N3, const size_t N4, const size_t N5, - const size_t N6, const size_t /* N7 */) { - return ( - (N6 == unspecified && N5 == unspecified && N4 == unspecified && - N3 == unspecified && N2 == unspecified && N1 == unspecified && - N0 == unspecified) - ? 0 - : ((N6 == unspecified && N5 == unspecified && N4 == unspecified && - N3 == unspecified && N2 == unspecified && N1 == unspecified) - ? 1 - : ((N6 == unspecified && N5 == unspecified && - N4 == unspecified && N3 == unspecified && - N2 == unspecified) - ? 2 - : ((N6 == unspecified && N5 == unspecified && - N4 == unspecified && N3 == unspecified) - ? 3 - : ((N6 == unspecified && N5 == unspecified && - N4 == unspecified) - ? 4 - : ((N6 == unspecified && - N5 == unspecified) - ? 5 - : ((N6 == unspecified) - ? 6 - : 7))))))); - } - - // Compute the rank of the view from the nonzero layout arguments. - template - KOKKOS_INLINE_FUNCTION static size_t computeRank(const Layout& layout) { - return computeRank(layout.dimension[0], layout.dimension[1], - layout.dimension[2], layout.dimension[3], - layout.dimension[4], layout.dimension[5], - layout.dimension[6], layout.dimension[7]); - } - - // Extra overload to match that for specialize types v2 - template - KOKKOS_INLINE_FUNCTION static size_t computeRank( - const Kokkos::Impl::ViewCtorProp& /* prop */, - const Layout& layout) { - return computeRank(layout); - } - - // Create the layout for the rank-7 view. - // Non-strided Layout - template - KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value || - std::is_same::value), - Layout> - createLayout(const Layout& layout) { - return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, - layout.dimension[1] != unspecified ? layout.dimension[1] : 1, - layout.dimension[2] != unspecified ? layout.dimension[2] : 1, - layout.dimension[3] != unspecified ? layout.dimension[3] : 1, - layout.dimension[4] != unspecified ? layout.dimension[4] : 1, - layout.dimension[5] != unspecified ? layout.dimension[5] : 1, - layout.dimension[6] != unspecified ? layout.dimension[6] : 1, - layout.dimension[7] != unspecified ? layout.dimension[7] : 1); - } - - // LayoutStride - template - KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value), Layout> - createLayout(const Layout& layout) { - return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, - layout.stride[0], - layout.dimension[1] != unspecified ? layout.dimension[1] : 1, - layout.stride[1], - layout.dimension[2] != unspecified ? layout.dimension[2] : 1, - layout.stride[2], - layout.dimension[3] != unspecified ? layout.dimension[3] : 1, - layout.stride[3], - layout.dimension[4] != unspecified ? layout.dimension[4] : 1, - layout.stride[4], - layout.dimension[5] != unspecified ? layout.dimension[5] : 1, - layout.stride[5], - layout.dimension[6] != unspecified ? layout.dimension[6] : 1, - layout.stride[6], - layout.dimension[7] != unspecified ? layout.dimension[7] : 1, - layout.stride[7]); - } - - // Extra overload to match that for specialize types - template - KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value || - std::is_same::value || - std::is_same::value), - typename Traits::array_layout> - createLayout(const Kokkos::Impl::ViewCtorProp& /* prop */, - const typename Traits::array_layout& layout) { - return createLayout(layout); - } - - // Create a view from the given dimension arguments. - // This is only necessary because the shmem constructor doesn't take a layout. - // NDE shmem View's are not compatible with the added view_alloc value_type - // / fad_dim deduction functionality - template - static ViewType createView(const ViewArg& arg, const size_t N0, - const size_t N1, const size_t N2, const size_t N3, - const size_t N4, const size_t N5, const size_t N6, - const size_t N7) { - return ViewType(arg, N0 != unspecified ? N0 : 1, N1 != unspecified ? N1 : 1, - N2 != unspecified ? N2 : 1, N3 != unspecified ? N3 : 1, - N4 != unspecified ? N4 : 1, N5 != unspecified ? N5 : 1, - N6 != unspecified ? N6 : 1, N7 != unspecified ? N7 : 1); - } -}; - -// Non-strided Layout -template -KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value || - std::is_same::value) && - std::is_integral::value, - Layout> -reconstructLayout(const Layout& layout, iType dynrank) { - return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, - dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, - dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, - dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, - dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, - dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, - dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, - dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX); -} - -// LayoutStride -template -KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value) && - std::is_integral::value, - Layout> -reconstructLayout(const Layout& layout, iType dynrank) { - return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, - dynrank > 0 ? layout.stride[0] : (0), - dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, - dynrank > 1 ? layout.stride[1] : (0), - dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, - dynrank > 2 ? layout.stride[2] : (0), - dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, - dynrank > 3 ? layout.stride[3] : (0), - dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, - dynrank > 4 ? layout.stride[4] : (0), - dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, - dynrank > 5 ? layout.stride[5] : (0), - dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, - dynrank > 6 ? layout.stride[6] : (0), - dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX, - dynrank > 7 ? layout.stride[7] : (0)); -} - -/** \brief Debug bounds-checking routines */ -// Enhanced debug checking - most infrastructure matches that of functions in -// Kokkos_ViewMapping; additional checks for extra arguments beyond rank are 0 -template -KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( - const iType0&, const MapType&) { - return true; -} - -template -KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( - const iType0& rank, const MapType& map, const iType1& i, Args... args) { - if (static_cast(R) < rank) { - return (size_t(i) < map.extent(R)) && - dyn_rank_view_verify_operator_bounds(rank, map, args...); - } else if (i != 0) { - KOKKOS_IMPL_DO_NOT_USE_PRINTF( - "DynRankView Debug Bounds Checking Error: at rank %u\n Extra " - "arguments beyond the rank must be zero \n", - R); - return (false) && - dyn_rank_view_verify_operator_bounds(rank, map, args...); - } else { - return (true) && - dyn_rank_view_verify_operator_bounds(rank, map, args...); - } -} - -template -inline void dyn_rank_view_error_operator_bounds(char*, int, const MapType&) {} - -template -inline void dyn_rank_view_error_operator_bounds(char* buf, int len, - const MapType& map, - const iType& i, Args... args) { - const int n = snprintf( - buf, len, " %ld < %ld %c", static_cast(i), - static_cast(map.extent(R)), (sizeof...(Args) ? ',' : ')')); - dyn_rank_view_error_operator_bounds(buf + n, len - n, map, args...); -} - -// op_rank = rank of the operator version that was called -template -KOKKOS_INLINE_FUNCTION void dyn_rank_view_verify_operator_bounds( - const iType0& op_rank, const iType1& rank, - const Kokkos::Impl::SharedAllocationTracker& tracker, const MapType& map, - Args... args) { - if (static_cast(rank) > op_rank) { - Kokkos::abort( - "DynRankView Bounds Checking Error: Need at least rank arguments to " - "the operator()"); - } - - if (!dyn_rank_view_verify_operator_bounds<0>(rank, map, args...)) { - KOKKOS_IF_ON_HOST( - (enum {LEN = 1024}; char buffer[LEN]; - const std::string label = tracker.template get_label(); - int n = snprintf(buffer, LEN, "DynRankView bounds error of view %s (", - label.c_str()); - dyn_rank_view_error_operator_bounds<0>(buffer + n, LEN - n, map, - args...); - Kokkos::Impl::throw_runtime_exception(std::string(buffer));)) - - KOKKOS_IF_ON_DEVICE( - ((void)tracker; Kokkos::abort("DynRankView bounds error");)) - } -} - -/** \brief Assign compatible default mappings */ -struct ViewToDynRankViewTag {}; - -} // namespace Impl - -namespace Impl { - -template -class ViewMapping< - DstTraits, SrcTraits, - std::enable_if_t<(std::is_same::value && - std::is_void::value && - std::is_void::value && - (std::is_same::value || - ((std::is_same::value || - std::is_same::value || - std::is_same::value) && - (std::is_same::value || - std::is_same::value || - std::is_same::value)))), - Kokkos::Impl::ViewToDynRankViewTag>> { - private: - enum { - is_assignable_value_type = - std::is_same::value || - std::is_same::value - }; - - enum { - is_assignable_layout = - std::is_same::value || - std::is_same::value - }; - - public: - enum { is_assignable = is_assignable_value_type && is_assignable_layout }; - - using DstType = ViewMapping; - using SrcType = ViewMapping; - - template - KOKKOS_INLINE_FUNCTION static void assign( - Kokkos::DynRankView& dst, const Kokkos::View& src) { - static_assert( - is_assignable_value_type, - "View assignment must have same value type or const = non-const"); - - static_assert( - is_assignable_layout, - "View assignment must have compatible layout or have rank <= 1"); - - // Removed dimension checks... - - using dst_offset_type = typename DstType::offset_type; - dst.m_map.m_impl_offset = dst_offset_type( - std::integral_constant(), - src.layout()); // Check this for integer input1 for padding, etc - dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle::assign( - src.m_map.m_impl_handle, src.m_track.m_tracker); - dst.m_track.assign(src.m_track.m_tracker, DstTraits::is_managed); - dst.m_rank = src.Rank; - } -}; - -} // namespace Impl - -/* \class DynRankView - * \brief Container that creates a Kokkos view with rank determined at runtime. - * Essentially this is a rank 7 view - * - * Changes from View - * 1. The rank of the DynRankView is returned by the method rank() - * 2. Max rank of a DynRankView is 7 - * 3. subview called with 'subview(...)' or 'subdynrankview(...)' (backward - * compatibility) - * 4. Every subview is returned with LayoutStride - * 5. Copy and Copy-Assign View to DynRankView - * 6. deep_copy between Views and DynRankViews - * 7. rank( view ); returns the rank of View or DynRankView - * - */ - -template -struct is_dyn_rank_view : public std::false_type {}; - -template -struct is_dyn_rank_view> : public std::true_type { -}; - -template -class DynRankView : public ViewTraits { - static_assert(!std::is_array::value && - !std::is_pointer::value, - "Cannot template DynRankView with array or pointer datatype - " - "must be pod"); - - private: - template - friend class DynRankView; - template - friend class Kokkos::Impl::ViewMapping; - - public: - using drvtraits = ViewTraits; - - using view_type = View; - - using traits = ViewTraits; - - private: - using map_type = - Kokkos::Impl::ViewMapping; - using track_type = Kokkos::Impl::SharedAllocationTracker; - - track_type m_track; - map_type m_map; - unsigned m_rank; - - public: - KOKKOS_INLINE_FUNCTION - view_type& DownCast() const { return (view_type&)(*this); } - KOKKOS_INLINE_FUNCTION - const view_type& ConstDownCast() const { return (const view_type&)(*this); } - - // Types below - at least the HostMirror requires the value_type, NOT the rank - // 7 data_type of the traits - - /** \brief Compatible view of array of scalar types */ - using array_type = DynRankView< - typename drvtraits::scalar_array_type, typename drvtraits::array_layout, - typename drvtraits::device_type, typename drvtraits::memory_traits>; - - /** \brief Compatible view of const data type */ - using const_type = DynRankView< - typename drvtraits::const_data_type, typename drvtraits::array_layout, - typename drvtraits::device_type, typename drvtraits::memory_traits>; - - /** \brief Compatible view of non-const data type */ - using non_const_type = DynRankView< - typename drvtraits::non_const_data_type, typename drvtraits::array_layout, - typename drvtraits::device_type, typename drvtraits::memory_traits>; - - /** \brief Compatible HostMirror view */ - using HostMirror = DynRankView; - - //---------------------------------------- - // Domain rank and extents - - // enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the - // enum? - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> - extent(const iType& r) const { - return m_map.extent(r); - } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, int> - extent_int(const iType& r) const { - return static_cast(m_map.extent(r)); - } - - KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() const; - - //---------------------------------------- - /* Deprecate all 'dimension' functions in favor of - * ISO/C++ vocabulary 'extent'. - */ - - KOKKOS_INLINE_FUNCTION constexpr size_t size() const { - return m_map.extent(0) * m_map.extent(1) * m_map.extent(2) * - m_map.extent(3) * m_map.extent(4) * m_map.extent(5) * - m_map.extent(6) * m_map.extent(7); - } - - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { - return m_map.stride_0(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { - return m_map.stride_1(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { - return m_map.stride_2(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { - return m_map.stride_3(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { - return m_map.stride_4(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { - return m_map.stride_5(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { - return m_map.stride_6(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { - return m_map.stride_7(); - } - - template - KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { - m_map.stride(s); - } - - //---------------------------------------- - // Range span is the span which contains all members. - - using reference_type = typename map_type::reference_type; - using pointer_type = typename map_type::pointer_type; - - enum { - reference_type_is_lvalue_reference = - std::is_lvalue_reference::value - }; - - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } - KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { - return m_map.span_is_contiguous(); - } - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { - return m_map.data(); - } - KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { - return (m_map.data() != nullptr); - } - - //---------------------------------------- - // Allow specializations to query their specialized map - KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping& - impl_map() const { - return m_map; - } - - //---------------------------------------- - - private: - enum { - is_layout_left = - std::is_same::value, - - is_layout_right = - std::is_same::value, - - is_layout_stride = std::is_same::value, - - is_default_map = std::is_void::value && - (is_layout_left || is_layout_right || is_layout_stride) - }; - -// Bounds checking macros -#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) - -// rank of the calling operator - included as first argument in ARG -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " \ - "space"); \ - Kokkos::Impl::dyn_rank_view_verify_operator_bounds< \ - typename traits::memory_space> \ - ARG; - -#else - -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " \ - "space"); - -#endif - - public: - KOKKOS_INLINE_FUNCTION - constexpr unsigned rank() const { return m_rank; } - - // operators () - // Rank 0 - KOKKOS_INLINE_FUNCTION - reference_type operator()() const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) - return impl_map().reference(); - // return m_map.reference(0,0,0,0,0,0,0); - } - - // Rank 1 - // This assumes a contiguous underlying memory (i.e. no padding, no - // striding...) - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - std::is_same::value && - std::is_integral::value, - reference_type> - operator[](const iType& i0) const { - // Phalanx is violating this, since they use the operator to access ALL - // elements in the allocation KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , - // this->rank(), m_track, m_map) ) - return data()[i0]; - } - - // This assumes a contiguous underlying memory (i.e. no padding, no - // striding... AND a Trilinos/Sacado scalar type ) - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - !std::is_same::value && - std::is_integral::value, - reference_type> - operator[](const iType& i0) const { - // auto map = impl_map(); - const size_t dim_scalar = m_map.dimension_scalar(); - const size_t bytes = this->span() / dim_scalar; - - using tmp_view_type = Kokkos::View< - DataType*, typename traits::array_layout, typename traits::device_type, - Kokkos::MemoryTraits>; - tmp_view_type rankone_view(this->data(), bytes, dim_scalar); - return rankone_view(i0); - } - - // Rank 1 parenthesis - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t<(std::is_void::value && - std::is_integral::value), - reference_type> - operator()(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0, 0, 0, 0, 0, 0, 0); - } - - // Rank 2 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1, 0, 0, 0, 0, 0); - } - - // Rank 3 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2, 0, 0, 0, 0); - } - - // Rank 4 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3, 0, 0, 0); - } - - // Rank 5 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4, 0, 0); - } - - // Rank 6 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5, 0); - } - - // Rank 7 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5, - const iType6& i6) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) - return m_map.reference(i0, i1, i2, i3, i4, i5, i6); - } - - // Rank 0 - KOKKOS_INLINE_FUNCTION - reference_type access() const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) - return impl_map().reference(); - // return m_map.reference(0,0,0,0,0,0,0); - } - - // Rank 1 - // Rank 1 parenthesis - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t<(std::is_void::value && - std::is_integral::value), - reference_type> - access(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0, 0, 0, 0, 0, 0, 0); - } - - // Rank 2 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1, 0, 0, 0, 0, 0); - } - - // Rank 3 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2, 0, 0, 0, 0); - } - - // Rank 4 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3, 0, 0, 0); - } - - // Rank 5 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, - const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4, 0, 0); - } - - // Rank 6 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, - const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5, 0); - } - - // Rank 7 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, - const iType4& i4, const iType5& i5, const iType6& i6) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) - return m_map.reference(i0, i1, i2, i3, i4, i5, i6); - } - -#undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY - - //---------------------------------------- - // Standard constructor, destructor, and assignment operators... - - KOKKOS_DEFAULTED_FUNCTION - ~DynRankView() = default; - - KOKKOS_INLINE_FUNCTION - DynRankView() : m_track(), m_map(), m_rank() {} // Default ctor - - KOKKOS_INLINE_FUNCTION - DynRankView(const DynRankView& rhs) - : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} - - KOKKOS_INLINE_FUNCTION - DynRankView(DynRankView&& rhs) - : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} - - KOKKOS_INLINE_FUNCTION - DynRankView& operator=(const DynRankView& rhs) { - m_track = rhs.m_track; - m_map = rhs.m_map; - m_rank = rhs.m_rank; - return *this; - } - - KOKKOS_INLINE_FUNCTION - DynRankView& operator=(DynRankView&& rhs) { - m_track = rhs.m_track; - m_map = rhs.m_map; - m_rank = rhs.m_rank; - return *this; - } - - //---------------------------------------- - // Compatible view copy constructor and assignment - // may assign unmanaged from managed. - template - KOKKOS_INLINE_FUNCTION DynRankView(const DynRankView& rhs) - : m_track(rhs.m_track, traits::is_managed), m_map(), m_rank(rhs.m_rank) { - using SrcTraits = typename DynRankView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible DynRankView copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track); - } - - template - KOKKOS_INLINE_FUNCTION DynRankView& operator=( - const DynRankView& rhs) { - using SrcTraits = typename DynRankView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible DynRankView copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track); - m_track.assign(rhs.m_track, traits::is_managed); - m_rank = rhs.rank(); - return *this; - } - - // Copy/Assign View to DynRankView - template - KOKKOS_INLINE_FUNCTION DynRankView(const View& rhs) - : m_track(), m_map(), m_rank(rhs.Rank) { - using SrcTraits = typename View::traits; - using Mapping = - Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible View to DynRankView copy construction"); - Mapping::assign(*this, rhs); - } - - template - KOKKOS_INLINE_FUNCTION DynRankView& operator=(const View& rhs) { - using SrcTraits = typename View::traits; - using Mapping = - Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible View to DynRankView copy assignment"); - Mapping::assign(*this, rhs); - return *this; - } - - //---------------------------------------- - // Allocation tracking properties - - KOKKOS_INLINE_FUNCTION - int use_count() const { return m_track.use_count(); } - - inline const std::string label() const { - return m_track.template get_label(); - } - - //---------------------------------------- - // Allocation according to allocation properties and array layout - // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that - // rank deduction can properly take place - template - explicit inline DynRankView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout) - : m_track(), - m_map(), - m_rank(Impl::DynRankDimTraits:: - template computeRank( - arg_prop, arg_layout)) { - // Append layout and spaces if not input - using alloc_prop_input = Kokkos::Impl::ViewCtorProp; - - // use 'std::integral_constant' for non-types - // to avoid duplicate class error. - using alloc_prop = Kokkos::Impl::ViewCtorProp< - P..., - std::conditional_t, std::string>, - std::conditional_t, - typename traits::device_type::memory_space>, - std::conditional_t, - typename traits::device_type::execution_space>>; - - static_assert(traits::is_managed, - "View allocation constructor requires managed memory"); - - if (alloc_prop::initialize && - !alloc_prop::execution_space::impl_is_initialized()) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception( - "Constructing DynRankView and initializing data with uninitialized " - "execution space"); - } - - // Copy the input allocation properties with possibly defaulted properties - alloc_prop prop_copy(arg_prop); - -//------------------------------------------------------------ -#if defined(KOKKOS_ENABLE_CUDA) - // If allocating in CudaUVMSpace must fence before and after - // the allocation to protect against possible concurrent access - // on the CPU and the GPU. - // Fence using the trait's executon space (which will be Kokkos::Cuda) - // to avoid incomplete type errors from usng Kokkos::Cuda directly. - if (std::is_same::value) { - typename traits::device_type::memory_space::execution_space().fence( - "Kokkos::DynRankView<>::DynRankView: fence before UVM allocation"); - } -#endif - //------------------------------------------------------------ - - Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( - prop_copy, - Impl::DynRankDimTraits:: - template createLayout(arg_prop, arg_layout), - Impl::ViewCtorProp::has_execution_space); - -//------------------------------------------------------------ -#if defined(KOKKOS_ENABLE_CUDA) - if (std::is_same::value) { - typename traits::device_type::memory_space::execution_space().fence( - "Kokkos::DynRankView<>::DynRankView: fence after UVM allocation"); - } -#endif - //------------------------------------------------------------ - - // Setup and initialization complete, start tracking - m_track.assign_allocated_record_to_uninitialized(record); - } - - // Wrappers - template - explicit KOKKOS_INLINE_FUNCTION DynRankView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout) - : m_track() // No memory tracking - , - m_map(arg_prop, - Impl::DynRankDimTraits:: - template createLayout(arg_prop, arg_layout)), - m_rank(Impl::DynRankDimTraits:: - template computeRank( - arg_prop, arg_layout)) { - static_assert( - std::is_same::pointer_type>::value, - "Constructing DynRankView to wrap user memory must supply matching " - "pointer type"); - } - - //---------------------------------------- - // Constructor(s) - - // Simple dimension-only layout - template - explicit inline DynRankView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - size_t> const arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) - : DynRankView(arg_prop, typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, - arg_N5, arg_N6, arg_N7)) {} - - template - explicit KOKKOS_INLINE_FUNCTION DynRankView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - size_t> const arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) - : DynRankView(arg_prop, typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, - arg_N5, arg_N6, arg_N7)) {} - - // Allocate with label and layout - template - explicit inline DynRankView( - const Label& arg_label, - std::enable_if_t::value, - typename traits::array_layout> const& arg_layout) - : DynRankView(Kokkos::Impl::ViewCtorProp(arg_label), - arg_layout) {} - - // Allocate label and layout, must disambiguate from subview constructor - template - explicit inline DynRankView( - const Label& arg_label, - std::enable_if_t::value, const size_t> - arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) - : DynRankView( - Kokkos::Impl::ViewCtorProp(arg_label), - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) {} - - //---------------------------------------- - // Memory span required to wrap these dimensions. - static constexpr size_t required_allocation_size( - const size_t arg_N0 = 0, const size_t arg_N1 = 0, const size_t arg_N2 = 0, - const size_t arg_N3 = 0, const size_t arg_N4 = 0, const size_t arg_N5 = 0, - const size_t arg_N6 = 0, const size_t arg_N7 = 0) { - return map_type::memory_span(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); - } - - explicit KOKKOS_INLINE_FUNCTION DynRankView( - pointer_type arg_ptr, const size_t arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) - : DynRankView(Kokkos::Impl::ViewCtorProp(arg_ptr), arg_N0, - arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7) {} - - explicit KOKKOS_INLINE_FUNCTION DynRankView( - pointer_type arg_ptr, typename traits::array_layout& arg_layout) - : DynRankView(Kokkos::Impl::ViewCtorProp(arg_ptr), - arg_layout) {} - - //---------------------------------------- - // Shared scratch memory constructor - - static inline size_t shmem_size(const size_t arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) { - const size_t num_passed_args = - (arg_N0 != KOKKOS_INVALID_INDEX) + (arg_N1 != KOKKOS_INVALID_INDEX) + - (arg_N2 != KOKKOS_INVALID_INDEX) + (arg_N3 != KOKKOS_INVALID_INDEX) + - (arg_N4 != KOKKOS_INVALID_INDEX) + (arg_N5 != KOKKOS_INVALID_INDEX) + - (arg_N6 != KOKKOS_INVALID_INDEX) + (arg_N7 != KOKKOS_INVALID_INDEX); - - if (std::is_void::value && - num_passed_args != traits::rank_dynamic) { - Kokkos::abort( - "Kokkos::View::shmem_size() rank_dynamic != number of arguments.\n"); - } - {} - - return map_type::memory_span(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); - } - - explicit KOKKOS_INLINE_FUNCTION DynRankView( - const typename traits::execution_space::scratch_memory_space& arg_space, - const typename traits::array_layout& arg_layout) - : DynRankView( - Kokkos::Impl::ViewCtorProp( - reinterpret_cast( - arg_space.get_shmem(map_type::memory_span( - Impl::DynRankDimTraits:: - createLayout(arg_layout) // is this correct? - )))), - arg_layout) {} - - explicit KOKKOS_INLINE_FUNCTION DynRankView( - const typename traits::execution_space::scratch_memory_space& arg_space, - const size_t arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) - - : DynRankView( - Kokkos::Impl::ViewCtorProp( - reinterpret_cast( - arg_space.get_shmem(map_type::memory_span( - Impl::DynRankDimTraits:: - createLayout(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, - arg_N6, arg_N7)))))), - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) {} -}; - -template -KOKKOS_INLINE_FUNCTION constexpr unsigned rank( - const DynRankView& DRV) { - return DRV.rank(); -} // needed for transition to common constexpr method in view and dynrankview - // to return rank - -//---------------------------------------------------------------------------- -// Subview mapping. -// Deduce destination view type from source view traits and subview arguments - -namespace Impl { - -struct DynRankSubviewTag {}; - -} // namespace Impl - -namespace Impl { - -template -class ViewMapping< - std::enable_if_t<(std::is_void::value && - (std::is_same::value || - std::is_same::value || - std::is_same::value)), - Kokkos::Impl::DynRankSubviewTag>, - SrcTraits, Args...> { - private: - enum { - RZ = false, - R0 = bool(is_integral_extent<0, Args...>::value), - R1 = bool(is_integral_extent<1, Args...>::value), - R2 = bool(is_integral_extent<2, Args...>::value), - R3 = bool(is_integral_extent<3, Args...>::value), - R4 = bool(is_integral_extent<4, Args...>::value), - R5 = bool(is_integral_extent<5, Args...>::value), - R6 = bool(is_integral_extent<6, Args...>::value) - }; - - enum { - rank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) + - unsigned(R4) + unsigned(R5) + unsigned(R6) - }; - - using array_layout = Kokkos::LayoutStride; - - using value_type = typename SrcTraits::value_type; - - using data_type = value_type*******; - - public: - using traits_type = Kokkos::ViewTraits; - - using type = - Kokkos::View; - - template - struct apply { - static_assert(Kokkos::is_memory_traits::value, ""); - - using traits_type = - Kokkos::ViewTraits; - - using type = Kokkos::View; - }; - - using dimension = typename SrcTraits::dimension; - - template - struct ExtentGenerator { - KOKKOS_INLINE_FUNCTION - static SubviewExtents<7, rank> generator( - const dimension& dim, Arg0 arg0 = Arg0(), Arg1 arg1 = Arg1(), - Arg2 arg2 = Arg2(), Arg3 arg3 = Arg3(), Arg4 arg4 = Arg4(), - Arg5 arg5 = Arg5(), Arg6 arg6 = Arg6()) { - return SubviewExtents<7, rank>(dim, arg0, arg1, arg2, arg3, arg4, arg5, - arg6); - } - }; - - using ret_type = Kokkos::DynRankView; - - template - KOKKOS_INLINE_FUNCTION static ret_type subview( - const unsigned src_rank, Kokkos::DynRankView const& src, - Args... args) { - using DstType = ViewMapping; - - using DstDimType = std::conditional_t< - (rank == 0), ViewDimension<>, - std::conditional_t< - (rank == 1), ViewDimension<0>, - std::conditional_t< - (rank == 2), ViewDimension<0, 0>, - std::conditional_t< - (rank == 3), ViewDimension<0, 0, 0>, - std::conditional_t< - (rank == 4), ViewDimension<0, 0, 0, 0>, - std::conditional_t< - (rank == 5), ViewDimension<0, 0, 0, 0, 0>, - std::conditional_t< - (rank == 6), ViewDimension<0, 0, 0, 0, 0, 0>, - ViewDimension<0, 0, 0, 0, 0, 0, 0>>>>>>>>; - - using dst_offset_type = ViewOffset; - using dst_handle_type = typename DstType::handle_type; - - ret_type dst; - - const SubviewExtents<7, rank> extents = ExtentGenerator::generator( - src.m_map.m_impl_offset.m_dim, args...); - - dst_offset_type tempdst(src.m_map.m_impl_offset, extents); - - dst.m_track = src.m_track; - - dst.m_map.m_impl_offset.m_dim.N0 = tempdst.m_dim.N0; - dst.m_map.m_impl_offset.m_dim.N1 = tempdst.m_dim.N1; - dst.m_map.m_impl_offset.m_dim.N2 = tempdst.m_dim.N2; - dst.m_map.m_impl_offset.m_dim.N3 = tempdst.m_dim.N3; - dst.m_map.m_impl_offset.m_dim.N4 = tempdst.m_dim.N4; - dst.m_map.m_impl_offset.m_dim.N5 = tempdst.m_dim.N5; - dst.m_map.m_impl_offset.m_dim.N6 = tempdst.m_dim.N6; - - dst.m_map.m_impl_offset.m_stride.S0 = tempdst.m_stride.S0; - dst.m_map.m_impl_offset.m_stride.S1 = tempdst.m_stride.S1; - dst.m_map.m_impl_offset.m_stride.S2 = tempdst.m_stride.S2; - dst.m_map.m_impl_offset.m_stride.S3 = tempdst.m_stride.S3; - dst.m_map.m_impl_offset.m_stride.S4 = tempdst.m_stride.S4; - dst.m_map.m_impl_offset.m_stride.S5 = tempdst.m_stride.S5; - dst.m_map.m_impl_offset.m_stride.S6 = tempdst.m_stride.S6; - - dst.m_map.m_impl_handle = - dst_handle_type(src.m_map.m_impl_handle + - src.m_map.m_impl_offset( - extents.domain_offset(0), extents.domain_offset(1), - extents.domain_offset(2), extents.domain_offset(3), - extents.domain_offset(4), extents.domain_offset(5), - extents.domain_offset(6))); - - dst.m_rank = - (src_rank > 0 ? unsigned(R0) : 0) + (src_rank > 1 ? unsigned(R1) : 0) + - (src_rank > 2 ? unsigned(R2) : 0) + (src_rank > 3 ? unsigned(R3) : 0) + - (src_rank > 4 ? unsigned(R4) : 0) + (src_rank > 5 ? unsigned(R5) : 0) + - (src_rank > 6 ? unsigned(R6) : 0); - - return dst; - } -}; - -} // namespace Impl - -template -using Subdynrankview = - typename Kokkos::Impl::ViewMapping::ret_type; - -template -KOKKOS_INLINE_FUNCTION Subdynrankview, Args...> -subdynrankview(const Kokkos::DynRankView& src, Args... args) { - if (src.rank() > sizeof...(Args)) // allow sizeof...(Args) >= src.rank(), - // ignore the remaining args - { - Kokkos::abort( - "subdynrankview: num of args must be >= rank of the source " - "DynRankView"); - } - - using metafcn = - Kokkos::Impl::ViewMapping, Args...>; - - return metafcn::subview(src.rank(), src, args...); -} - -// Wrapper to allow subview function name -template -KOKKOS_INLINE_FUNCTION Subdynrankview, Args...> -subview(const Kokkos::DynRankView& src, Args... args) { - return subdynrankview(src, args...); -} - -} // namespace Kokkos - -namespace Kokkos { - -// overload == and != -template -KOKKOS_INLINE_FUNCTION bool operator==(const DynRankView& lhs, - const DynRankView& rhs) { - // Same data, layout, dimensions - using lhs_traits = ViewTraits; - using rhs_traits = ViewTraits; - - return std::is_same::value && - std::is_same::value && - std::is_same::value && - lhs.rank() == rhs.rank() && lhs.data() == rhs.data() && - lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && - lhs.extent(1) == rhs.extent(1) && lhs.extent(2) == rhs.extent(2) && - lhs.extent(3) == rhs.extent(3) && lhs.extent(4) == rhs.extent(4) && - lhs.extent(5) == rhs.extent(5) && lhs.extent(6) == rhs.extent(6) && - lhs.extent(7) == rhs.extent(7); -} - -template -KOKKOS_INLINE_FUNCTION bool operator!=(const DynRankView& lhs, - const DynRankView& rhs) { - return !(operator==(lhs, rhs)); -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -namespace Kokkos { -namespace Impl { - -template -struct DynRankViewFill { - using const_value_type = typename OutputView::traits::const_value_type; - - const OutputView output; - const_value_type input; - - KOKKOS_INLINE_FUNCTION - void operator()(const size_t i0) const { - const size_t n1 = output.extent(1); - const size_t n2 = output.extent(2); - const size_t n3 = output.extent(3); - const size_t n4 = output.extent(4); - const size_t n5 = output.extent(5); - const size_t n6 = output.extent(6); - - for (size_t i1 = 0; i1 < n1; ++i1) { - for (size_t i2 = 0; i2 < n2; ++i2) { - for (size_t i3 = 0; i3 < n3; ++i3) { - for (size_t i4 = 0; i4 < n4; ++i4) { - for (size_t i5 = 0; i5 < n5; ++i5) { - for (size_t i6 = 0; i6 < n6; ++i6) { - output.access(i0, i1, i2, i3, i4, i5, i6) = input; - } - } - } - } - } - } - } - - DynRankViewFill(const OutputView& arg_out, const_value_type& arg_in) - : output(arg_out), input(arg_in) { - using execution_space = typename OutputView::execution_space; - using Policy = Kokkos::RangePolicy; - - Kokkos::parallel_for("Kokkos::DynRankViewFill", Policy(0, output.extent(0)), - *this); - } -}; - -template -struct DynRankViewFill> { - DynRankViewFill(const OutputView& dst, - const typename OutputView::const_value_type& src) { - Kokkos::Impl::DeepCopy( - dst.data(), &src, sizeof(typename OutputView::const_value_type)); - } -}; - -template -struct DynRankViewRemap { - const OutputView output; - const InputView input; - const size_t n0; - const size_t n1; - const size_t n2; - const size_t n3; - const size_t n4; - const size_t n5; - const size_t n6; - const size_t n7; - - DynRankViewRemap(const ExecSpace& exec_space, const OutputView& arg_out, - const InputView& arg_in) - : output(arg_out), - input(arg_in), - n0(std::min((size_t)arg_out.extent(0), (size_t)arg_in.extent(0))), - n1(std::min((size_t)arg_out.extent(1), (size_t)arg_in.extent(1))), - n2(std::min((size_t)arg_out.extent(2), (size_t)arg_in.extent(2))), - n3(std::min((size_t)arg_out.extent(3), (size_t)arg_in.extent(3))), - n4(std::min((size_t)arg_out.extent(4), (size_t)arg_in.extent(4))), - n5(std::min((size_t)arg_out.extent(5), (size_t)arg_in.extent(5))), - n6(std::min((size_t)arg_out.extent(6), (size_t)arg_in.extent(6))), - n7(std::min((size_t)arg_out.extent(7), (size_t)arg_in.extent(7))) { - using Policy = Kokkos::RangePolicy; - - Kokkos::parallel_for("Kokkos::DynRankViewRemap", Policy(exec_space, 0, n0), - *this); - } - - DynRankViewRemap(const OutputView& arg_out, const InputView& arg_in) - : output(arg_out), - input(arg_in), - n0(std::min((size_t)arg_out.extent(0), (size_t)arg_in.extent(0))), - n1(std::min((size_t)arg_out.extent(1), (size_t)arg_in.extent(1))), - n2(std::min((size_t)arg_out.extent(2), (size_t)arg_in.extent(2))), - n3(std::min((size_t)arg_out.extent(3), (size_t)arg_in.extent(3))), - n4(std::min((size_t)arg_out.extent(4), (size_t)arg_in.extent(4))), - n5(std::min((size_t)arg_out.extent(5), (size_t)arg_in.extent(5))), - n6(std::min((size_t)arg_out.extent(6), (size_t)arg_in.extent(6))), - n7(std::min((size_t)arg_out.extent(7), (size_t)arg_in.extent(7))) { - using Policy = Kokkos::RangePolicy; - - Kokkos::parallel_for("Kokkos::DynRankViewRemap", Policy(0, n0), *this); - } - - KOKKOS_INLINE_FUNCTION - void operator()(const size_t i0) const { - for (size_t i1 = 0; i1 < n1; ++i1) { - for (size_t i2 = 0; i2 < n2; ++i2) { - for (size_t i3 = 0; i3 < n3; ++i3) { - for (size_t i4 = 0; i4 < n4; ++i4) { - for (size_t i5 = 0; i5 < n5; ++i5) { - for (size_t i6 = 0; i6 < n6; ++i6) { - output.access(i0, i1, i2, i3, i4, i5, i6) = - input.access(i0, i1, i2, i3, i4, i5, i6); - } - } - } - } - } - } - } -}; - -} /* namespace Impl */ -} /* namespace Kokkos */ - -namespace Kokkos { - -namespace Impl { - -/* \brief Returns a View of the requested rank, aliasing the - underlying memory, to facilitate implementation of deep_copy() and - other routines that are defined on View */ -template -KOKKOS_FUNCTION auto as_view_of_rank_n(DynRankView v) { - if (v.rank() != N) { - KOKKOS_IF_ON_HOST( - const std::string message = - "Converting DynRankView of rank " + std::to_string(v.rank()) + - " to a View of mis-matched rank " + std::to_string(N) + "!"; - Kokkos::abort(message.c_str());) - KOKKOS_IF_ON_DEVICE( - Kokkos::abort("Converting DynRankView to a View of mis-matched rank!");) - } - - return View::type, Args...>( - v.data(), v.impl_map().layout()); -} - -template -void apply_to_view_of_static_rank(Function&& f, DynRankView a) { - switch (rank(a)) { - case 0: f(as_view_of_rank_n<0>(a)); break; - case 1: f(as_view_of_rank_n<1>(a)); break; - case 2: f(as_view_of_rank_n<2>(a)); break; - case 3: f(as_view_of_rank_n<3>(a)); break; - case 4: f(as_view_of_rank_n<4>(a)); break; - case 5: f(as_view_of_rank_n<5>(a)); break; - case 6: f(as_view_of_rank_n<6>(a)); break; - case 7: f(as_view_of_rank_n<7>(a)); break; - default: - KOKKOS_IF_ON_HOST( - Kokkos::abort( - std::string( - "Trying to apply a function to a view of unexpected rank " + - std::to_string(rank(a))) - .c_str());) - KOKKOS_IF_ON_DEVICE( - Kokkos::abort( - "Trying to apply a function to a view of unexpected rank");) - } -} - -} // namespace Impl - -template -KOKKOS_INLINE_FUNCTION constexpr auto DynRankView::layout() const -> - typename traits::array_layout { - switch (rank()) { - case 0: return Impl::as_view_of_rank_n<0>(*this).layout(); - case 1: return Impl::as_view_of_rank_n<1>(*this).layout(); - case 2: return Impl::as_view_of_rank_n<2>(*this).layout(); - case 3: return Impl::as_view_of_rank_n<3>(*this).layout(); - case 4: return Impl::as_view_of_rank_n<4>(*this).layout(); - case 5: return Impl::as_view_of_rank_n<5>(*this).layout(); - case 6: return Impl::as_view_of_rank_n<6>(*this).layout(); - case 7: return Impl::as_view_of_rank_n<7>(*this).layout(); - default: - KOKKOS_IF_ON_HOST( - Kokkos::abort( - std::string( - "Calling DynRankView::layout on DRV of unexpected rank " + - std::to_string(rank())) - .c_str());) - KOKKOS_IF_ON_DEVICE( - Kokkos::abort( - "Calling DynRankView::layout on DRV of unexpected rank");) - } - // control flow should never reach here - return m_map.layout(); -} - -/** \brief Deep copy a value from Host memory into a view. */ -template -inline void deep_copy( - const ExecSpace& e, const DynRankView& dst, - typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - static_assert( - std::is_same::non_const_value_type, - typename ViewTraits::value_type>::value, - "deep_copy requires non-const type"); - - Impl::apply_to_view_of_static_rank( - [=](auto view) { deep_copy(e, view, value); }, dst); -} - -template -inline void deep_copy( - const DynRankView& dst, - typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - Impl::apply_to_view_of_static_rank([=](auto view) { deep_copy(view, value); }, - dst); -} - -/** \brief Deep copy into a value in Host memory from a view. */ -template -inline void deep_copy( - const ExecSpace& e, - typename ViewTraits::non_const_value_type& dst, - const DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = 0) { - deep_copy(e, dst, Impl::as_view_of_rank_n<0>(src)); -} - -template -inline void deep_copy( - typename ViewTraits::non_const_value_type& dst, - const DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = 0) { - deep_copy(dst, Impl::as_view_of_rank_n<0>(src)); -} - -//---------------------------------------------------------------------------- -/** \brief A deep copy between views of the default specialization, compatible - * type, same rank, same contiguous layout. - * - * A rank mismatch will error out in the attempt to convert to a View - */ -template -inline void deep_copy( - const ExecSpace& exec_space, const DstType& dst, const SrcType& src, - std::enable_if_t< - (std::is_void::value && - std::is_void::value && - (Kokkos::is_dyn_rank_view::value || - Kokkos::is_dyn_rank_view::value))>* = nullptr) { - static_assert( - std::is_same::value, - "deep_copy requires non-const destination type"); - - switch (rank(dst)) { - case 0: - deep_copy(exec_space, Impl::as_view_of_rank_n<0>(dst), - Impl::as_view_of_rank_n<0>(src)); - break; - case 1: - deep_copy(exec_space, Impl::as_view_of_rank_n<1>(dst), - Impl::as_view_of_rank_n<1>(src)); - break; - case 2: - deep_copy(exec_space, Impl::as_view_of_rank_n<2>(dst), - Impl::as_view_of_rank_n<2>(src)); - break; - case 3: - deep_copy(exec_space, Impl::as_view_of_rank_n<3>(dst), - Impl::as_view_of_rank_n<3>(src)); - break; - case 4: - deep_copy(exec_space, Impl::as_view_of_rank_n<4>(dst), - Impl::as_view_of_rank_n<4>(src)); - break; - case 5: - deep_copy(exec_space, Impl::as_view_of_rank_n<5>(dst), - Impl::as_view_of_rank_n<5>(src)); - break; - case 6: - deep_copy(exec_space, Impl::as_view_of_rank_n<6>(dst), - Impl::as_view_of_rank_n<6>(src)); - break; - case 7: - deep_copy(exec_space, Impl::as_view_of_rank_n<7>(dst), - Impl::as_view_of_rank_n<7>(src)); - break; - default: - Kokkos::Impl::throw_runtime_exception( - "Calling DynRankView deep_copy with a view of unexpected rank " + - std::to_string(rank(dst))); - } -} - -template -inline void deep_copy( - const DstType& dst, const SrcType& src, - std::enable_if_t< - (std::is_void::value && - std::is_void::value && - (Kokkos::is_dyn_rank_view::value || - Kokkos::is_dyn_rank_view::value))>* = nullptr) { - static_assert( - std::is_same::value, - "deep_copy requires non-const destination type"); - - switch (rank(dst)) { - case 0: - deep_copy(Impl::as_view_of_rank_n<0>(dst), - Impl::as_view_of_rank_n<0>(src)); - break; - case 1: - deep_copy(Impl::as_view_of_rank_n<1>(dst), - Impl::as_view_of_rank_n<1>(src)); - break; - case 2: - deep_copy(Impl::as_view_of_rank_n<2>(dst), - Impl::as_view_of_rank_n<2>(src)); - break; - case 3: - deep_copy(Impl::as_view_of_rank_n<3>(dst), - Impl::as_view_of_rank_n<3>(src)); - break; - case 4: - deep_copy(Impl::as_view_of_rank_n<4>(dst), - Impl::as_view_of_rank_n<4>(src)); - break; - case 5: - deep_copy(Impl::as_view_of_rank_n<5>(dst), - Impl::as_view_of_rank_n<5>(src)); - break; - case 6: - deep_copy(Impl::as_view_of_rank_n<6>(dst), - Impl::as_view_of_rank_n<6>(src)); - break; - case 7: - deep_copy(Impl::as_view_of_rank_n<7>(dst), - Impl::as_view_of_rank_n<7>(src)); - break; - default: - Kokkos::Impl::throw_runtime_exception( - "Calling DynRankView deep_copy with a view of unexpected rank " + - std::to_string(rank(dst))); - } -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -// Deduce Mirror Types -template -struct MirrorDRViewType { - // The incoming view_type - using src_view_type = typename Kokkos::DynRankView; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it. - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using dest_view_type = Kokkos::DynRankView; - // If it is the same memory_space return the existsing view_type - // This will also keep the unmanaged trait if necessary - using view_type = - std::conditional_t; -}; - -template -struct MirrorDRVType { - // The incoming view_type - using src_view_type = typename Kokkos::DynRankView; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it. - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using view_type = Kokkos::DynRankView; -}; - -} // namespace Impl - -namespace Impl { -template -inline typename DynRankView::HostMirror create_mirror( - const DynRankView& src, - const Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_memory_space>* = - nullptr) { - using src_type = DynRankView; - using dst_type = typename src_type::HostMirror; - - using alloc_prop_input = Impl::ViewCtorProp; - - static_assert( - !alloc_prop_input::has_label, - "The view constructor arguments passed to Kokkos::create_mirror " - "must not include a label!"); - static_assert( - !alloc_prop_input::has_pointer, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not include a pointer!"); - static_assert( - !alloc_prop_input::allow_padding, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not explicitly allow padding!"); - - using alloc_prop = Impl::ViewCtorProp; - alloc_prop prop_copy(arg_prop); - static_cast&>(prop_copy).value = - std::string(src.label()).append("_mirror"); - - return dst_type(prop_copy, Impl::reconstructLayout(src.layout(), src.rank())); -} - -template -inline auto create_mirror( - const DynRankView& src, - const Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_memory_space>* = - nullptr) { - using dst_type = typename Impl::MirrorDRVType< - typename Impl::ViewCtorProp::memory_space, T, - P...>::view_type; - - using alloc_prop_input = Impl::ViewCtorProp; - - static_assert( - !alloc_prop_input::has_label, - "The view constructor arguments passed to Kokkos::create_mirror " - "must not include a label!"); - static_assert( - !alloc_prop_input::has_pointer, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not include a pointer!"); - static_assert( - !alloc_prop_input::allow_padding, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not explicitly allow padding!"); - - using alloc_prop = Impl::ViewCtorProp; - alloc_prop prop_copy(arg_prop); - static_cast&>(prop_copy).value = - std::string(src.label()).append("_mirror"); - - return dst_type(prop_copy, Impl::reconstructLayout(src.layout(), src.rank())); -} - -} // namespace Impl - -// Create a mirror in host space -template -inline typename DynRankView::HostMirror create_mirror( - const DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - return Impl::create_mirror(src, Kokkos::Impl::ViewCtorProp<>{}); -} - -template -inline typename DynRankView::HostMirror create_mirror( - Kokkos::Impl::WithoutInitializing_t wi, const DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - return Impl::create_mirror(src, Kokkos::view_alloc(wi)); -} - -template -inline typename DynRankView::HostMirror create_mirror( - const Impl::ViewCtorProp& arg_prop, - const DynRankView& src, - std::enable_if_t< - std::is_void::specialize>::value && - !Impl::ViewCtorProp::has_memory_space>* = nullptr) { - return Impl::create_mirror(src, arg_prop); -} - -// Create a mirror in a new space -template ::value && - std::is_void::specialize>::value>> -typename Impl::MirrorDRVType::view_type create_mirror( - const Space&, const Kokkos::DynRankView& src) { - return Impl::create_mirror( - src, Kokkos::view_alloc(typename Space::memory_space{})); -} - -template -typename Impl::MirrorDRVType::view_type create_mirror( - Kokkos::Impl::WithoutInitializing_t wi, const Space&, - const Kokkos::DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - return Impl::create_mirror( - src, Kokkos::view_alloc(wi, typename Space::memory_space{})); -} - -template -inline auto create_mirror( - const Impl::ViewCtorProp& arg_prop, - const DynRankView& src, - std::enable_if_t< - std::is_void::specialize>::value && - Impl::ViewCtorProp::has_memory_space>* = nullptr) { - using ReturnType = typename Impl::MirrorDRVType< - typename Impl::ViewCtorProp::memory_space, T, - P...>::view_type; - return ReturnType{Impl::create_mirror(src, arg_prop)}; -} - -namespace Impl { -template -inline std::enable_if_t< - std::is_same< - typename DynRankView::memory_space, - typename DynRankView::HostMirror::memory_space>::value && - std::is_same< - typename DynRankView::data_type, - typename DynRankView::HostMirror::data_type>::value, - typename DynRankView::HostMirror> -create_mirror_view(const DynRankView& src, - const typename Impl::ViewCtorProp&) { - return src; -} - -template -inline std::enable_if_t< - !(std::is_same< - typename DynRankView::memory_space, - typename DynRankView::HostMirror::memory_space>::value && - std::is_same< - typename DynRankView::data_type, - typename DynRankView::HostMirror::data_type>::value), - typename DynRankView::HostMirror> -create_mirror_view( - const DynRankView& src, - const typename Impl::ViewCtorProp& arg_prop) { - return Kokkos::Impl::create_mirror(src, arg_prop); -} - -template -inline std::enable_if_t< - Kokkos::is_space::value && - Impl::MirrorDRViewType::is_same_memspace, - typename Impl::MirrorDRViewType::view_type> -create_mirror_view(const Space&, const Kokkos::DynRankView& src, - const typename Impl::ViewCtorProp&) { - return src; -} - -template -inline std::enable_if_t< - Kokkos::is_space::value && - !Impl::MirrorDRViewType::is_same_memspace, - typename Impl::MirrorDRViewType::view_type> -create_mirror_view( - const Space&, const Kokkos::DynRankView& src, - const typename Impl::ViewCtorProp& arg_prop) { - using MemorySpace = typename Space::memory_space; - using alloc_prop = Impl::ViewCtorProp; - alloc_prop prop_copy(arg_prop); - - return Kokkos::Impl::create_mirror(src, prop_copy); -} -} // namespace Impl - -// Create a mirror view in host space -template -inline std::enable_if_t< - (std::is_same< - typename DynRankView::memory_space, - typename DynRankView::HostMirror::memory_space>::value && - std::is_same::data_type, - typename DynRankView::HostMirror::data_type>::value), - typename DynRankView::HostMirror> -create_mirror_view(const Kokkos::DynRankView& src) { - return src; -} - -template -inline std::enable_if_t< - !(std::is_same< - typename DynRankView::memory_space, - typename DynRankView::HostMirror::memory_space>::value && - std::is_same< - typename DynRankView::data_type, - typename DynRankView::HostMirror::data_type>::value), - typename DynRankView::HostMirror> -create_mirror_view(const Kokkos::DynRankView& src) { - return Kokkos::create_mirror(src); -} - -template -inline auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi, - const DynRankView& src) { - return Impl::create_mirror_view(src, Kokkos::view_alloc(wi)); -} - -// Create a mirror view in a new space -// FIXME_C++17 Improve SFINAE here. -template ::value>> -inline typename Impl::MirrorDRViewType::view_type -create_mirror_view( - const Space&, const Kokkos::DynRankView& src, - std::enable_if_t< - Impl::MirrorDRViewType::is_same_memspace>* = nullptr) { - return src; -} - -// FIXME_C++17 Improve SFINAE here. -template ::value>> -inline typename Impl::MirrorDRViewType::view_type -create_mirror_view( - const Space& space, const Kokkos::DynRankView& src, - std::enable_if_t< - !Impl::MirrorDRViewType::is_same_memspace>* = nullptr) { - return Kokkos::create_mirror(space, src); -} - -template -inline auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi, - const Space& space, - const Kokkos::DynRankView& src) { - return Impl::create_mirror_view(space, src, Kokkos::view_alloc(wi)); -} - -template -inline auto create_mirror_view( - const typename Impl::ViewCtorProp& arg_prop, - const Kokkos::DynRankView& src) { - return Impl::create_mirror_view(src, arg_prop); -} - -template -auto create_mirror_view_and_copy( - const Impl::ViewCtorProp&, - const Kokkos::DynRankView& src, - std::enable_if_t< - std::is_void::specialize>::value && - Impl::MirrorDRViewType< - typename Impl::ViewCtorProp::memory_space, T, - P...>::is_same_memspace>* = nullptr) { - using alloc_prop_input = Impl::ViewCtorProp; - static_assert( - alloc_prop_input::has_memory_space, - "The view constructor arguments passed to " - "Kokkos::create_mirror_view_and_copy must include a memory space!"); - static_assert(!alloc_prop_input::has_pointer, - "The view constructor arguments passed to " - "Kokkos::create_mirror_view_and_copy must " - "not include a pointer!"); - static_assert(!alloc_prop_input::allow_padding, - "The view constructor arguments passed to " - "Kokkos::create_mirror_view_and_copy must " - "not explicitly allow padding!"); - - // same behavior as deep_copy(src, src) - if (!alloc_prop_input::has_execution_space) - fence( - "Kokkos::create_mirror_view_and_copy: fence before returning src view"); - return src; -} - -template -auto create_mirror_view_and_copy( - const Impl::ViewCtorProp& arg_prop, - const Kokkos::DynRankView& src, - std::enable_if_t< - std::is_void::specialize>::value && - !Impl::MirrorDRViewType< - typename Impl::ViewCtorProp::memory_space, T, - P...>::is_same_memspace>* = nullptr) { - using alloc_prop_input = Impl::ViewCtorProp; - static_assert( - alloc_prop_input::has_memory_space, - "The view constructor arguments passed to " - "Kokkos::create_mirror_view_and_copy must include a memory space!"); - static_assert(!alloc_prop_input::has_pointer, - "The view constructor arguments passed to " - "Kokkos::create_mirror_view_and_copy must " - "not include a pointer!"); - static_assert(!alloc_prop_input::allow_padding, - "The view constructor arguments passed to " - "Kokkos::create_mirror_view_and_copy must " - "not explicitly allow padding!"); - using Space = typename alloc_prop_input::memory_space; - using Mirror = typename Impl::MirrorDRViewType::view_type; - - // Add some properties if not provided to avoid need for if constexpr - using alloc_prop = Impl::ViewCtorProp< - ViewCtorArgs..., - std::conditional_t, std::string>, - std::conditional_t, - Impl::WithoutInitializing_t>, - std::conditional_t, - typename Space::execution_space>>; - alloc_prop arg_prop_copy(arg_prop); - - std::string& label = - static_cast&>(arg_prop_copy).value; - if (label.empty()) label = src.label(); - auto mirror = typename Mirror::non_const_type{ - arg_prop_copy, Impl::reconstructLayout(src.layout(), src.rank())}; - if (alloc_prop_input::has_execution_space) { - using ExecutionSpace = typename alloc_prop::execution_space; - deep_copy( - static_cast&>(arg_prop_copy) - .value, - mirror, src); - } else - deep_copy(mirror, src); - return mirror; -} - -template -auto create_mirror_view_and_copy(const Space&, - const Kokkos::DynRankView& src, - std::string const& name = "") { - return create_mirror_view_and_copy( - Kokkos::view_alloc(typename Space::memory_space{}, name), src); -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -/** \brief Resize a view with copying old data to new data at the corresponding - * indices. */ -template -inline void impl_resize(const Impl::ViewCtorProp& arg_prop, - DynRankView& v, const size_t n0, - const size_t n1, const size_t n2, const size_t n3, - const size_t n4, const size_t n5, const size_t n6, - const size_t n7) { - using drview_type = DynRankView; - using alloc_prop_input = Impl::ViewCtorProp; - - static_assert(Kokkos::ViewTraits::is_managed, - "Can only resize managed views"); - static_assert(!alloc_prop_input::has_label, - "The view constructor arguments passed to Kokkos::resize " - "must not include a label!"); - static_assert(!alloc_prop_input::has_pointer, - "The view constructor arguments passed to Kokkos::resize must " - "not include a pointer!"); - static_assert(!alloc_prop_input::has_memory_space, - "The view constructor arguments passed to Kokkos::resize must " - "not include a memory space instance!"); - - // Add execution space here to avoid the need for if constexpr below - using alloc_prop = Impl::ViewCtorProp< - ViewCtorArgs..., std::string, - std::conditional_t, - typename drview_type::execution_space>>; - alloc_prop prop_copy(arg_prop); - static_cast&>(prop_copy).value = - v.label(); - - drview_type v_resized(prop_copy, n0, n1, n2, n3, n4, n5, n6, n7); - - if (alloc_prop_input::has_execution_space) - Kokkos::Impl::DynRankViewRemap( - static_cast&>(prop_copy) - .value, - v_resized, v); - else - Kokkos::Impl::DynRankViewRemap(v_resized, v); - - v = v_resized; -} - -template -inline void resize(DynRankView& v, - const size_t n0 = KOKKOS_INVALID_INDEX, - const size_t n1 = KOKKOS_INVALID_INDEX, - const size_t n2 = KOKKOS_INVALID_INDEX, - const size_t n3 = KOKKOS_INVALID_INDEX, - const size_t n4 = KOKKOS_INVALID_INDEX, - const size_t n5 = KOKKOS_INVALID_INDEX, - const size_t n6 = KOKKOS_INVALID_INDEX, - const size_t n7 = KOKKOS_INVALID_INDEX) { - impl_resize(Impl::ViewCtorProp<>{}, v, n0, n1, n2, n3, n4, n5, n6, n7); -} - -template -void resize(const Impl::ViewCtorProp& arg_prop, - DynRankView& v, - const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) { - impl_resize(arg_prop, v, n0, n1, n2, n3, n4, n5, n6, n7); -} - -template -inline std::enable_if_t::value> resize( - const I& arg_prop, DynRankView& v, - const size_t n0 = KOKKOS_INVALID_INDEX, - const size_t n1 = KOKKOS_INVALID_INDEX, - const size_t n2 = KOKKOS_INVALID_INDEX, - const size_t n3 = KOKKOS_INVALID_INDEX, - const size_t n4 = KOKKOS_INVALID_INDEX, - const size_t n5 = KOKKOS_INVALID_INDEX, - const size_t n6 = KOKKOS_INVALID_INDEX, - const size_t n7 = KOKKOS_INVALID_INDEX) { - impl_resize(Kokkos::view_alloc(arg_prop), v, n0, n1, n2, n3, n4, n5, n6, n7); -} - -/** \brief Resize a view with copying old data to new data at the corresponding - * indices. */ -template -inline void impl_realloc(DynRankView& v, const size_t n0, - const size_t n1, const size_t n2, const size_t n3, - const size_t n4, const size_t n5, const size_t n6, - const size_t n7, - const Impl::ViewCtorProp& arg_prop) { - using drview_type = DynRankView; - using alloc_prop_input = Impl::ViewCtorProp; - - static_assert(Kokkos::ViewTraits::is_managed, - "Can only realloc managed views"); - static_assert(!alloc_prop_input::has_label, - "The view constructor arguments passed to Kokkos::realloc must " - "not include a label!"); - static_assert(!alloc_prop_input::has_pointer, - "The view constructor arguments passed to Kokkos::realloc must " - "not include a pointer!"); - static_assert(!alloc_prop_input::has_memory_space, - "The view constructor arguments passed to Kokkos::realloc must " - "not include a memory space instance!"); - - using alloc_prop = Impl::ViewCtorProp; - alloc_prop arg_prop_copy(arg_prop); - static_cast&>(arg_prop_copy) - .value = v.label(); - - v = drview_type(); // Deallocate first, if the only view to allocation - v = drview_type(arg_prop_copy, n0, n1, n2, n3, n4, n5, n6, n7); -} - -template -inline void realloc(const Impl::ViewCtorProp& arg_prop, - DynRankView& v, - const size_t n0 = KOKKOS_INVALID_INDEX, - const size_t n1 = KOKKOS_INVALID_INDEX, - const size_t n2 = KOKKOS_INVALID_INDEX, - const size_t n3 = KOKKOS_INVALID_INDEX, - const size_t n4 = KOKKOS_INVALID_INDEX, - const size_t n5 = KOKKOS_INVALID_INDEX, - const size_t n6 = KOKKOS_INVALID_INDEX, - const size_t n7 = KOKKOS_INVALID_INDEX) { - impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, arg_prop); -} - -template -inline void realloc(DynRankView& v, - const size_t n0 = KOKKOS_INVALID_INDEX, - const size_t n1 = KOKKOS_INVALID_INDEX, - const size_t n2 = KOKKOS_INVALID_INDEX, - const size_t n3 = KOKKOS_INVALID_INDEX, - const size_t n4 = KOKKOS_INVALID_INDEX, - const size_t n5 = KOKKOS_INVALID_INDEX, - const size_t n6 = KOKKOS_INVALID_INDEX, - const size_t n7 = KOKKOS_INVALID_INDEX) { - impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Impl::ViewCtorProp<>{}); -} - -template -inline std::enable_if_t::value> realloc( - const I& arg_prop, DynRankView& v, - const size_t n0 = KOKKOS_INVALID_INDEX, - const size_t n1 = KOKKOS_INVALID_INDEX, - const size_t n2 = KOKKOS_INVALID_INDEX, - const size_t n3 = KOKKOS_INVALID_INDEX, - const size_t n4 = KOKKOS_INVALID_INDEX, - const size_t n5 = KOKKOS_INVALID_INDEX, - const size_t n6 = KOKKOS_INVALID_INDEX, - const size_t n7 = KOKKOS_INVALID_INDEX) { - impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Kokkos::view_alloc(arg_prop)); -} - -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNRANKVIEW -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNRANKVIEW -#endif -#endif diff --git a/bundled/kokkos-3.7.00/containers/src/Kokkos_Functional.hpp b/bundled/kokkos-3.7.00/containers/src/Kokkos_Functional.hpp deleted file mode 100644 index 478a087d0c..0000000000 --- a/bundled/kokkos-3.7.00/containers/src/Kokkos_Functional.hpp +++ /dev/null @@ -1,183 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER - -#ifndef KOKKOS_FUNCTIONAL_HPP -#define KOKKOS_FUNCTIONAL_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_FUNCTIONAL -#endif - -#include -#include - -namespace Kokkos { - -// These should work for most types - -template -struct pod_hash { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using argument_type KOKKOS_DEPRECATED = T; - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = uint32_t; - using result_type KOKKOS_DEPRECATED = uint32_t; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - uint32_t operator()(T const& t) const { - return Impl::MurmurHash3_x86_32(&t, sizeof(T), 0); - } - - KOKKOS_FORCEINLINE_FUNCTION - uint32_t operator()(T const& t, uint32_t seed) const { - return Impl::MurmurHash3_x86_32(&t, sizeof(T), seed); - } -}; - -template -struct pod_equal_to { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { - return Impl::bitwise_equal(&a, &b); - } -}; - -template -struct pod_not_equal_to { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { - return !Impl::bitwise_equal(&a, &b); - } -}; - -template -struct equal_to { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { return a == b; } -}; - -template -struct not_equal_to { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { return a != b; } -}; - -template -struct greater { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { return a > b; } -}; - -template -struct less { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { return a < b; } -}; - -template -struct greater_equal { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { return a >= b; } -}; - -template -struct less_equal { -#if defined KOKKOS_ENABLE_DEPRECATED_CODE_3 - using first_argument_type KOKKOS_DEPRECATED = T; - using second_argument_type KOKKOS_DEPRECATED = T; - using result_type KOKKOS_DEPRECATED = bool; -#endif - - KOKKOS_FORCEINLINE_FUNCTION - bool operator()(T const& a, T const& b) const { return a <= b; } -}; - -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_FUNCTIONAL -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_FUNCTIONAL -#endif -#endif // KOKKOS_FUNCTIONAL_HPP diff --git a/bundled/kokkos-3.7.00/containers/src/Kokkos_OffsetView.hpp b/bundled/kokkos-3.7.00/containers/src/Kokkos_OffsetView.hpp deleted file mode 100644 index 0b54d1bdd9..0000000000 --- a/bundled/kokkos-3.7.00/containers/src/Kokkos_OffsetView.hpp +++ /dev/null @@ -1,2100 +0,0 @@ -/* - * Kokkos_OffsetView.hpp - * - * Created on: Apr 23, 2018 - * Author: swbova - */ - -#ifndef KOKKOS_OFFSETVIEW_HPP_ -#define KOKKOS_OFFSETVIEW_HPP_ -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_OFFSETVIEW -#endif - -#include - -#include - -namespace Kokkos { - -namespace Experimental { -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -class OffsetView; - -template -struct is_offset_view : public std::false_type {}; - -template -struct is_offset_view> : public std::true_type {}; - -template -struct is_offset_view> : public std::true_type {}; - -#define KOKKOS_INVALID_OFFSET int64_t(0x7FFFFFFFFFFFFFFFLL) -#define KOKKOS_INVALID_INDEX_RANGE \ - { KOKKOS_INVALID_OFFSET, KOKKOS_INVALID_OFFSET } - -template ::value && - std::is_signed::value, - iType> = 0> -using IndexRange = Kokkos::Array; - -using index_list_type = std::initializer_list; - -// template ::value && -// std::is_signed::value, iType > = 0> using min_index_type = -// std::initializer_list; - -namespace Impl { - -template -struct GetOffsetViewTypeFromViewType { - using type = - OffsetView; -}; - -template -KOKKOS_INLINE_FUNCTION bool offsetview_verify_operator_bounds( - const MapType&, const BeginsType&) { - return true; -} - -template -KOKKOS_INLINE_FUNCTION bool offsetview_verify_operator_bounds( - const MapType& map, const BeginsType& begins, const iType& i, - Args... args) { - const bool legalIndex = - (int64_t(i) >= begins[R]) && - (int64_t(i) <= int64_t(begins[R] + map.extent(R) - 1)); - return legalIndex && - offsetview_verify_operator_bounds(map, begins, args...); -} -template -inline void offsetview_error_operator_bounds(char*, int, const MapType&, - const BeginsType&) {} - -template -inline void offsetview_error_operator_bounds(char* buf, int len, - const MapType& map, - const BeginsType begins, - const iType& i, Args... args) { - const int64_t b = begins[R]; - const int64_t e = b + map.extent(R) - 1; - const int n = - snprintf(buf, len, " %ld <= %ld <= %ld %c", static_cast(b), - static_cast(i), static_cast(e), - (sizeof...(Args) ? ',' : ')')); - offsetview_error_operator_bounds(buf + n, len - n, map, begins, - args...); -} - -template -KOKKOS_INLINE_FUNCTION void offsetview_verify_operator_bounds( - Kokkos::Impl::SharedAllocationTracker const& tracker, const MapType& map, - const BeginsType& begins, Args... args) { - if (!offsetview_verify_operator_bounds<0>(map, begins, args...)) { - KOKKOS_IF_ON_HOST( - (enum {LEN = 1024}; char buffer[LEN]; - const std::string label = tracker.template get_label(); - int n = snprintf(buffer, LEN, - "OffsetView bounds error of view labeled %s (", - label.c_str()); - offsetview_error_operator_bounds<0>(buffer + n, LEN - n, map, begins, - args...); - Kokkos::Impl::throw_runtime_exception(std::string(buffer));)) - - KOKKOS_IF_ON_DEVICE(( - /* Check #1: is there a SharedAllocationRecord? - (we won't use it, but if it is not there then there isn't - a corresponding SharedAllocationHeader containing a label). - This check should cover the case of Views that don't - have the Unmanaged trait but were initialized by pointer. */ - if (tracker.has_record()) { - Kokkos::Impl::operator_bounds_error_on_device(map); - } else { Kokkos::abort("OffsetView bounds error"); })) - } -} - -inline void runtime_check_rank_host(const size_t rank_dynamic, - const size_t rank, - const index_list_type minIndices, - const std::string& label) { - bool isBad = false; - std::string message = - "Kokkos::Experimental::OffsetView ERROR: for OffsetView labeled '" + - label + "':"; - if (rank_dynamic != rank) { - message += - "The full rank must be the same as the dynamic rank. full rank = "; - message += std::to_string(rank) + - " dynamic rank = " + std::to_string(rank_dynamic) + "\n"; - isBad = true; - } - - size_t numOffsets = 0; - for (size_t i = 0; i < minIndices.size(); ++i) { - if (minIndices.begin()[i] != KOKKOS_INVALID_OFFSET) numOffsets++; - } - if (numOffsets != rank_dynamic) { - message += "The number of offsets provided ( " + - std::to_string(numOffsets) + - " ) must equal the dynamic rank ( " + - std::to_string(rank_dynamic) + " )."; - isBad = true; - } - - if (isBad) Kokkos::abort(message.c_str()); -} - -KOKKOS_INLINE_FUNCTION -void runtime_check_rank_device(const size_t rank_dynamic, const size_t rank, - const index_list_type minIndices) { - if (rank_dynamic != rank) { - Kokkos::abort( - "The full rank of an OffsetView must be the same as the dynamic rank."); - } - size_t numOffsets = 0; - for (size_t i = 0; i < minIndices.size(); ++i) { - if (minIndices.begin()[i] != KOKKOS_INVALID_OFFSET) numOffsets++; - } - if (numOffsets != rank) { - Kokkos::abort( - "The number of offsets provided to an OffsetView constructor must " - "equal the dynamic rank."); - } -} -} // namespace Impl - -template -class OffsetView : public ViewTraits { - public: - using traits = ViewTraits; - - private: - template - friend class OffsetView; - template - friend class View; // FIXME delete this line - template - friend class Kokkos::Impl::ViewMapping; - - using map_type = Kokkos::Impl::ViewMapping; - using track_type = Kokkos::Impl::SharedAllocationTracker; - - public: - enum { Rank = map_type::Rank }; - using begins_type = Kokkos::Array; - - template ::value, iType> = 0> - KOKKOS_INLINE_FUNCTION int64_t begin(const iType local_dimension) const { - return local_dimension < Rank ? m_begins[local_dimension] - : KOKKOS_INVALID_OFFSET; - } - - KOKKOS_INLINE_FUNCTION - begins_type begins() const { return m_begins; } - - template ::value, iType> = 0> - KOKKOS_INLINE_FUNCTION int64_t end(const iType local_dimension) const { - return begin(local_dimension) + m_map.extent(local_dimension); - } - - private: - track_type m_track; - map_type m_map; - begins_type m_begins; - - public: - //---------------------------------------- - /** \brief Compatible view of array of scalar types */ - using array_type = - OffsetView; - - /** \brief Compatible view of const data type */ - using const_type = - OffsetView; - - /** \brief Compatible view of non-const data type */ - using non_const_type = - OffsetView; - - /** \brief Compatible HostMirror view */ - using HostMirror = OffsetView; - - //---------------------------------------- - // Domain rank and extents - - /** \brief rank() to be implemented - */ - // KOKKOS_INLINE_FUNCTION - // static - // constexpr unsigned rank() { return map_type::Rank; } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> - extent(const iType& r) const { - return m_map.extent(r); - } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, int> - extent_int(const iType& r) const { - return static_cast(m_map.extent(r)); - } - - KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() - const { - return m_map.layout(); - } - - KOKKOS_INLINE_FUNCTION constexpr size_t size() const { - return m_map.dimension_0() * m_map.dimension_1() * m_map.dimension_2() * - m_map.dimension_3() * m_map.dimension_4() * m_map.dimension_5() * - m_map.dimension_6() * m_map.dimension_7(); - } - - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { - return m_map.stride_0(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { - return m_map.stride_1(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { - return m_map.stride_2(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { - return m_map.stride_3(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { - return m_map.stride_4(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { - return m_map.stride_5(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { - return m_map.stride_6(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { - return m_map.stride_7(); - } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> - stride(iType r) const { - return ( - r == 0 - ? m_map.stride_0() - : (r == 1 - ? m_map.stride_1() - : (r == 2 - ? m_map.stride_2() - : (r == 3 - ? m_map.stride_3() - : (r == 4 - ? m_map.stride_4() - : (r == 5 - ? m_map.stride_5() - : (r == 6 - ? m_map.stride_6() - : m_map.stride_7()))))))); - } - - template - KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { - m_map.stride(s); - } - - //---------------------------------------- - // Range span is the span which contains all members. - - using reference_type = typename map_type::reference_type; - using pointer_type = typename map_type::pointer_type; - - enum { - reference_type_is_lvalue_reference = - std::is_lvalue_reference::value - }; - - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } - KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { - return m_map.span_is_contiguous(); - } - KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { - return m_map.data() != nullptr; - } - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { - return m_map.data(); - } - - //---------------------------------------- - // Allow specializations to query their specialized map - - KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping& implementation_map() const { - return m_map; - } - - //---------------------------------------- - - private: - static constexpr bool is_layout_left = - std::is_same::value; - - static constexpr bool is_layout_right = - std::is_same::value; - - static constexpr bool is_layout_stride = - std::is_same::value; - - static constexpr bool is_default_map = - std::is_void::value && - (is_layout_left || is_layout_right || is_layout_stride); - -#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) - -#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::OffsetView ERROR: attempt to access inaccessible memory " \ - "space"); \ - Kokkos::Experimental::Impl::offsetview_verify_operator_bounds< \ - typename traits::memory_space> \ - ARG; - -#else - -#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::OffsetView ERROR: attempt to access inaccessible memory " \ - "space"); - -#endif - public: - //------------------------------ - // Rank 0 operator() - - KOKKOS_FORCEINLINE_FUNCTION - reference_type operator()() const { return m_map.reference(); } - //------------------------------ - // Rank 1 operator() - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (1 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.reference(j0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && !is_layout_stride), - reference_type> - operator()(const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[j0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && is_layout_stride), - reference_type> - operator()(const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * j0]; - } - //------------------------------ - // Rank 1 operator[] - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (1 == Rank) && !is_default_map), - reference_type> - operator[](const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.reference(j0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && !is_layout_stride), - reference_type> - operator[](const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[j0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && is_layout_stride), - reference_type> - operator[](const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * j0]; - } - - //------------------------------ - // Rank 2 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (2 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.reference(j0, j1); - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (2 == Rank) && - is_default_map && is_layout_left && (traits::rank_dynamic == 0)), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.m_impl_handle[j0 + m_map.m_impl_offset.m_dim.N0 * j1]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (2 == Rank) && - is_default_map && is_layout_left && (traits::rank_dynamic != 0)), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.m_impl_handle[j0 + m_map.m_impl_offset.m_stride * j1]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (2 == Rank) && - is_default_map && is_layout_right && (traits::rank_dynamic == 0)), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.m_impl_handle[j1 + m_map.m_impl_offset.m_dim.N1 * j0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (2 == Rank) && - is_default_map && is_layout_right && (traits::rank_dynamic != 0)), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.m_impl_handle[j1 + m_map.m_impl_offset.m_stride * j0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (2 == Rank) && is_default_map && is_layout_stride), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.m_impl_handle[j0 * m_map.m_impl_offset.m_stride.S0 + - j1 * m_map.m_impl_offset.m_stride.S1]; - } - - //------------------------------ - // Rank 3 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (3 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (3 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - return m_map.reference(j0, j1, j2); - } - - //------------------------------ - // Rank 4 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (4 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (4 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - return m_map.reference(j0, j1, j2, j3); - } - - //------------------------------ - // Rank 5 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (5 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (5 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - return m_map.reference(j0, j1, j2, j3, j4); - } - - //------------------------------ - // Rank 6 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (6 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (6 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - return m_map.reference(j0, j1, j2, j3, j4, j5); - } - - //------------------------------ - // Rank 7 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (7 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5, j6)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (7 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - return m_map.reference(j0, j1, j2, j3, j4, j5, j6); - } - - //------------------------------ - // Rank 8 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (8 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6, const I7& i7) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6, i7)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - const size_t j7 = i7 - m_begins[7]; - return m_map - .m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5, j6, j7)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (8 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6, const I7& i7) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6, i7)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - const size_t j7 = i7 - m_begins[7]; - return m_map.reference(j0, j1, j2, j3, j4, j5, j6, j7); - } - -#undef KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY - - //---------------------------------------- - // Standard destructor, constructors, and assignment operators - - KOKKOS_DEFAULTED_FUNCTION - ~OffsetView() = default; - - KOKKOS_INLINE_FUNCTION - OffsetView() : m_track(), m_map() { - for (size_t i = 0; i < Rank; ++i) m_begins[i] = KOKKOS_INVALID_OFFSET; - } - - KOKKOS_INLINE_FUNCTION - OffsetView(const OffsetView& rhs) - : m_track(rhs.m_track, traits::is_managed), - m_map(rhs.m_map), - m_begins(rhs.m_begins) {} - - KOKKOS_INLINE_FUNCTION - OffsetView(OffsetView&& rhs) - : m_track(std::move(rhs.m_track)), - m_map(std::move(rhs.m_map)), - m_begins(std::move(rhs.m_begins)) {} - - KOKKOS_INLINE_FUNCTION - OffsetView& operator=(const OffsetView& rhs) { - m_track = rhs.m_track; - m_map = rhs.m_map; - m_begins = rhs.m_begins; - return *this; - } - - KOKKOS_INLINE_FUNCTION - OffsetView& operator=(OffsetView&& rhs) { - m_track = std::move(rhs.m_track); - m_map = std::move(rhs.m_map); - m_begins = std::move(rhs.m_begins); - return *this; - } - - // interoperability with View - private: - using view_type = - View; - - public: - KOKKOS_INLINE_FUNCTION - view_type view() const { - view_type v(m_track, m_map); - return v; - } - - template - KOKKOS_INLINE_FUNCTION OffsetView(const View& aview) - : m_track(aview.impl_track()), m_map() { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, aview.impl_map(), m_track); - - for (int i = 0; i < aview.Rank; ++i) { - m_begins[i] = 0; - } - } - - template - KOKKOS_INLINE_FUNCTION OffsetView(const View& aview, - const index_list_type& minIndices) - : m_track(aview.impl_track()), m_map() { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, aview.impl_map(), m_track); - - KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( - traits::rank_dynamic, Rank, minIndices, label());)) - - KOKKOS_IF_ON_DEVICE((Kokkos::Experimental::Impl::runtime_check_rank_device( - traits::rank_dynamic, Rank, minIndices);)) - - for (size_t i = 0; i < minIndices.size(); ++i) { - m_begins[i] = minIndices.begin()[i]; - } - } - template - KOKKOS_INLINE_FUNCTION OffsetView(const View& aview, - const begins_type& beg) - : m_track(aview.impl_track()), m_map(), m_begins(beg) { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, aview.impl_map(), m_track); - } - - // may assign unmanaged from managed. - - template - KOKKOS_INLINE_FUNCTION OffsetView(const OffsetView& rhs) - : m_track(rhs.m_track, traits::is_managed), - m_map(), - m_begins(rhs.m_begins) { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track); // swb what about assign? - } - - private: - enum class subtraction_failure { - none, - negative, - overflow, - }; - - // Subtraction should return a non-negative number and not overflow - KOKKOS_INLINE_FUNCTION static subtraction_failure check_subtraction( - int64_t lhs, int64_t rhs) { - if (lhs < rhs) return subtraction_failure::negative; - - if (static_cast(-1) / static_cast(2) < - static_cast(lhs) - static_cast(rhs)) - return subtraction_failure::overflow; - - return subtraction_failure::none; - } - - // Need a way to get at an element from both begins_type (aka Kokkos::Array - // which doesn't have iterators) and index_list_type (aka - // std::initializer_list which doesn't have .data() or operator[]). - // Returns by value - KOKKOS_INLINE_FUNCTION - static int64_t at(const begins_type& a, size_t pos) { return a[pos]; } - - KOKKOS_INLINE_FUNCTION - static int64_t at(index_list_type a, size_t pos) { - return *(a.begin() + pos); - } - - // Check that begins < ends for all elements - // B, E can be begins_type and/or index_list_type - template - static subtraction_failure runtime_check_begins_ends_host(const B& begins, - const E& ends) { - std::string message; - if (begins.size() != Rank) - message += - "begins.size() " - "(" + - std::to_string(begins.size()) + - ")" - " != Rank " - "(" + - std::to_string(Rank) + - ")" - "\n"; - - if (ends.size() != Rank) - message += - "ends.size() " - "(" + - std::to_string(begins.size()) + - ")" - " != Rank " - "(" + - std::to_string(Rank) + - ")" - "\n"; - - // If there are no errors so far, then rank == Rank - // Otherwise, check as much as possible - size_t rank = begins.size() < ends.size() ? begins.size() : ends.size(); - for (size_t i = 0; i != rank; ++i) { - subtraction_failure sf = check_subtraction(at(ends, i), at(begins, i)); - if (sf != subtraction_failure::none) { - message += - "(" - "ends[" + - std::to_string(i) + - "]" - " " - "(" + - std::to_string(at(ends, i)) + - ")" - " - " - "begins[" + - std::to_string(i) + - "]" - " " - "(" + - std::to_string(at(begins, i)) + - ")" - ")"; - switch (sf) { - case subtraction_failure::negative: - message += " must be non-negative\n"; - break; - case subtraction_failure::overflow: message += " overflows\n"; break; - default: break; - } - } - } - - if (!message.empty()) { - message = - "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView\n" + - message; - Kokkos::Impl::throw_runtime_exception(message); - } - - return subtraction_failure::none; - } - - // Check the begins < ends for all elements - template - KOKKOS_INLINE_FUNCTION static subtraction_failure - runtime_check_begins_ends_device(const B& begins, const E& ends) { - if (begins.size() != Rank) - Kokkos::abort( - "Kokkos::Experimental::OffsetView ERROR: for unmanaged " - "OffsetView: begins has bad Rank"); - if (ends.size() != Rank) - Kokkos::abort( - "Kokkos::Experimental::OffsetView ERROR: for unmanaged " - "OffsetView: ends has bad Rank"); - - for (size_t i = 0; i != begins.size(); ++i) { - switch (check_subtraction(at(ends, i), at(begins, i))) { - case subtraction_failure::negative: - Kokkos::abort( - "Kokkos::Experimental::OffsetView ERROR: for unmanaged " - "OffsetView: bad range"); - break; - case subtraction_failure::overflow: - Kokkos::abort( - "Kokkos::Experimental::OffsetView ERROR: for unmanaged " - "OffsetView: range overflows"); - break; - default: break; - } - } - - return subtraction_failure::none; - } - - template - KOKKOS_INLINE_FUNCTION static subtraction_failure runtime_check_begins_ends( - const B& begins, const E& ends) { - KOKKOS_IF_ON_HOST((return runtime_check_begins_ends_host(begins, ends);)) - KOKKOS_IF_ON_DEVICE( - (return runtime_check_begins_ends_device(begins, ends);)) - } - - // Constructor around unmanaged data after checking begins < ends for all - // elements - // Each of B, E can be begins_type and/or index_list_type - // Precondition: begins.size() == ends.size() == m_begins.size() == Rank - template - KOKKOS_INLINE_FUNCTION OffsetView(const pointer_type& p, const B& begins_, - const E& ends_, - subtraction_failure) - : m_track() // no tracking - , - m_map(Kokkos::Impl::ViewCtorProp(p), - typename traits::array_layout( - Rank > 0 ? at(ends_, 0) - at(begins_, 0) : 0, - Rank > 1 ? at(ends_, 1) - at(begins_, 1) : 0, - Rank > 2 ? at(ends_, 2) - at(begins_, 2) : 0, - Rank > 3 ? at(ends_, 3) - at(begins_, 3) : 0, - Rank > 4 ? at(ends_, 4) - at(begins_, 4) : 0, - Rank > 5 ? at(ends_, 5) - at(begins_, 5) : 0, - Rank > 6 ? at(ends_, 6) - at(begins_, 6) : 0, - Rank > 7 ? at(ends_, 7) - at(begins_, 7) : 0)) { - for (size_t i = 0; i != m_begins.size(); ++i) { - m_begins[i] = at(begins_, i); - }; - } - - public: - // Constructor around unmanaged data - // Four overloads, as both begins and ends can be either - // begins_type or index_list_type - KOKKOS_INLINE_FUNCTION - OffsetView(const pointer_type& p, const begins_type& begins_, - const begins_type& ends_) - : OffsetView(p, begins_, ends_, - runtime_check_begins_ends(begins_, ends_)) {} - - KOKKOS_INLINE_FUNCTION - OffsetView(const pointer_type& p, const begins_type& begins_, - index_list_type ends_) - : OffsetView(p, begins_, ends_, - runtime_check_begins_ends(begins_, ends_)) {} - - KOKKOS_INLINE_FUNCTION - OffsetView(const pointer_type& p, index_list_type begins_, - const begins_type& ends_) - : OffsetView(p, begins_, ends_, - runtime_check_begins_ends(begins_, ends_)) {} - - KOKKOS_INLINE_FUNCTION - OffsetView(const pointer_type& p, index_list_type begins_, - index_list_type ends_) - : OffsetView(p, begins_, ends_, - runtime_check_begins_ends(begins_, ends_)) {} - - //---------------------------------------- - // Allocation tracking properties - KOKKOS_INLINE_FUNCTION - int use_count() const { return m_track.use_count(); } - - inline const std::string label() const { - return m_track.template get_label(); - } - - // Choosing std::pair as type for the arguments allows constructing an - // OffsetView using list initialization syntax, e.g., - // OffsetView dummy("dummy", {-1, 3}, {-2,2}); - // We could allow arbitrary types RangeType that support - // std::get<{0,1}>(RangeType const&) with std::tuple_size::value==2 - // but this wouldn't allow using the syntax in the example above. - template - explicit inline OffsetView( - const Label& arg_label, - std::enable_if_t::value, - const std::pair> - range0, - const std::pair range1 = KOKKOS_INVALID_INDEX_RANGE, - const std::pair range2 = KOKKOS_INVALID_INDEX_RANGE, - const std::pair range3 = KOKKOS_INVALID_INDEX_RANGE, - const std::pair range4 = KOKKOS_INVALID_INDEX_RANGE, - const std::pair range5 = KOKKOS_INVALID_INDEX_RANGE, - const std::pair range6 = KOKKOS_INVALID_INDEX_RANGE, - const std::pair range7 = KOKKOS_INVALID_INDEX_RANGE - - ) - : OffsetView( - Kokkos::Impl::ViewCtorProp(arg_label), - typename traits::array_layout(range0.second - range0.first + 1, - range1.second - range1.first + 1, - range2.second - range2.first + 1, - range3.second - range3.first + 1, - range4.second - range4.first + 1, - range5.second - range5.first + 1, - range6.second - range6.first + 1, - range7.second - range7.first + 1), - {range0.first, range1.first, range2.first, range3.first, - range4.first, range5.first, range6.first, range7.first}) {} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - template - KOKKOS_DEPRECATED_WITH_COMMENT( - "Use the constructor taking std::pair arguments " - "instead!") - explicit inline OffsetView( - const Label& arg_label, - std::enable_if_t::value, - const index_list_type> - range0, - const index_list_type range1 = KOKKOS_INVALID_INDEX_RANGE, - const index_list_type range2 = KOKKOS_INVALID_INDEX_RANGE, - const index_list_type range3 = KOKKOS_INVALID_INDEX_RANGE, - const index_list_type range4 = KOKKOS_INVALID_INDEX_RANGE, - const index_list_type range5 = KOKKOS_INVALID_INDEX_RANGE, - const index_list_type range6 = KOKKOS_INVALID_INDEX_RANGE, - const index_list_type range7 = KOKKOS_INVALID_INDEX_RANGE) - : OffsetView( - arg_label, - std::pair(range0.begin()[0], range0.begin()[1]), - std::pair(range1.begin()[0], range1.begin()[1]), - std::pair(range2.begin()[0], range2.begin()[1]), - std::pair(range3.begin()[0], range3.begin()[1]), - std::pair(range4.begin()[0], range4.begin()[1]), - std::pair(range5.begin()[0], range5.begin()[1]), - std::pair(range6.begin()[0], range6.begin()[1]), - std::pair(range7.begin()[0], range7.begin()[1])) { - } -#endif - - template - explicit KOKKOS_INLINE_FUNCTION OffsetView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout, - const index_list_type minIndices) - : m_track() // No memory tracking - , - m_map(arg_prop, arg_layout) { - for (size_t i = 0; i < minIndices.size(); ++i) { - m_begins[i] = minIndices.begin()[i]; - } - static_assert( - std::is_same::pointer_type>::value, - "When constructing OffsetView to wrap user memory, you must supply " - "matching pointer type"); - } - - template - explicit inline OffsetView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout, - const index_list_type minIndices) - : m_track(), - m_map() - - { - for (size_t i = 0; i < Rank; ++i) m_begins[i] = minIndices.begin()[i]; - - // Append layout and spaces if not input - using alloc_prop_input = Kokkos::Impl::ViewCtorProp; - - // use 'std::integral_constant' for non-types - // to avoid duplicate class error. - using alloc_prop = Kokkos::Impl::ViewCtorProp< - P..., - std::conditional_t, std::string>, - std::conditional_t, - typename traits::device_type::memory_space>, - std::conditional_t, - typename traits::device_type::execution_space>>; - - static_assert(traits::is_managed, - "OffsetView allocation constructor requires managed memory"); - - if (alloc_prop::initialize && - !alloc_prop::execution_space::impl_is_initialized()) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception( - "Constructing OffsetView and initializing data with uninitialized " - "execution space"); - } - - // Copy the input allocation properties with possibly defaulted properties - alloc_prop prop_copy(arg_prop); - - //------------------------------------------------------------ -#if defined(KOKKOS_ENABLE_CUDA) - // If allocating in CudaUVMSpace must fence before and after - // the allocation to protect against possible concurrent access - // on the CPU and the GPU. - // Fence using the trait's executon space (which will be Kokkos::Cuda) - // to avoid incomplete type errors from usng Kokkos::Cuda directly. - if (std::is_same::value) { - typename traits::device_type::memory_space::execution_space().fence( - "Kokkos::OffsetView::OffsetView(): fence before UVM allocation"); - } -#endif - //------------------------------------------------------------ - - Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( - prop_copy, arg_layout, - Kokkos::Impl::ViewCtorProp::has_execution_space); - - //------------------------------------------------------------ -#if defined(KOKKOS_ENABLE_CUDA) - if (std::is_same::value) { - typename traits::device_type::memory_space::execution_space().fence( - "Kokkos::OffsetView::OffsetView(): fence after UVM allocation"); - } -#endif - //------------------------------------------------------------ - - // Setup and initialization complete, start tracking - m_track.assign_allocated_record_to_uninitialized(record); - - KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( - traits::rank_dynamic, Rank, minIndices, label());)) - - KOKKOS_IF_ON_DEVICE((Kokkos::Experimental::Impl::runtime_check_rank_device( - traits::rank_dynamic, Rank, minIndices);)) - } -}; - -/** \brief Temporary free function rank() - * until rank() is implemented - * in the View - */ -template -KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const OffsetView& V) { - return V.Rank; -} // Temporary until added to view - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -namespace Impl { - -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -shift_input(const T arg, const int64_t offset) { - return arg - offset; -} - -KOKKOS_INLINE_FUNCTION -Kokkos::Impl::ALL_t shift_input(const Kokkos::Impl::ALL_t arg, - const int64_t /*offset*/) { - return arg; -} - -template -KOKKOS_INLINE_FUNCTION - std::enable_if_t::value, Kokkos::pair> - shift_input(const Kokkos::pair arg, const int64_t offset) { - return Kokkos::make_pair(arg.first - offset, arg.second - offset); -} -template -inline std::enable_if_t::value, std::pair> -shift_input(const std::pair arg, const int64_t offset) { - return std::make_pair(arg.first - offset, arg.second - offset); -} - -template -KOKKOS_INLINE_FUNCTION void map_arg_to_new_begin( - const size_t i, Kokkos::Array& subviewBegins, - std::enable_if_t shiftedArg, const Arg arg, - const A viewBegins, size_t& counter) { - if (!std::is_integral::value) { - subviewBegins[counter] = shiftedArg == arg ? viewBegins[i] : 0; - counter++; - } -} - -template -KOKKOS_INLINE_FUNCTION void map_arg_to_new_begin( - const size_t /*i*/, Kokkos::Array& /*subviewBegins*/, - std::enable_if_t /*shiftedArg*/, const Arg /*arg*/, - const A /*viewBegins*/, size_t& /*counter*/) {} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping, T>::type>::type - subview_offset(const OffsetView& src, T arg) { - auto theView = src.view(); - auto begins = src.begins(); - - T shiftedArg = shift_input(arg, begins[0]); - - constexpr size_t rank = - Kokkos::Impl::ViewMapping, T>::type::Rank; - - auto theSubview = Kokkos::subview(theView, shiftedArg); - - Kokkos::Array subviewBegins; - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin(0, subviewBegins, shiftedArg, - arg, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping, T>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1>::type>::type - subview_offset(const Kokkos::Experimental::OffsetView& src, - T0 arg0, T1 arg1) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - - auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1); - constexpr size_t rank = - Kokkos::Impl::ViewMapping, T0, T1>::type::Rank; - - Kokkos::Array subviewBegins; - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1>::type>::type offsetView(theSubview, - subviewBegins); - - return offsetView; -} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2>::type>::type - subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - T2 shiftedArg2 = shift_input(arg2, begins[2]); - - auto theSubview = - Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2); - - constexpr size_t rank = - Kokkos::Impl::ViewMapping, T0, T1, T2>::type::Rank; - - Kokkos::Array subviewBegins; - - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 2, subviewBegins, shiftedArg2, arg2, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3>::type>::type - subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, - T3 arg3) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - T2 shiftedArg2 = shift_input(arg2, begins[2]); - T3 shiftedArg3 = shift_input(arg3, begins[3]); - - auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, - shiftedArg2, shiftedArg3); - - constexpr size_t rank = Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3>::type::Rank; - Kokkos::Array subviewBegins; - - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 2, subviewBegins, shiftedArg2, arg2, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 3, subviewBegins, shiftedArg3, arg3, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4>::type>::type - subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, - T3 arg3, T4 arg4) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - T2 shiftedArg2 = shift_input(arg2, begins[2]); - T3 shiftedArg3 = shift_input(arg3, begins[3]); - T4 shiftedArg4 = shift_input(arg4, begins[4]); - - auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, - shiftedArg2, shiftedArg3, shiftedArg4); - - constexpr size_t rank = Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4>::type::Rank; - Kokkos::Array subviewBegins; - - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 2, subviewBegins, shiftedArg2, arg2, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 3, subviewBegins, shiftedArg3, arg3, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 4, subviewBegins, shiftedArg4, arg4, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5>::type>::type - subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, - T3 arg3, T4 arg4, T5 arg5) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - T2 shiftedArg2 = shift_input(arg2, begins[2]); - T3 shiftedArg3 = shift_input(arg3, begins[3]); - T4 shiftedArg4 = shift_input(arg4, begins[4]); - T5 shiftedArg5 = shift_input(arg5, begins[5]); - - auto theSubview = - Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, - shiftedArg3, shiftedArg4, shiftedArg5); - - constexpr size_t rank = Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5>::type::Rank; - - Kokkos::Array subviewBegins; - - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 2, subviewBegins, shiftedArg2, arg2, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 3, subviewBegins, shiftedArg3, arg3, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 4, subviewBegins, shiftedArg4, arg4, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 5, subviewBegins, shiftedArg5, arg5, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5, T6>::type>::type - subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, - T3 arg3, T4 arg4, T5 arg5, T6 arg6) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - T2 shiftedArg2 = shift_input(arg2, begins[2]); - T3 shiftedArg3 = shift_input(arg3, begins[3]); - T4 shiftedArg4 = shift_input(arg4, begins[4]); - T5 shiftedArg5 = shift_input(arg5, begins[5]); - T6 shiftedArg6 = shift_input(arg6, begins[6]); - - auto theSubview = - Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, - shiftedArg3, shiftedArg4, shiftedArg5, shiftedArg6); - - constexpr size_t rank = Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5, T6>::type::Rank; - - Kokkos::Array subviewBegins; - - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 2, subviewBegins, shiftedArg2, arg2, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 3, subviewBegins, shiftedArg3, arg3, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 4, subviewBegins, shiftedArg4, arg4, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 5, subviewBegins, shiftedArg5, arg5, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 6, subviewBegins, shiftedArg6, arg6, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5, T6>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5, T6, T7>::type>::type - subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, - T3 arg3, T4 arg4, T5 arg5, T6 arg6, T7 arg7) { - auto theView = src.view(); - auto begins = src.begins(); - - T0 shiftedArg0 = shift_input(arg0, begins[0]); - T1 shiftedArg1 = shift_input(arg1, begins[1]); - T2 shiftedArg2 = shift_input(arg2, begins[2]); - T3 shiftedArg3 = shift_input(arg3, begins[3]); - T4 shiftedArg4 = shift_input(arg4, begins[4]); - T5 shiftedArg5 = shift_input(arg5, begins[5]); - T6 shiftedArg6 = shift_input(arg6, begins[6]); - T7 shiftedArg7 = shift_input(arg7, begins[7]); - - auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, - shiftedArg2, shiftedArg3, shiftedArg4, - shiftedArg5, shiftedArg6, shiftedArg7); - - constexpr size_t rank = Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5, T6, T7>::type::Rank; - - Kokkos::Array subviewBegins; - - size_t counter = 0; - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 0, subviewBegins, shiftedArg0, arg0, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 1, subviewBegins, shiftedArg1, arg1, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 2, subviewBegins, shiftedArg2, arg2, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 3, subviewBegins, shiftedArg3, arg3, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 4, subviewBegins, shiftedArg4, arg4, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 5, subviewBegins, shiftedArg5, arg5, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 6, subviewBegins, shiftedArg6, arg6, begins, counter); - Kokkos::Experimental::Impl::map_arg_to_new_begin( - 7, subviewBegins, shiftedArg7, arg7, begins, counter); - - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, T0, T1, T2, T3, T4, T5, T6, T7>::type>::type - offsetView(theSubview, subviewBegins); - - return offsetView; -} -} // namespace Impl - -template -KOKKOS_INLINE_FUNCTION - typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< - typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - ViewTraits, Args...>::type>::type - subview(const OffsetView& src, Args... args) { - static_assert( - OffsetView::Rank == sizeof...(Args), - "subview requires one argument for each source OffsetView rank"); - - return Kokkos::Experimental::Impl::subview_offset(src, args...); -} - -} // namespace Experimental -} // namespace Kokkos -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Experimental { -template -KOKKOS_INLINE_FUNCTION bool operator==(const OffsetView& lhs, - const OffsetView& rhs) { - // Same data, layout, dimensions - using lhs_traits = ViewTraits; - using rhs_traits = ViewTraits; - - return std::is_same::value && - std::is_same::value && - std::is_same::value && - unsigned(lhs_traits::rank) == unsigned(rhs_traits::rank) && - lhs.data() == rhs.data() && lhs.span() == rhs.span() && - lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && - lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && - lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && - lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7) && - lhs.begin(0) == rhs.begin(0) && lhs.begin(1) == rhs.begin(1) && - lhs.begin(2) == rhs.begin(2) && lhs.begin(3) == rhs.begin(3) && - lhs.begin(4) == rhs.begin(4) && lhs.begin(5) == rhs.begin(5) && - lhs.begin(6) == rhs.begin(6) && lhs.begin(7) == rhs.begin(7); -} - -template -KOKKOS_INLINE_FUNCTION bool operator!=(const OffsetView& lhs, - const OffsetView& rhs) { - return !(operator==(lhs, rhs)); -} - -template -KOKKOS_INLINE_FUNCTION bool operator==(const View& lhs, - const OffsetView& rhs) { - // Same data, layout, dimensions - using lhs_traits = ViewTraits; - using rhs_traits = ViewTraits; - - return std::is_same::value && - std::is_same::value && - std::is_same::value && - unsigned(lhs_traits::rank) == unsigned(rhs_traits::rank) && - lhs.data() == rhs.data() && lhs.span() == rhs.span() && - lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && - lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && - lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && - lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7); -} - -template -KOKKOS_INLINE_FUNCTION bool operator==(const OffsetView& lhs, - const View& rhs) { - return rhs == lhs; -} - -} // namespace Experimental -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -template -inline void deep_copy( - const Experimental::OffsetView& dst, - typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - static_assert( - std::is_same::non_const_value_type, - typename ViewTraits::value_type>::value, - "deep_copy requires non-const type"); - - auto dstView = dst.view(); - Kokkos::deep_copy(dstView, value); -} - -template -inline void deep_copy( - const Experimental::OffsetView& dst, - const Experimental::OffsetView& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - static_assert( - std::is_same::value_type, - typename ViewTraits::non_const_value_type>::value, - "deep_copy requires matching non-const destination type"); - - auto dstView = dst.view(); - Kokkos::deep_copy(dstView, value.view()); -} -template -inline void deep_copy( - const Experimental::OffsetView& dst, - const View& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - static_assert( - std::is_same::value_type, - typename ViewTraits::non_const_value_type>::value, - "deep_copy requires matching non-const destination type"); - - auto dstView = dst.view(); - Kokkos::deep_copy(dstView, value); -} - -template -inline void deep_copy( - const View& dst, - const Experimental::OffsetView& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { - static_assert( - std::is_same::value_type, - typename ViewTraits::non_const_value_type>::value, - "deep_copy requires matching non-const destination type"); - - Kokkos::deep_copy(dst, value.view()); -} - -namespace Impl { - -// Deduce Mirror Types -template -struct MirrorOffsetViewType { - // The incoming view_type - using src_view_type = typename Kokkos::Experimental::OffsetView; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it.) - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using dest_view_type = - Kokkos::Experimental::OffsetView; - // If it is the same memory_space return the existing view_type - // This will also keep the unmanaged trait if necessary - using view_type = - std::conditional_t; -}; - -template -struct MirrorOffsetType { - // The incoming view_type - using src_view_type = typename Kokkos::Experimental::OffsetView; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it.) - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using view_type = - Kokkos::Experimental::OffsetView; -}; - -} // namespace Impl - -namespace Impl { -template -inline typename Kokkos::Experimental::OffsetView::HostMirror -create_mirror(const Kokkos::Experimental::OffsetView& src, - const Impl::ViewCtorProp& arg_prop) { - return typename Kokkos::Experimental::OffsetView::HostMirror( - Kokkos::create_mirror(arg_prop, src.view()), src.begins()); -} - -template -inline typename Kokkos::Impl::MirrorOffsetType::view_type -create_mirror(const Space&, - const Kokkos::Experimental::OffsetView& src, - const Impl::ViewCtorProp& arg_prop) { - using alloc_prop_input = Impl::ViewCtorProp; - - static_assert( - !alloc_prop_input::has_label, - "The view constructor arguments passed to Kokkos::create_mirror " - "must not include a label!"); - static_assert( - !alloc_prop_input::has_pointer, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not include a pointer!"); - static_assert( - !alloc_prop_input::has_memory_space, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not include a memory space instance!"); - static_assert( - !alloc_prop_input::allow_padding, - "The view constructor arguments passed to Kokkos::create_mirror must " - "not explicitly allow padding!"); - - using alloc_prop = Impl::ViewCtorProp; - alloc_prop prop_copy(arg_prop); - static_cast&>(prop_copy).value = - std::string(src.label()).append("_mirror"); - - return typename Kokkos::Impl::MirrorOffsetType::view_type( - prop_copy, src.layout(), - {src.begin(0), src.begin(1), src.begin(2), src.begin(3), src.begin(4), - src.begin(5), src.begin(6), src.begin(7)}); -} -} // namespace Impl - -// Create a mirror in host space -template -inline auto create_mirror( - const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror(src, Impl::ViewCtorProp<>{}); -} - -template -inline auto create_mirror( - Kokkos::Impl::WithoutInitializing_t wi, - const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror(src, Kokkos::view_alloc(wi)); -} - -// Create a mirror in a new space -template ::value>> -inline auto create_mirror( - const Space& space, const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror(space, src, Impl::ViewCtorProp<>{}); -} - -template -typename Kokkos::Impl::MirrorOffsetType::view_type -create_mirror(Kokkos::Impl::WithoutInitializing_t wi, const Space& space, - const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror(space, src, Kokkos::view_alloc(wi)); -} - -template -inline auto create_mirror( - const Impl::ViewCtorProp& arg_prop, - const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror(src, arg_prop); -} - -namespace Impl { -template -inline std::enable_if_t< - (std::is_same< - typename Kokkos::Experimental::OffsetView::memory_space, - typename Kokkos::Experimental::OffsetView< - T, P...>::HostMirror::memory_space>::value && - std::is_same::data_type, - typename Kokkos::Experimental::OffsetView< - T, P...>::HostMirror::data_type>::value), - typename Kokkos::Experimental::OffsetView::HostMirror> -create_mirror_view( - const typename Kokkos::Experimental::OffsetView& src, - const Impl::ViewCtorProp&) { - return src; -} - -template -inline std::enable_if_t< - !(std::is_same< - typename Kokkos::Experimental::OffsetView::memory_space, - typename Kokkos::Experimental::OffsetView< - T, P...>::HostMirror::memory_space>::value && - std::is_same< - typename Kokkos::Experimental::OffsetView::data_type, - typename Kokkos::Experimental::OffsetView< - T, P...>::HostMirror::data_type>::value), - typename Kokkos::Experimental::OffsetView::HostMirror> -create_mirror_view(const Kokkos::Experimental::OffsetView& src, - const Impl::ViewCtorProp& arg_prop) { - return Kokkos::create_mirror(arg_prop, src); -} - -template -inline std::enable_if_t< - Impl::MirrorOffsetViewType::is_same_memspace, - Kokkos::Experimental::OffsetView> -create_mirror_view(const Space&, - const Kokkos::Experimental::OffsetView& src, - const Impl::ViewCtorProp&) { - return src; -} - -template -std::enable_if_t< - !Impl::MirrorOffsetViewType::is_same_memspace, - typename Kokkos::Impl::MirrorOffsetViewType::view_type> -create_mirror_view(const Space& space, - const Kokkos::Experimental::OffsetView& src, - const Impl::ViewCtorProp& arg_prop) { - return create_mirror(space, src, arg_prop); -} -} // namespace Impl - -// Create a mirror view in host space -template -inline auto create_mirror_view( - const typename Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror_view(src, Impl::ViewCtorProp<>{}); -} - -template -inline auto create_mirror_view( - Kokkos::Impl::WithoutInitializing_t wi, - const typename Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror_view(src, Kokkos::view_alloc(wi)); -} - -// Create a mirror view in a new space -template ::value>> -inline auto create_mirror_view( - const Space& space, const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror_view(space, src, Impl::ViewCtorProp<>{}); -} - -template -inline auto create_mirror_view( - Kokkos::Impl::WithoutInitializing_t wi, const Space& space, - const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror_view(space, src, Kokkos::view_alloc(wi)); -} - -template -inline auto create_mirror_view( - const Impl::ViewCtorProp& arg_prop, - const Kokkos::Experimental::OffsetView& src) { - return Impl::create_mirror_view(src, arg_prop); -} - -// Create a mirror view and deep_copy in a new space -template -typename Kokkos::Impl::MirrorOffsetViewType< - typename Impl::ViewCtorProp::memory_space, T, - P...>::view_type -create_mirror_view_and_copy( - const Impl::ViewCtorProp& arg_prop, - const Kokkos::Experimental::OffsetView& src) { - return {create_mirror_view_and_copy(arg_prop, src.view()), src.begins()}; -} - -template -typename Kokkos::Impl::MirrorOffsetViewType::view_type -create_mirror_view_and_copy( - const Space& space, const Kokkos::Experimental::OffsetView& src, - std::string const& name = "") { - return {create_mirror_view_and_copy(space, src.view(), name), src.begins()}; -} -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_OFFSETVIEW -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_OFFSETVIEW -#endif -#endif /* KOKKOS_OFFSETVIEW_HPP_ */ diff --git a/bundled/kokkos-3.7.00/containers/src/impl/Kokkos_Bitset_impl.hpp b/bundled/kokkos-3.7.00/containers/src/impl/Kokkos_Bitset_impl.hpp deleted file mode 100644 index 134b30769f..0000000000 --- a/bundled/kokkos-3.7.00/containers/src/impl/Kokkos_Bitset_impl.hpp +++ /dev/null @@ -1,100 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_BITSET_IMPL_HPP -#define KOKKOS_BITSET_IMPL_HPP - -#include -#include -#include - -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -KOKKOS_FORCEINLINE_FUNCTION -unsigned rotate_right(unsigned i, int r) { - constexpr int size = static_cast(sizeof(unsigned) * CHAR_BIT); - return r ? ((i >> r) | (i << (size - r))) : i; -} - -template -struct BitsetCount { - using bitset_type = Bitset; - using execution_space = - typename bitset_type::execution_space::execution_space; - using size_type = typename bitset_type::size_type; - using value_type = size_type; - - bitset_type m_bitset; - - BitsetCount(bitset_type const& bitset) : m_bitset(bitset) {} - - size_type apply() const { - size_type count = 0u; - parallel_reduce("Kokkos::Impl::BitsetCount::apply", - m_bitset.m_blocks.extent(0), *this, count); - return count; - } - - KOKKOS_INLINE_FUNCTION - void init(value_type& count) const { count = 0u; } - - KOKKOS_INLINE_FUNCTION - void join(value_type& count, const size_type& incr) const { count += incr; } - - KOKKOS_INLINE_FUNCTION - void operator()(size_type i, value_type& count) const { - count += bit_count(m_bitset.m_blocks[i]); - } -}; - -} // namespace Impl -} // namespace Kokkos - -#endif // KOKKOS_BITSET_IMPL_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_CudaSpace.cpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_CudaSpace.cpp deleted file mode 100644 index b2161bc1fa..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_CudaSpace.cpp +++ /dev/null @@ -1,716 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#ifdef KOKKOS_ENABLE_CUDA - -#include -#include -#include - -#include -#include -#include -#include -#include - -//#include -#include -#include - -#include - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -cudaStream_t Kokkos::Impl::cuda_get_deep_copy_stream() { - static cudaStream_t s = nullptr; - if (s == nullptr) { - cudaStreamCreate(&s); - } - return s; -} - -const std::unique_ptr &Kokkos::Impl::cuda_get_deep_copy_space( - bool initialize) { - static std::unique_ptr space = nullptr; - if (!space && initialize) - space = std::make_unique(Kokkos::Impl::cuda_get_deep_copy_stream()); - return space; -} - -namespace Kokkos { -namespace Impl { - -namespace { - -static std::atomic num_uvm_allocations(0); - -} // namespace - -void DeepCopyCuda(void *dst, const void *src, size_t n) { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaMemcpy(dst, src, n, cudaMemcpyDefault)); -} - -void DeepCopyAsyncCuda(const Cuda &instance, void *dst, const void *src, - size_t n) { - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMemcpyAsync(dst, src, n, cudaMemcpyDefault, instance.cuda_stream())); -} - -void DeepCopyAsyncCuda(void *dst, const void *src, size_t n) { - cudaStream_t s = cuda_get_deep_copy_stream(); - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMemcpyAsync(dst, src, n, cudaMemcpyDefault, s)); - Impl::cuda_stream_synchronize( - s, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - DeepCopyResourceSynchronization, - "Kokkos::Impl::DeepCopyAsyncCuda: Deep Copy Stream Sync"); -} - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -KOKKOS_DEPRECATED void CudaSpace::access_error() { - const std::string msg( - "Kokkos::CudaSpace::access_error attempt to execute Cuda function from " - "non-Cuda space"); - Kokkos::Impl::throw_runtime_exception(msg); -} - -KOKKOS_DEPRECATED void CudaSpace::access_error(const void *const) { - const std::string msg( - "Kokkos::CudaSpace::access_error attempt to execute Cuda function from " - "non-Cuda space"); - Kokkos::Impl::throw_runtime_exception(msg); -} -#endif - -/*--------------------------------------------------------------------------*/ - -bool CudaUVMSpace::available() { -#if defined(CUDA_VERSION) && !defined(__APPLE__) - enum : bool { UVM_available = true }; -#else - enum : bool { UVM_available = false }; -#endif - return UVM_available; -} - -/*--------------------------------------------------------------------------*/ - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -int CudaUVMSpace::number_of_allocations() { - return Kokkos::Impl::num_uvm_allocations.load(); -} -#endif -#ifdef KOKKOS_IMPL_DEBUG_CUDA_PIN_UVM_TO_HOST -// The purpose of the following variable is to allow a state-based choice -// for pinning UVM allocations to the CPU. For now this is considered -// an experimental debugging capability - with the potential to work around -// some CUDA issues. -bool CudaUVMSpace::kokkos_impl_cuda_pin_uvm_to_host_v = false; - -bool CudaUVMSpace::cuda_pin_uvm_to_host() { - return CudaUVMSpace::kokkos_impl_cuda_pin_uvm_to_host_v; -} -void CudaUVMSpace::cuda_set_pin_uvm_to_host(bool val) { - CudaUVMSpace::kokkos_impl_cuda_pin_uvm_to_host_v = val; -} -#endif -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_DEBUG_CUDA_PIN_UVM_TO_HOST -bool kokkos_impl_cuda_pin_uvm_to_host() { - return Kokkos::CudaUVMSpace::cuda_pin_uvm_to_host(); -} - -void kokkos_impl_cuda_set_pin_uvm_to_host(bool val) { - Kokkos::CudaUVMSpace::cuda_set_pin_uvm_to_host(val); -} -#endif - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { - -CudaSpace::CudaSpace() : m_device(Kokkos::Cuda().cuda_device()) {} - -CudaUVMSpace::CudaUVMSpace() : m_device(Kokkos::Cuda().cuda_device()) {} - -CudaHostPinnedSpace::CudaHostPinnedSpace() {} - -int memory_threshold_g = 40000; // 40 kB - -//============================================================================== -// {{{1 - -void *CudaSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} - -void *CudaSpace::allocate(const Cuda &exec_space, const char *arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(exec_space, arg_label, arg_alloc_size, arg_logical_size); -} -void *CudaSpace::allocate(const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} - -namespace { -void *impl_allocate_common(const Cuda &exec_space, const char *arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle, - bool exec_space_provided) { - void *ptr = nullptr; - -#ifndef CUDART_VERSION -#error CUDART_VERSION undefined! -#elif (defined(KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC) && CUDART_VERSION >= 11020) - cudaError_t error_code; - if (arg_alloc_size >= memory_threshold_g) { - if (exec_space_provided) { - cudaStream_t stream = exec_space.cuda_stream(); - error_code = cudaMallocAsync(&ptr, arg_alloc_size, stream); - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaStreamSynchronize(stream)); - } else { - error_code = cudaMallocAsync(&ptr, arg_alloc_size, 0); - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaDeviceSynchronize()); - } - } else { - error_code = cudaMalloc(&ptr, arg_alloc_size); - } -#else - (void)exec_space; - (void)exec_space_provided; - auto error_code = cudaMalloc(&ptr, arg_alloc_size); -#endif - if (error_code != cudaSuccess) { // TODO tag as unlikely branch - cudaGetLastError(); // This is the only way to clear the last error, which - // we should do here since we're turning it into an - // exception here - throw Experimental::CudaRawMemoryAllocationFailure( - arg_alloc_size, error_code, - Experimental::RawMemoryAllocationFailure::AllocationMechanism:: - CudaMalloc); - } - - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - return ptr; -} -} // namespace - -void *CudaSpace::impl_allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - return impl_allocate_common(Kokkos::Cuda{}, arg_label, arg_alloc_size, - arg_logical_size, arg_handle, false); -} - -void *CudaSpace::impl_allocate( - const Cuda &exec_space, const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - return impl_allocate_common(exec_space, arg_label, arg_alloc_size, - arg_logical_size, arg_handle, true); -} - -void *CudaUVMSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void *CudaUVMSpace::allocate(const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void *CudaUVMSpace::impl_allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - void *ptr = nullptr; - - Cuda::impl_static_fence( - "Kokkos::CudaUVMSpace::impl_allocate: Pre UVM Allocation"); - if (arg_alloc_size > 0) { - Kokkos::Impl::num_uvm_allocations++; - - auto error_code = - cudaMallocManaged(&ptr, arg_alloc_size, cudaMemAttachGlobal); - -#ifdef KOKKOS_IMPL_DEBUG_CUDA_PIN_UVM_TO_HOST - if (Kokkos::CudaUVMSpace::cuda_pin_uvm_to_host()) - cudaMemAdvise(ptr, arg_alloc_size, cudaMemAdviseSetPreferredLocation, - cudaCpuDeviceId); -#endif - - if (error_code != cudaSuccess) { // TODO tag as unlikely branch - cudaGetLastError(); // This is the only way to clear the last error, - // which we should do here since we're turning it - // into an exception here - throw Experimental::CudaRawMemoryAllocationFailure( - arg_alloc_size, error_code, - Experimental::RawMemoryAllocationFailure::AllocationMechanism:: - CudaMallocManaged); - } - } - Cuda::impl_static_fence( - "Kokkos::CudaUVMSpace::impl_allocate: Post UVM Allocation"); - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - return ptr; -} -void *CudaHostPinnedSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void *CudaHostPinnedSpace::allocate(const char *arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void *CudaHostPinnedSpace::impl_allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - void *ptr = nullptr; - - auto error_code = cudaHostAlloc(&ptr, arg_alloc_size, cudaHostAllocDefault); - if (error_code != cudaSuccess) { // TODO tag as unlikely branch - cudaGetLastError(); // This is the only way to clear the last error, which - // we should do here since we're turning it into an - // exception here - throw Experimental::CudaRawMemoryAllocationFailure( - arg_alloc_size, error_code, - Experimental::RawMemoryAllocationFailure::AllocationMechanism:: - CudaHostAlloc); - } - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - return ptr; -} - -// end allocate() }}}1 -//============================================================================== -void CudaSpace::deallocate(void *const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} -void CudaSpace::deallocate(const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void CudaSpace::impl_deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - try { -#ifndef CUDART_VERSION -#error CUDART_VERSION undefined! -#elif (defined(KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC) && CUDART_VERSION >= 11020) - if (arg_alloc_size >= memory_threshold_g) { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaDeviceSynchronize()); - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFreeAsync(arg_alloc_ptr, 0)); - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaDeviceSynchronize()); - } else { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFree(arg_alloc_ptr)); - } -#else - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFree(arg_alloc_ptr)); -#endif - } catch (...) { - } -} -void CudaUVMSpace::deallocate(void *const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void CudaUVMSpace::deallocate(const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size - - , - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void CudaUVMSpace::impl_deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size - - , - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - Cuda::impl_static_fence( - "Kokkos::CudaUVMSpace::impl_deallocate: Pre UVM Deallocation"); - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - try { - if (arg_alloc_ptr != nullptr) { - Kokkos::Impl::num_uvm_allocations--; - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFree(arg_alloc_ptr)); - } - } catch (...) { - } - Cuda::impl_static_fence( - "Kokkos::CudaUVMSpace::impl_deallocate: Post UVM Deallocation"); -} - -void CudaHostPinnedSpace::deallocate(void *const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} -void CudaHostPinnedSpace::deallocate(const char *arg_label, - void *const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} - -void CudaHostPinnedSpace::impl_deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - try { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFreeHost(arg_alloc_ptr)); - } catch (...) { - } -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -#ifdef KOKKOS_ENABLE_DEBUG -SharedAllocationRecord - SharedAllocationRecord::s_root_record; - -SharedAllocationRecord - SharedAllocationRecord::s_root_record; - -SharedAllocationRecord - SharedAllocationRecord::s_root_record; -#endif - -::cudaTextureObject_t -SharedAllocationRecord::attach_texture_object( - const unsigned sizeof_alias, void *const alloc_ptr, - size_t const alloc_size) { - enum { TEXTURE_BOUND_1D = 1u << 27 }; - - if ((alloc_ptr == nullptr) || - (sizeof_alias * TEXTURE_BOUND_1D <= alloc_size)) { - std::ostringstream msg; - msg << "Kokkos::CudaSpace ERROR: Cannot attach texture object to" - << " alloc_ptr(" << alloc_ptr << ")" - << " alloc_size(" << alloc_size << ")" - << " max_size(" << (sizeof_alias * TEXTURE_BOUND_1D) << ")"; - std::cerr << msg.str() << std::endl; - std::cerr.flush(); - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - - ::cudaTextureObject_t tex_obj; - - struct cudaResourceDesc resDesc; - struct cudaTextureDesc texDesc; - - memset(&resDesc, 0, sizeof(resDesc)); - memset(&texDesc, 0, sizeof(texDesc)); - - resDesc.resType = cudaResourceTypeLinear; - resDesc.res.linear.desc = - (sizeof_alias == 4 - ? cudaCreateChannelDesc() - : (sizeof_alias == 8 - ? cudaCreateChannelDesc< ::int2>() - : - /* sizeof_alias == 16 */ cudaCreateChannelDesc< ::int4>())); - resDesc.res.linear.sizeInBytes = alloc_size; - resDesc.res.linear.devPtr = alloc_ptr; - - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaCreateTextureObject(&tex_obj, &resDesc, &texDesc, nullptr)); - - return tex_obj; -} - -//============================================================================== -// {{{1 - -SharedAllocationRecord::~SharedAllocationRecord() { - auto alloc_size = SharedAllocationRecord::m_alloc_size; - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - alloc_size, (alloc_size - sizeof(SharedAllocationHeader))); -} - -void SharedAllocationRecord::deep_copy_header_no_exec( - void *ptr, const void *header) { - Kokkos::Cuda exec; - Kokkos::Impl::DeepCopy(exec, ptr, header, - sizeof(SharedAllocationHeader)); - exec.fence( - "SharedAllocationRecord::SharedAllocationRecord(): fence after copying header from " - "HostSpace"); -} - -SharedAllocationRecord::~SharedAllocationRecord() { - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size, - (SharedAllocationRecord::m_alloc_size - - sizeof(SharedAllocationHeader))); -} - -SharedAllocationRecord::~SharedAllocationRecord() { - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size, - (SharedAllocationRecord::m_alloc_size - - sizeof(SharedAllocationHeader))); -} - -// end SharedAllocationRecord destructors }}}1 -//============================================================================== - -//============================================================================== -// {{{1 - -SharedAllocationRecord::SharedAllocationRecord( - const Kokkos::CudaSpace &arg_space, const std::string &arg_label, - const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_tex_obj(0), - m_space(arg_space) { - - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, arg_label); - - // Copy to device memory - Kokkos::Cuda exec; - Kokkos::Impl::DeepCopy( - exec, RecordBase::m_alloc_ptr, &header, sizeof(SharedAllocationHeader)); - exec.fence( - "SharedAllocationRecord::SharedAllocationRecord(): fence after copying header from " - "HostSpace"); -} - -SharedAllocationRecord::SharedAllocationRecord( - const Kokkos::Cuda &arg_exec_space, const Kokkos::CudaSpace &arg_space, - const std::string &arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_exec_space, arg_space, - arg_label, arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_tex_obj(0), - m_space(arg_space) { - - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, arg_label); - - // Copy to device memory - Kokkos::Impl::DeepCopy(arg_exec_space, - RecordBase::m_alloc_ptr, &header, - sizeof(SharedAllocationHeader)); -} - -SharedAllocationRecord::SharedAllocationRecord( - const Kokkos::CudaUVMSpace &arg_space, const std::string &arg_label, - const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_tex_obj(0), - m_space(arg_space) { - this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, - arg_label); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::CudaHostPinnedSpace &arg_space, - const std::string &arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, - arg_label); -} - -// end SharedAllocationRecord constructors }}}1 -//============================================================================== - -void cuda_prefetch_pointer(const Cuda &space, const void *ptr, size_t bytes, - bool to_device) { - if ((ptr == nullptr) || (bytes == 0)) return; - cudaPointerAttributes attr; - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaPointerGetAttributes(&attr, ptr)); - // I measured this and it turns out prefetching towards the host slows - // DualView syncs down. Probably because the latency is not too bad in the - // first place for the pull down. If we want to change that provde - // cudaCpuDeviceId as the device if to_device is false -#if CUDA_VERSION < 10000 - bool is_managed = attr.isManaged; -#else - bool is_managed = attr.type == cudaMemoryTypeManaged; -#endif - if (to_device && is_managed && - space.cuda_device_prop().concurrentManagedAccess) { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaMemPrefetchAsync( - ptr, bytes, space.cuda_device(), space.cuda_stream())); - } -} - -} // namespace Impl -} // namespace Kokkos - -//============================================================================== -// {{{1 - -#include - -namespace Kokkos { -namespace Impl { - -// To avoid additional compilation cost for something that's (mostly?) not -// performance sensitive, we explicity instantiate these CRTP base classes here, -// where we have access to the associated *_timpl.hpp header files. -template class SharedAllocationRecordCommon; -template class HostInaccessibleSharedAllocationRecordCommon; -template class SharedAllocationRecordCommon; -template class SharedAllocationRecordCommon; - -} // end namespace Impl -} // end namespace Kokkos - -// end Explicit instantiations of CRTP Base classes }}}1 -//============================================================================== - -#else -void KOKKOS_CORE_SRC_CUDA_CUDASPACE_PREVENT_LINK_ERROR() {} -#endif // KOKKOS_ENABLE_CUDA diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Alloc.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Alloc.hpp deleted file mode 100644 index e76133fae8..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Alloc.hpp +++ /dev/null @@ -1,153 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_ALLOCATION_TRACKING_HPP -#define KOKKOS_CUDA_ALLOCATION_TRACKING_HPP - -#include -#ifdef KOKKOS_ENABLE_CUDA - -#include - -namespace Kokkos { -namespace Impl { - -template -SharedAllocationRecord* shared_allocation_record( - Kokkos::CudaSpace const& arg_space, void* const arg_alloc_ptr, - DestructFunctor const& arg_destruct) { - SharedAllocationRecord* const record = - SharedAllocationRecord::get_record(arg_alloc_ptr); - - // assert: record != 0 - - // assert: sizeof(DestructFunctor) <= record->m_destruct_size - - // assert: record->m_destruct_function == 0 - - DestructFunctor* const functor = reinterpret_cast( - reinterpret_cast(record) + sizeof(SharedAllocationRecord)); - - new (functor) DestructFunctor(arg_destruct); - - record->m_destruct_functor = &shared_allocation_destroy; - - return record; -} - -/// class CudaUnmanagedAllocator -/// does nothing when deallocate(ptr,size) is called -struct CudaUnmanagedAllocator { - static const char* name() { return "Cuda Unmanaged Allocator"; } - - static void deallocate(void* /*ptr*/, size_t /*size*/) {} - - static bool support_texture_binding() { return true; } -}; - -/// class CudaUnmanagedAllocator -/// does nothing when deallocate(ptr,size) is called -struct CudaUnmanagedUVMAllocator { - static const char* name() { return "Cuda Unmanaged UVM Allocator"; } - - static void deallocate(void* /*ptr*/, size_t /*size*/) {} - - static bool support_texture_binding() { return true; } -}; - -/// class CudaUnmanagedHostAllocator -/// does nothing when deallocate(ptr,size) is called -class CudaUnmanagedHostAllocator { - public: - static const char* name() { return "Cuda Unmanaged Host Allocator"; } - // Unmanaged deallocate does nothing - static void deallocate(void* /*ptr*/, size_t /*size*/) {} -}; - -/// class CudaMallocAllocator -class CudaMallocAllocator { - public: - static const char* name() { return "Cuda Malloc Allocator"; } - - static void* allocate(size_t size); - - static void deallocate(void* ptr, size_t); - - static void* reallocate(void* old_ptr, size_t old_size, size_t new_size); - - static bool support_texture_binding() { return true; } -}; - -/// class CudaUVMAllocator -class CudaUVMAllocator { - public: - static const char* name() { return "Cuda UVM Allocator"; } - - static void* allocate(size_t size); - - static void deallocate(void* ptr, size_t); - - static void* reallocate(void* old_ptr, size_t old_size, size_t new_size); - - static bool support_texture_binding() { return true; } -}; - -/// class CudaHostAllocator -class CudaHostAllocator { - public: - static const char* name() { return "Cuda Host Allocator"; } - - static void* allocate(size_t size); - - static void deallocate(void* ptr, size_t); - - static void* reallocate(void* old_ptr, size_t old_size, size_t new_size); -}; - -} // namespace Impl -} // namespace Kokkos - -#endif // KOKKOS_ENABLE_CUDA - -#endif // #ifndef KOKKOS_CUDA_ALLOCATION_TRACKING_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Atomic_Intrinsics.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Atomic_Intrinsics.hpp deleted file mode 100644 index 8e8dff6776..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Atomic_Intrinsics.hpp +++ /dev/null @@ -1,1003 +0,0 @@ -/* -@HEADER -================================================================================ - -ORIGINAL LICENSE ----------------- - -Copyright (c) 2018, NVIDIA Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -================================================================================ - -LICENSE ASSOCIATED WITH SUBSEQUENT MODIFICATIONS ------------------------------------------------- - -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -@HEADER -*/ - -#include -#if defined(__CUDA_ARCH__) && defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - -#include - -#ifndef _SIMT_DETAILS_CONFIG -#define _SIMT_DETAILS_CONFIG - -namespace Kokkos { -namespace Impl { - -#ifndef __simt_scope -// Modification: Kokkos GPU atomics should default to `gpu` scope -#define __simt_scope "gpu" -#endif - -#define __simt_fence_signal_() asm volatile("" ::: "memory") -#define __simt_fence_sc_() \ - asm volatile("fence.sc." __simt_scope ";" ::: "memory") -#define __simt_fence_() asm volatile("fence." __simt_scope ";" ::: "memory") - -#define __simt_load_acquire_8_as_32(ptr, ret) \ - asm volatile("ld.acquire." __simt_scope ".b8 %0, [%1];" \ - : "=r"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_load_relaxed_8_as_32(ptr, ret) \ - asm volatile("ld.relaxed." __simt_scope ".b8 %0, [%1];" \ - : "=r"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_store_release_8_as_32(ptr, desired) \ - asm volatile("st.release." __simt_scope ".b8 [%0], %1;" ::"l"(ptr), \ - "r"(desired) \ - : "memory") -#define __simt_store_relaxed_8_as_32(ptr, desired) \ - asm volatile("st.relaxed." __simt_scope ".b8 [%0], %1;" ::"l"(ptr), \ - "r"(desired) \ - : "memory") - -#define __simt_load_acquire_16(ptr, ret) \ - asm volatile("ld.acquire." __simt_scope ".b16 %0, [%1];" \ - : "=h"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_load_relaxed_16(ptr, ret) \ - asm volatile("ld.relaxed." __simt_scope ".b16 %0, [%1];" \ - : "=h"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_store_release_16(ptr, desired) \ - asm volatile("st.release." __simt_scope ".b16 [%0], %1;" ::"l"(ptr), \ - "h"(desired) \ - : "memory") -#define __simt_store_relaxed_16(ptr, desired) \ - asm volatile("st.relaxed." __simt_scope ".b16 [%0], %1;" ::"l"(ptr), \ - "h"(desired) \ - : "memory") - -#define __simt_load_acquire_32(ptr, ret) \ - asm volatile("ld.acquire." __simt_scope ".b32 %0, [%1];" \ - : "=r"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_load_relaxed_32(ptr, ret) \ - asm volatile("ld.relaxed." __simt_scope ".b32 %0, [%1];" \ - : "=r"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_store_release_32(ptr, desired) \ - asm volatile("st.release." __simt_scope ".b32 [%0], %1;" ::"l"(ptr), \ - "r"(desired) \ - : "memory") -#define __simt_store_relaxed_32(ptr, desired) \ - asm volatile("st.relaxed." __simt_scope ".b32 [%0], %1;" ::"l"(ptr), \ - "r"(desired) \ - : "memory") -#define __simt_exch_release_32(ptr, old, desired) \ - asm volatile("atom.exch.release." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(desired) \ - : "memory") -#define __simt_exch_acquire_32(ptr, old, desired) \ - asm volatile("atom.exch.acquire." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(desired) \ - : "memory") -#define __simt_exch_acq_rel_32(ptr, old, desired) \ - asm volatile("atom.exch.acq_rel." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(desired) \ - : "memory") -#define __simt_exch_relaxed_32(ptr, old, desired) \ - asm volatile("atom.exch.relaxed." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(desired) \ - : "memory") -#define __simt_cas_release_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.release." __simt_scope ".b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define __simt_cas_acquire_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.acquire." __simt_scope ".b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define __simt_cas_acq_rel_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.acq_rel." __simt_scope ".b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define __simt_cas_relaxed_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.relaxed." __simt_scope ".b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define __simt_add_release_32(ptr, old, addend) \ - asm volatile("atom.add.release." __simt_scope ".u32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(addend) \ - : "memory") -#define __simt_add_acquire_32(ptr, old, addend) \ - asm volatile("atom.add.acquire." __simt_scope ".u32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(addend) \ - : "memory") -#define __simt_add_acq_rel_32(ptr, old, addend) \ - asm volatile("atom.add.acq_rel." __simt_scope ".u32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(addend) \ - : "memory") -#define __simt_add_relaxed_32(ptr, old, addend) \ - asm volatile("atom.add.relaxed." __simt_scope ".u32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(addend) \ - : "memory") -#define __simt_and_release_32(ptr, old, andend) \ - asm volatile("atom.and.release." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(andend) \ - : "memory") -#define __simt_and_acquire_32(ptr, old, andend) \ - asm volatile("atom.and.acquire." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(andend) \ - : "memory") -#define __simt_and_acq_rel_32(ptr, old, andend) \ - asm volatile("atom.and.acq_rel." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(andend) \ - : "memory") -#define __simt_and_relaxed_32(ptr, old, andend) \ - asm volatile("atom.and.relaxed." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(andend) \ - : "memory") -#define __simt_or_release_32(ptr, old, orend) \ - asm volatile("atom.or.release." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(orend) \ - : "memory") -#define __simt_or_acquire_32(ptr, old, orend) \ - asm volatile("atom.or.acquire." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(orend) \ - : "memory") -#define __simt_or_acq_rel_32(ptr, old, orend) \ - asm volatile("atom.or.acq_rel." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(orend) \ - : "memory") -#define __simt_or_relaxed_32(ptr, old, orend) \ - asm volatile("atom.or.relaxed." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(orend) \ - : "memory") -#define __simt_xor_release_32(ptr, old, xorend) \ - asm volatile("atom.xor.release." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(xorend) \ - : "memory") -#define __simt_xor_acquire_32(ptr, old, xorend) \ - asm volatile("atom.xor.acquire." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(xorend) \ - : "memory") -#define __simt_xor_acq_rel_32(ptr, old, xorend) \ - asm volatile("atom.xor.acq_rel." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(xorend) \ - : "memory") -#define __simt_xor_relaxed_32(ptr, old, xorend) \ - asm volatile("atom.xor.relaxed." __simt_scope ".b32 %0, [%1], %2;" \ - : "=r"(old) \ - : "l"(ptr), "r"(xorend) \ - : "memory") - -#define __simt_load_acquire_64(ptr, ret) \ - asm volatile("ld.acquire." __simt_scope ".b64 %0, [%1];" \ - : "=l"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_load_relaxed_64(ptr, ret) \ - asm volatile("ld.relaxed." __simt_scope ".b64 %0, [%1];" \ - : "=l"(ret) \ - : "l"(ptr) \ - : "memory") -#define __simt_store_release_64(ptr, desired) \ - asm volatile("st.release." __simt_scope ".b64 [%0], %1;" ::"l"(ptr), \ - "l"(desired) \ - : "memory") -#define __simt_store_relaxed_64(ptr, desired) \ - asm volatile("st.relaxed." __simt_scope ".b64 [%0], %1;" ::"l"(ptr), \ - "l"(desired) \ - : "memory") -#define __simt_exch_release_64(ptr, old, desired) \ - asm volatile("atom.exch.release." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(desired) \ - : "memory") -#define __simt_exch_acquire_64(ptr, old, desired) \ - asm volatile("atom.exch.acquire." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(desired) \ - : "memory") -#define __simt_exch_acq_rel_64(ptr, old, desired) \ - asm volatile("atom.exch.acq_rel." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(desired) \ - : "memory") -#define __simt_exch_relaxed_64(ptr, old, desired) \ - asm volatile("atom.exch.relaxed." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(desired) \ - : "memory") -#define __simt_cas_release_64(ptr, old, expected, desired) \ - asm volatile("atom.cas.release." __simt_scope ".b64 %0, [%1], %2, %3;" \ - : "=l"(old) \ - : "l"(ptr), "l"(expected), "l"(desired) \ - : "memory") -#define __simt_cas_acquire_64(ptr, old, expected, desired) \ - asm volatile("atom.cas.acquire." __simt_scope ".b64 %0, [%1], %2, %3;" \ - : "=l"(old) \ - : "l"(ptr), "l"(expected), "l"(desired) \ - : "memory") -#define __simt_cas_acq_rel_64(ptr, old, expected, desired) \ - asm volatile("atom.cas.acq_rel." __simt_scope ".b64 %0, [%1], %2, %3;" \ - : "=l"(old) \ - : "l"(ptr), "l"(expected), "l"(desired) \ - : "memory") -#define __simt_cas_relaxed_64(ptr, old, expected, desired) \ - asm volatile("atom.cas.relaxed." __simt_scope ".b64 %0, [%1], %2, %3;" \ - : "=l"(old) \ - : "l"(ptr), "l"(expected), "l"(desired) \ - : "memory") -#define __simt_add_release_64(ptr, old, addend) \ - asm volatile("atom.add.release." __simt_scope ".u64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(addend) \ - : "memory") -#define __simt_add_acquire_64(ptr, old, addend) \ - asm volatile("atom.add.acquire." __simt_scope ".u64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(addend) \ - : "memory") -#define __simt_add_acq_rel_64(ptr, old, addend) \ - asm volatile("atom.add.acq_rel." __simt_scope ".u64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(addend) \ - : "memory") -#define __simt_add_relaxed_64(ptr, old, addend) \ - asm volatile("atom.add.relaxed." __simt_scope ".u64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(addend) \ - : "memory") -#define __simt_and_release_64(ptr, old, andend) \ - asm volatile("atom.and.release." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(andend) \ - : "memory") -#define __simt_and_acquire_64(ptr, old, andend) \ - asm volatile("atom.and.acquire." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(andend) \ - : "memory") -#define __simt_and_acq_rel_64(ptr, old, andend) \ - asm volatile("atom.and.acq_rel." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(andend) \ - : "memory") -#define __simt_and_relaxed_64(ptr, old, andend) \ - asm volatile("atom.and.relaxed." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(andend) \ - : "memory") -#define __simt_or_release_64(ptr, old, orend) \ - asm volatile("atom.or.release." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(orend) \ - : "memory") -#define __simt_or_acquire_64(ptr, old, orend) \ - asm volatile("atom.or.acquire." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(orend) \ - : "memory") -#define __simt_or_acq_rel_64(ptr, old, orend) \ - asm volatile("atom.or.acq_rel." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(orend) \ - : "memory") -#define __simt_or_relaxed_64(ptr, old, orend) \ - asm volatile("atom.or.relaxed." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(orend) \ - : "memory") -#define __simt_xor_release_64(ptr, old, xorend) \ - asm volatile("atom.xor.release." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(xorend) \ - : "memory") -#define __simt_xor_acquire_64(ptr, old, xorend) \ - asm volatile("atom.xor.acquire." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(xorend) \ - : "memory") -#define __simt_xor_acq_rel_64(ptr, old, xorend) \ - asm volatile("atom.xor.acq_rel." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(xorend) \ - : "memory") -#define __simt_xor_relaxed_64(ptr, old, xorend) \ - asm volatile("atom.xor.relaxed." __simt_scope ".b64 %0, [%1], %2;" \ - : "=l"(old) \ - : "l"(ptr), "l"(xorend) \ - : "memory") - -#define __simt_nanosleep(timeout) \ - asm volatile("nanosleep.u32 %0;" ::"r"(unsigned(timeout)) :) - -/* - definitions -*/ - -#ifndef __GCC_ATOMIC_BOOL_LOCK_FREE -#define __GCC_ATOMIC_BOOL_LOCK_FREE 2 -#define __GCC_ATOMIC_CHAR_LOCK_FREE 2 -#define __GCC_ATOMIC_CHAR16_T_LOCK_FREE 2 -#define __GCC_ATOMIC_CHAR32_T_LOCK_FREE 2 -#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2 -#define __GCC_ATOMIC_SHORT_LOCK_FREE 2 -#define __GCC_ATOMIC_INT_LOCK_FREE 2 -#define __GCC_ATOMIC_LONG_LOCK_FREE 2 -#define __GCC_ATOMIC_LLONG_LOCK_FREE 2 -#define __GCC_ATOMIC_POINTER_LOCK_FREE 2 -#endif - -#ifndef __ATOMIC_RELAXED -#define __ATOMIC_RELAXED 0 -#define __ATOMIC_CONSUME 1 -#define __ATOMIC_ACQUIRE 2 -#define __ATOMIC_RELEASE 3 -#define __ATOMIC_ACQ_REL 4 -#define __ATOMIC_SEQ_CST 5 -#endif - -inline __device__ int __stronger_order_simt_(int a, int b) { - if (b == __ATOMIC_SEQ_CST) return __ATOMIC_SEQ_CST; - if (b == __ATOMIC_RELAXED) return a; - switch (a) { - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: return a; - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - if (b != __ATOMIC_ACQUIRE) - return __ATOMIC_ACQ_REL; - else - return __ATOMIC_ACQUIRE; - case __ATOMIC_RELEASE: - if (b != __ATOMIC_RELEASE) - return __ATOMIC_ACQ_REL; - else - return __ATOMIC_RELEASE; - case __ATOMIC_RELAXED: return b; - default: assert(0); - } - return __ATOMIC_SEQ_CST; -} - -/* - base -*/ - -#define DO__atomic_load_simt_(bytes, bits) \ - template = 0> \ - void __device__ __atomic_load_simt_(const type *ptr, type *ret, \ - int memorder) { \ - int##bits##_t tmp = 0; \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_load_acquire_##bits(ptr, tmp); break; \ - case __ATOMIC_RELAXED: __simt_load_relaxed_##bits(ptr, tmp); break; \ - default: assert(0); \ - } \ - memcpy(ret, &tmp, bytes); \ - } -DO__atomic_load_simt_(1, 32) DO__atomic_load_simt_(2, 16) - DO__atomic_load_simt_(4, 32) DO__atomic_load_simt_(8, 64) - - template - type __device__ __atomic_load_n_simt_(const type *ptr, int memorder) { - type ret; - __atomic_load_simt_(ptr, &ret, memorder); - return ret; -} - -#define DO__atomic_store_simt_(bytes, bits) \ - template = 0> \ - void __device__ __atomic_store_simt_(type *ptr, type *val, int memorder) { \ - int##bits##_t tmp = 0; \ - memcpy(&tmp, val, bytes); \ - switch (memorder) { \ - case __ATOMIC_RELEASE: __simt_store_release_##bits(ptr, tmp); break; \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_RELAXED: __simt_store_relaxed_##bits(ptr, tmp); break; \ - default: assert(0); \ - } \ - } -DO__atomic_store_simt_(1, 32) DO__atomic_store_simt_(2, 16) - DO__atomic_store_simt_(4, 32) DO__atomic_store_simt_(8, 64) - - template - void __device__ - __atomic_store_n_simt_(type *ptr, type val, int memorder) { - __atomic_store_simt_(ptr, &val, memorder); -} - -#define DO__atomic_compare_exchange_simt_(bytes, bits) \ - template = 0> \ - bool __device__ __atomic_compare_exchange_simt_( \ - type *ptr, type *expected, const type *desired, bool, \ - int success_memorder, int failure_memorder) { \ - int##bits##_t tmp = 0, old = 0, old_tmp; \ - memcpy(&tmp, desired, bytes); \ - memcpy(&old, expected, bytes); \ - old_tmp = old; \ - switch (__stronger_order_simt_(success_memorder, failure_memorder)) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: \ - __simt_cas_acquire_##bits(ptr, old, old_tmp, tmp); \ - break; \ - case __ATOMIC_ACQ_REL: \ - __simt_cas_acq_rel_##bits(ptr, old, old_tmp, tmp); \ - break; \ - case __ATOMIC_RELEASE: \ - __simt_cas_release_##bits(ptr, old, old_tmp, tmp); \ - break; \ - case __ATOMIC_RELAXED: \ - __simt_cas_relaxed_##bits(ptr, old, old_tmp, tmp); \ - break; \ - default: assert(0); \ - } \ - bool const ret = old == old_tmp; \ - memcpy(expected, &old, bytes); \ - return ret; \ - } -DO__atomic_compare_exchange_simt_(4, 32) - DO__atomic_compare_exchange_simt_(8, 64) - - template = 0> - bool __device__ - __atomic_compare_exchange_simt_(type *ptr, type *expected, - const type *desired, bool, - int success_memorder, - int failure_memorder) { - using R = std::conditional_t::value, volatile uint32_t, - uint32_t>; - auto const aligned = (R *)((intptr_t)ptr & ~(sizeof(uint32_t) - 1)); - auto const offset = uint32_t((intptr_t)ptr & (sizeof(uint32_t) - 1)) * 8; - auto const mask = ((1 << sizeof(type) * 8) - 1) << offset; - - uint32_t old = *expected << offset, old_value; - while (1) { - old_value = (old & mask) >> offset; - if (old_value != *expected) break; - uint32_t const attempt = (old & ~mask) | (*desired << offset); - if (__atomic_compare_exchange_simt_(aligned, &old, &attempt, true, - success_memorder, failure_memorder)) - return true; - } - *expected = old_value; - return false; -} - -template -bool __device__ __atomic_compare_exchange_n_simt_(type *ptr, type *expected, - type desired, bool weak, - int success_memorder, - int failure_memorder) { - return __atomic_compare_exchange_simt_(ptr, expected, &desired, weak, - success_memorder, failure_memorder); -} - -#define DO__atomic_exchange_simt_(bytes, bits) \ - template = 0> \ - void __device__ __atomic_exchange_simt_(type *ptr, type *val, type *ret, \ - int memorder) { \ - int##bits##_t tmp = 0; \ - memcpy(&tmp, val, bytes); \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_exch_acquire_##bits(ptr, tmp, tmp); break; \ - case __ATOMIC_ACQ_REL: __simt_exch_acq_rel_##bits(ptr, tmp, tmp); break; \ - case __ATOMIC_RELEASE: __simt_exch_release_##bits(ptr, tmp, tmp); break; \ - case __ATOMIC_RELAXED: __simt_exch_relaxed_##bits(ptr, tmp, tmp); break; \ - default: assert(0); \ - } \ - memcpy(ret, &tmp, bytes); \ - } -DO__atomic_exchange_simt_(4, 32) DO__atomic_exchange_simt_(8, 64) - - template = 0> - void __device__ - __atomic_exchange_simt_(type *ptr, type *val, type *ret, int memorder) { - type expected = __atomic_load_n_simt_(ptr, __ATOMIC_RELAXED); - while (!__atomic_compare_exchange_simt_(ptr, &expected, val, true, memorder, - memorder)) - ; - *ret = expected; -} - -template -type __device__ __atomic_exchange_n_simt_(type *ptr, type val, int memorder) { - type ret; - __atomic_exchange_simt_(ptr, &val, &ret, memorder); - return ret; -} - -#define DO__atomic_fetch_add_simt_(bytes, bits) \ - template = 0> \ - type __device__ __atomic_fetch_add_simt_(type *ptr, delta val, \ - int memorder) { \ - type ret; \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_add_acquire_##bits(ptr, ret, val); break; \ - case __ATOMIC_ACQ_REL: __simt_add_acq_rel_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELEASE: __simt_add_release_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELAXED: __simt_add_relaxed_##bits(ptr, ret, val); break; \ - default: assert(0); \ - } \ - return ret; \ - } -DO__atomic_fetch_add_simt_(4, 32) DO__atomic_fetch_add_simt_(8, 64) - - template = 0> - type __device__ - __atomic_fetch_add_simt_(type *ptr, delta val, int memorder) { - type expected = __atomic_load_n_simt_(ptr, __ATOMIC_RELAXED); - type const desired = expected + val; - while (!__atomic_compare_exchange_simt_(ptr, &expected, &desired, true, - memorder, memorder)) - ; - return expected; -} - -#define DO__atomic_fetch_sub_simt_(bytes, bits) \ - template = 0> \ - type __device__ __atomic_fetch_sub_simt_(type *ptr, delta val, \ - int memorder) { \ - type ret; \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_add_acquire_##bits(ptr, ret, -val); break; \ - case __ATOMIC_ACQ_REL: __simt_add_acq_rel_##bits(ptr, ret, -val); break; \ - case __ATOMIC_RELEASE: __simt_add_release_##bits(ptr, ret, -val); break; \ - case __ATOMIC_RELAXED: __simt_add_relaxed_##bits(ptr, ret, -val); break; \ - default: assert(0); \ - } \ - return ret; \ - } -DO__atomic_fetch_sub_simt_(4, 32) DO__atomic_fetch_sub_simt_(8, 64) - - template = 0> - type __device__ - __atomic_fetch_sub_simt_(type *ptr, delta val, int memorder) { - type expected = __atomic_load_n_simt_(ptr, __ATOMIC_RELAXED); - type const desired = expected - val; - while (!__atomic_compare_exchange_simt_(ptr, &expected, &desired, true, - memorder, memorder)) - ; - return expected; -} - -#define DO__atomic_fetch_and_simt_(bytes, bits) \ - template = 0> \ - type __device__ __atomic_fetch_and_simt_(type *ptr, type val, \ - int memorder) { \ - type ret; \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_and_acquire_##bits(ptr, ret, val); break; \ - case __ATOMIC_ACQ_REL: __simt_and_acq_rel_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELEASE: __simt_and_release_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELAXED: __simt_and_relaxed_##bits(ptr, ret, val); break; \ - default: assert(0); \ - } \ - return ret; \ - } -DO__atomic_fetch_and_simt_(4, 32) DO__atomic_fetch_and_simt_(8, 64) - - template = 0> - type __device__ - __atomic_fetch_and_simt_(type *ptr, delta val, int memorder) { - type expected = __atomic_load_n_simt_(ptr, __ATOMIC_RELAXED); - type const desired = expected & val; - while (!__atomic_compare_exchange_simt_(ptr, &expected, &desired, true, - memorder, memorder)) - ; - return expected; -} - -#define DO__atomic_fetch_xor_simt_(bytes, bits) \ - template = 0> \ - type __device__ __atomic_fetch_xor_simt_(type *ptr, type val, \ - int memorder) { \ - type ret; \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_xor_acquire_##bits(ptr, ret, val); break; \ - case __ATOMIC_ACQ_REL: __simt_xor_acq_rel_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELEASE: __simt_xor_release_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELAXED: __simt_xor_relaxed_##bits(ptr, ret, val); break; \ - default: assert(0); \ - } \ - return ret; \ - } -DO__atomic_fetch_xor_simt_(4, 32) DO__atomic_fetch_xor_simt_(8, 64) - - template = 0> - type __device__ - __atomic_fetch_xor_simt_(type *ptr, delta val, int memorder) { - type expected = __atomic_load_n_simt_(ptr, __ATOMIC_RELAXED); - type const desired = expected ^ val; - while (!__atomic_compare_exchange_simt_(ptr, &expected, &desired, true, - memorder, memorder)) - ; - return expected; -} - -#define DO__atomic_fetch_or_simt_(bytes, bits) \ - template = 0> \ - type __device__ __atomic_fetch_or_simt_(type *ptr, type val, int memorder) { \ - type ret; \ - switch (memorder) { \ - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); \ - case __ATOMIC_CONSUME: \ - case __ATOMIC_ACQUIRE: __simt_or_acquire_##bits(ptr, ret, val); break; \ - case __ATOMIC_ACQ_REL: __simt_or_acq_rel_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELEASE: __simt_or_release_##bits(ptr, ret, val); break; \ - case __ATOMIC_RELAXED: __simt_or_relaxed_##bits(ptr, ret, val); break; \ - default: assert(0); \ - } \ - return ret; \ - } -DO__atomic_fetch_or_simt_(4, 32) DO__atomic_fetch_or_simt_(8, 64) - - template = 0> - type __device__ - __atomic_fetch_or_simt_(type *ptr, delta val, int memorder) { - type expected = __atomic_load_n_simt_(ptr, __ATOMIC_RELAXED); - type const desired = expected | val; - while (!__atomic_compare_exchange_simt_(ptr, &expected, &desired, true, - memorder, memorder)) - ; - return expected; -} - -template -inline bool __device__ __atomic_test_and_set_simt_(type *ptr, int memorder) { - return __atomic_exchange_n_simt_((char *)ptr, (char)1, memorder) == 1; -} -template -inline void __device__ __atomic_clear_simt_(type *ptr, int memorder) { - return __atomic_store_n_simt_((char *)ptr, (char)0, memorder); -} - -inline constexpr __device__ bool __atomic_always_lock_free_simt_(size_t size, - void *) { - return size <= 8; -} -inline __device__ bool __atomic_is_lock_free_simt_(size_t size, void *ptr) { - return __atomic_always_lock_free_simt_(size, ptr); -} - -/* - fences -*/ - -inline void __device__ __atomic_thread_fence_simt(int memorder) { - switch (memorder) { - case __ATOMIC_SEQ_CST: __simt_fence_sc_(); break; - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __simt_fence_(); break; - case __ATOMIC_RELAXED: break; - default: assert(0); - } -} -inline void __device__ __atomic_signal_fence_simt(int memorder) { - __atomic_thread_fence_simt(memorder); -} - -/* - non-volatile -*/ - -template -type __device__ __atomic_load_n_simt(const type *ptr, int memorder) { - return __atomic_load_n_simt_(const_cast(ptr), memorder); -} -template -void __device__ __atomic_load_simt(const type *ptr, type *ret, int memorder) { - __atomic_load_simt_(const_cast(ptr), ret, memorder); -} -template -void __device__ __atomic_store_n_simt(type *ptr, type val, int memorder) { - __atomic_store_n_simt_(const_cast(ptr), val, memorder); -} -template -void __device__ __atomic_store_simt(type *ptr, type *val, int memorder) { - __atomic_store_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_exchange_n_simt(type *ptr, type val, int memorder) { - return __atomic_exchange_n_simt_(const_cast(ptr), val, memorder); -} -template -void __device__ __atomic_exchange_simt(type *ptr, type *val, type *ret, - int memorder) { - __atomic_exchange_simt_(const_cast(ptr), val, ret, memorder); -} -template -bool __device__ __atomic_compare_exchange_n_simt(type *ptr, type *expected, - type desired, bool weak, - int success_memorder, - int failure_memorder) { - return __atomic_compare_exchange_n_simt_(const_cast(ptr), expected, - desired, weak, success_memorder, - failure_memorder); -} -template -bool __device__ __atomic_compare_exchange_simt(type *ptr, type *expected, - type *desired, bool weak, - int success_memorder, - int failure_memorder) { - return __atomic_compare_exchange_simt_(const_cast(ptr), expected, - desired, weak, success_memorder, - failure_memorder); -} -template -type __device__ __atomic_fetch_add_simt(type *ptr, delta val, int memorder) { - return __atomic_fetch_add_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_sub_simt(type *ptr, delta val, int memorder) { - return __atomic_fetch_sub_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_and_simt(type *ptr, type val, int memorder) { - return __atomic_fetch_and_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_xor_simt(type *ptr, type val, int memorder) { - return __atomic_fetch_xor_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_or_simt(type *ptr, type val, int memorder) { - return __atomic_fetch_or_simt_(const_cast(ptr), val, memorder); -} -template -bool __device__ __atomic_test_and_set_simt(void *ptr, int memorder) { - return __atomic_test_and_set_simt_(const_cast(ptr), memorder); -} -template -void __device__ __atomic_clear_simt(void *ptr, int memorder) { - return __atomic_clear_simt_(const_cast(ptr), memorder); -} -inline bool __device__ __atomic_always_lock_free_simt(size_t size, void *ptr) { - return __atomic_always_lock_free_simt_(size, const_cast(ptr)); -} -inline bool __device__ __atomic_is_lock_free_simt(size_t size, void *ptr) { - return __atomic_is_lock_free_simt_(size, const_cast(ptr)); -} - -/* - volatile -*/ - -template -type __device__ __atomic_load_n_simt(const volatile type *ptr, int memorder) { - return __atomic_load_n_simt_(const_cast(ptr), memorder); -} -template -void __device__ __atomic_load_simt(const volatile type *ptr, type *ret, - int memorder) { - __atomic_load_simt_(const_cast(ptr), ret, memorder); -} -template -void __device__ __atomic_store_n_simt(volatile type *ptr, type val, - int memorder) { - __atomic_store_n_simt_(const_cast(ptr), val, memorder); -} -template -void __device__ __atomic_store_simt(volatile type *ptr, type *val, - int memorder) { - __atomic_store_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_exchange_n_simt(volatile type *ptr, type val, - int memorder) { - return __atomic_exchange_n_simt_(const_cast(ptr), val, memorder); -} -template -void __device__ __atomic_exchange_simt(volatile type *ptr, type *val, type *ret, - int memorder) { - __atomic_exchange_simt_(const_cast(ptr), val, ret, memorder); -} -template -bool __device__ __atomic_compare_exchange_n_simt(volatile type *ptr, - type *expected, type desired, - bool weak, - int success_memorder, - int failure_memorder) { - return __atomic_compare_exchange_n_simt_(const_cast(ptr), expected, - desired, weak, success_memorder, - failure_memorder); -} -template -bool __device__ __atomic_compare_exchange_simt(volatile type *ptr, - type *expected, type *desired, - bool weak, int success_memorder, - int failure_memorder) { - return __atomic_compare_exchange_simt_(const_cast(ptr), expected, - desired, weak, success_memorder, - failure_memorder); -} -template -type __device__ __atomic_fetch_add_simt(volatile type *ptr, delta val, - int memorder) { - return __atomic_fetch_add_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_sub_simt(volatile type *ptr, delta val, - int memorder) { - return __atomic_fetch_sub_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_and_simt(volatile type *ptr, type val, - int memorder) { - return __atomic_fetch_and_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_xor_simt(volatile type *ptr, type val, - int memorder) { - return __atomic_fetch_xor_simt_(const_cast(ptr), val, memorder); -} -template -type __device__ __atomic_fetch_or_simt(volatile type *ptr, type val, - int memorder) { - return __atomic_fetch_or_simt_(const_cast(ptr), val, memorder); -} -template -bool __device__ __atomic_test_and_set_simt(volatile void *ptr, int memorder) { - return __atomic_test_and_set_simt_(const_cast(ptr), memorder); -} -template -void __device__ __atomic_clear_simt(volatile void *ptr, int memorder) { - return __atomic_clear_simt_(const_cast(ptr), memorder); -} - -} // end namespace Impl -} // end namespace Kokkos - -#endif //_SIMT_DETAILS_CONFIG - -#ifndef KOKKOS_SIMT_ATOMIC_BUILTIN_REPLACEMENTS_DEFINED -/* - builtins -*/ - -#define __atomic_load_n __atomic_load_n_simt -#define __atomic_load __atomic_load_simt -#define __atomic_store_n __atomic_store_n_simt -#define __atomic_store __atomic_store_simt -#define __atomic_exchange_n __atomic_exchange_n_simt -#define __atomic_exchange __atomic_exchange_simt -#define __atomic_compare_exchange_n __atomic_compare_exchange_n_simt -#define __atomic_compare_exchange __atomic_compare_exchange_simt -#define __atomic_fetch_add __atomic_fetch_add_simt -#define __atomic_fetch_sub __atomic_fetch_sub_simt -#define __atomic_fetch_and __atomic_fetch_and_simt -#define __atomic_fetch_xor __atomic_fetch_xor_simt -#define __atomic_fetch_or __atomic_fetch_or_simt -#define __atomic_test_and_set __atomic_test_and_set_simt -#define __atomic_clear __atomic_clear_simt -#define __atomic_always_lock_free __atomic_always_lock_free_simt -#define __atomic_is_lock_free __atomic_is_lock_free_simt -#define __atomic_thread_fence __atomic_thread_fence_simt -#define __atomic_signal_fence __atomic_signal_fence_simt - -#define KOKKOS_SIMT_ATOMIC_BUILTIN_REPLACEMENTS_DEFINED - -#endif //__CUDA_ARCH__ && KOKKOS_ENABLE_CUDA_ASM_ATOMICS -#endif // KOKKOS_SIMT_ATOMIC_BUILTIN_REPLACEMENTS_DEFINED diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Atomic_Intrinsics_Restore_Builtins.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Atomic_Intrinsics_Restore_Builtins.hpp deleted file mode 100644 index d7cd1bab13..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Atomic_Intrinsics_Restore_Builtins.hpp +++ /dev/null @@ -1,68 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifdef KOKKOS_SIMT_ATOMIC_BUILTIN_REPLACEMENTS_DEFINED - -#undef __atomic_load_n -#undef __atomic_load -#undef __atomic_store_n -#undef __atomic_store -#undef __atomic_exchange_n -#undef __atomic_exchange -#undef __atomic_compare_exchange_n -#undef __atomic_compare_exchange -#undef __atomic_fetch_add -#undef __atomic_fetch_sub -#undef __atomic_fetch_and -#undef __atomic_fetch_xor -#undef __atomic_fetch_or -#undef __atomic_test_and_set -#undef __atomic_clear -#undef __atomic_always_lock_free -#undef __atomic_is_lock_free -#undef __atomic_thread_fence -#undef __atomic_signal_fence - -#undef KOKKOS_SIMT_ATOMIC_BUILTIN_REPLACEMENTS_DEFINED - -#endif // KOKKOS_SIMT_ATOMIC_BUILTIN_REPLACEMENTS_DEFINED diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Error.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Error.hpp deleted file mode 100644 index e28e964d36..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Error.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_ERROR_HPP -#define KOKKOS_CUDA_ERROR_HPP - -#include -#ifdef KOKKOS_ENABLE_CUDA - -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -void cuda_stream_synchronize( - const cudaStream_t stream, - Kokkos::Tools::Experimental::SpecialSynchronizationCases reason, - const std::string& name); -void cuda_device_synchronize(const std::string& name); -void cuda_stream_synchronize(const cudaStream_t stream, - const std::string& name); - -[[noreturn]] void cuda_internal_error_throw(cudaError e, const char* name, - const char* file = nullptr, - const int line = 0); - -#ifndef KOKKOS_COMPILER_NVHPC -[[noreturn]] -#endif - void cuda_internal_error_abort(cudaError e, const char* name, - const char* file = nullptr, - const int line = 0); - -inline void cuda_internal_safe_call(cudaError e, const char* name, - const char* file = nullptr, - const int line = 0) { - // 1. Success -> normal continuation. - // 2. Error codes for which, to continue using CUDA, the process must be - // terminated and relaunched -> call abort on the host-side. - // 3. Any other error code -> throw a runtime error. - switch (e) { - case cudaSuccess: break; - case cudaErrorIllegalAddress: - case cudaErrorAssert: - case cudaErrorHardwareStackError: - case cudaErrorIllegalInstruction: - case cudaErrorMisalignedAddress: - case cudaErrorInvalidAddressSpace: - case cudaErrorInvalidPc: - case cudaErrorLaunchFailure: - cuda_internal_error_abort(e, name, file, line); - break; - default: cuda_internal_error_throw(e, name, file, line); break; - } -} - -#define KOKKOS_IMPL_CUDA_SAFE_CALL(call) \ - Kokkos::Impl::cuda_internal_safe_call(call, #call, __FILE__, __LINE__) - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - -KOKKOS_DEPRECATED -inline void cuda_internal_safe_call_deprecated(cudaError e, const char* name, - const char* file = nullptr, - const int line = 0) { - cuda_internal_safe_call(e, name, file, line); -} - -#define CUDA_SAFE_CALL(call) \ - Kokkos::Impl::cuda_internal_safe_call_deprecated(call, #call, __FILE__, \ - __LINE__) - -#endif - -} // namespace Impl - -namespace Experimental { - -class CudaRawMemoryAllocationFailure : public RawMemoryAllocationFailure { - private: - using base_t = RawMemoryAllocationFailure; - - cudaError_t m_error_code = cudaSuccess; - - static FailureMode get_failure_mode(cudaError_t error_code) { - switch (error_code) { - case cudaErrorMemoryAllocation: return FailureMode::OutOfMemoryError; - case cudaErrorInvalidValue: return FailureMode::InvalidAllocationSize; - // TODO handle cudaErrorNotSupported for cudaMallocManaged - default: return FailureMode::Unknown; - } - } - - public: - // using base_t::base_t; - // would trigger - // - // error: cannot determine the exception specification of the default - // constructor due to a circular dependency - // - // using NVCC 9.1 and gcc 7.4 - CudaRawMemoryAllocationFailure( - size_t arg_attempted_size, size_t arg_attempted_alignment, - FailureMode arg_failure_mode = FailureMode::OutOfMemoryError, - AllocationMechanism arg_mechanism = - AllocationMechanism::StdMalloc) noexcept - : base_t(arg_attempted_size, arg_attempted_alignment, arg_failure_mode, - arg_mechanism) {} - - CudaRawMemoryAllocationFailure(size_t arg_attempted_size, - cudaError_t arg_error_code, - AllocationMechanism arg_mechanism) noexcept - : base_t(arg_attempted_size, /* CudaSpace doesn't handle alignment? */ 1, - get_failure_mode(arg_error_code), arg_mechanism), - m_error_code(arg_error_code) {} - - void append_additional_error_information(std::ostream& o) const override; -}; - -} // end namespace Experimental - -} // namespace Kokkos - -#endif // KOKKOS_ENABLE_CUDA -#endif // KOKKOS_CUDA_ERROR_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Half_Impl_Type.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Half_Impl_Type.hpp deleted file mode 100644 index e296a9239a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Half_Impl_Type.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_HALF_IMPL_TYPE_HPP_ -#define KOKKOS_CUDA_HALF_IMPL_TYPE_HPP_ - -#include -#ifdef KOKKOS_ENABLE_CUDA -#if !(defined(KOKKOS_COMPILER_CLANG) && KOKKOS_COMPILER_CLANG < 900) && \ - !(defined(KOKKOS_ARCH_KEPLER) || defined(KOKKOS_ARCH_MAXWELL50) || \ - defined(KOKKOS_ARCH_MAXWELL52)) -#include -#if (CUDA_VERSION >= 11000) -#include -#endif // CUDA_VERSION >= 11000 - -#ifndef KOKKOS_IMPL_HALF_TYPE_DEFINED -// Make sure no one else tries to define half_t -#define KOKKOS_IMPL_HALF_TYPE_DEFINED -#define KOKKOS_IMPL_CUDA_HALF_TYPE_DEFINED - -namespace Kokkos { -namespace Impl { -struct half_impl_t { - using type = __half; -}; -#if (CUDA_VERSION >= 11000) -#define KOKKOS_IMPL_BHALF_TYPE_DEFINED -struct bhalf_impl_t { - using type = __nv_bfloat16; -}; -#endif // CUDA_VERSION >= 11000 -} // namespace Impl -} // namespace Kokkos -#endif // KOKKOS_IMPL_HALF_TYPE_DEFINED -#endif // Disables for half_t on cuda: - // Clang/8||KEPLER30||KEPLER32||KEPLER37||MAXWELL50||MAXWELL52 -#endif // KOKKOS_ENABLE_CUDA -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Instance.cpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Instance.cpp deleted file mode 100644 index 5811498e01..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Instance.cpp +++ /dev/null @@ -1,1010 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/*--------------------------------------------------------------------------*/ -/* Kokkos interfaces */ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#ifdef KOKKOS_ENABLE_CUDA - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/*--------------------------------------------------------------------------*/ -/* Standard 'C' libraries */ -#include - -/* Standard 'C++' libraries */ -#include -#include -#include -#include - -#ifdef KOKKOS_IMPL_DEBUG_CUDA_SERIAL_EXECUTION -namespace Kokkos { -namespace Impl { - -bool CudaInternal::kokkos_impl_cuda_use_serial_execution_v = false; - -void CudaInternal::cuda_set_serial_execution(bool val) { - CudaInternal::kokkos_impl_cuda_use_serial_execution_v = val; -} -bool CudaInternal::cuda_use_serial_execution() { - return CudaInternal::kokkos_impl_cuda_use_serial_execution_v; -} - -} // namespace Impl -} // namespace Kokkos - -void kokkos_impl_cuda_set_serial_execution(bool val) { - Kokkos::Impl::CudaInternal::cuda_set_serial_execution(val); -} -bool kokkos_impl_cuda_use_serial_execution() { - return Kokkos::Impl::CudaInternal::cuda_use_serial_execution(); -} -#endif - -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE - -__device__ __constant__ unsigned long kokkos_impl_cuda_constant_memory_buffer - [Kokkos::Impl::CudaTraits::ConstantMemoryUsage / sizeof(unsigned long)]; - -#endif - -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Impl { - -namespace { - -__global__ void query_cuda_kernel_arch(int *d_arch) { -#ifdef _NVHPC_CUDA - *d_arch = __builtin_current_device_sm() * 10; -#else -#if defined(__CUDA_ARCH__) - *d_arch = __CUDA_ARCH__; -#else - *d_arch = 0; -#endif -#endif -} - -/** Query what compute capability is actually launched to the device: */ -int cuda_kernel_arch() { - int arch = 0; - int *d_arch = nullptr; - - cudaMalloc(reinterpret_cast(&d_arch), sizeof(int)); - cudaMemcpy(d_arch, &arch, sizeof(int), cudaMemcpyDefault); - - query_cuda_kernel_arch<<<1, 1>>>(d_arch); - - cudaMemcpy(&arch, d_arch, sizeof(int), cudaMemcpyDefault); - cudaFree(d_arch); - return arch; -} - -} // namespace - -Kokkos::View cuda_global_unique_token_locks( - bool deallocate) { - static Kokkos::View locks = - Kokkos::View(); - if (!deallocate && locks.extent(0) == 0) - locks = Kokkos::View( - "Kokkos::UniqueToken::m_locks", Kokkos::Cuda().concurrency()); - if (deallocate) locks = Kokkos::View(); - return locks; -} - -void cuda_device_synchronize(const std::string &name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - []() { // TODO: correct device ID - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaDeviceSynchronize()); - }); -} - -void cuda_stream_synchronize(const cudaStream_t stream, const CudaInternal *ptr, - const std::string &name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, - Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{ - ptr->impl_get_instance_id()}, - [&]() { // TODO: correct device ID - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaStreamSynchronize(stream)); - }); -} - -void cuda_stream_synchronize( - const cudaStream_t stream, - Kokkos::Tools::Experimental::SpecialSynchronizationCases reason, - const std::string &name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, reason, [&]() { // TODO: correct device ID - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaStreamSynchronize(stream)); - }); -} - -void cuda_internal_error_throw(cudaError e, const char *name, const char *file, - const int line) { - std::ostringstream out; - out << name << " error( " << cudaGetErrorName(e) - << "): " << cudaGetErrorString(e); - if (file) { - out << " " << file << ":" << line; - } - throw_runtime_exception(out.str()); -} - -void cuda_internal_error_abort(cudaError e, const char *name, const char *file, - const int line) { - std::ostringstream out; - out << name << " error( " << cudaGetErrorName(e) - << "): " << cudaGetErrorString(e); - if (file) { - out << " " << file << ":" << line; - } - abort(out.str().c_str()); -} - -//---------------------------------------------------------------------------- -// Some significant cuda device properties: -// -// cudaDeviceProp::name : Text label for device -// cudaDeviceProp::major : Device major number -// cudaDeviceProp::minor : Device minor number -// cudaDeviceProp::warpSize : number of threads per warp -// cudaDeviceProp::multiProcessorCount : number of multiprocessors -// cudaDeviceProp::sharedMemPerBlock : capacity of shared memory per block -// cudaDeviceProp::totalConstMem : capacity of constant memory -// cudaDeviceProp::totalGlobalMem : capacity of global memory -// cudaDeviceProp::maxGridSize[3] : maximum grid size - -// -// Section 4.4.2.4 of the CUDA Toolkit Reference Manual -// -// struct cudaDeviceProp { -// char name[256]; -// size_t totalGlobalMem; -// size_t sharedMemPerBlock; -// int regsPerBlock; -// int warpSize; -// size_t memPitch; -// int maxThreadsPerBlock; -// int maxThreadsDim[3]; -// int maxGridSize[3]; -// size_t totalConstMem; -// int major; -// int minor; -// int clockRate; -// size_t textureAlignment; -// int deviceOverlap; -// int multiProcessorCount; -// int kernelExecTimeoutEnabled; -// int integrated; -// int canMapHostMemory; -// int computeMode; -// int concurrentKernels; -// int ECCEnabled; -// int pciBusID; -// int pciDeviceID; -// int tccDriver; -// int asyncEngineCount; -// int unifiedAddressing; -// int memoryClockRate; -// int memoryBusWidth; -// int l2CacheSize; -// int maxThreadsPerMultiProcessor; -// }; - -namespace { - -class CudaInternalDevices { - public: - enum { MAXIMUM_DEVICE_COUNT = 64 }; - struct cudaDeviceProp m_cudaProp[MAXIMUM_DEVICE_COUNT]; - int m_cudaDevCount; - - CudaInternalDevices(); - - static const CudaInternalDevices &singleton(); -}; - -CudaInternalDevices::CudaInternalDevices() { - // See 'cudaSetDeviceFlags' for host-device thread interaction - // Section 4.4.2.6 of the CUDA Toolkit Reference Manual - - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaGetDeviceCount(&m_cudaDevCount)); - - if (m_cudaDevCount > MAXIMUM_DEVICE_COUNT) { - Kokkos::abort( - "Sorry, you have more GPUs per node than we thought anybody would ever " - "have. Please report this to github.com/kokkos/kokkos."); - } - for (int i = 0; i < m_cudaDevCount; ++i) { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaGetDeviceProperties(m_cudaProp + i, i)); - } -} - -const CudaInternalDevices &CudaInternalDevices::singleton() { - static CudaInternalDevices self; - return self; -} - -} // namespace - -unsigned long *CudaInternal::constantMemHostStaging = nullptr; -cudaEvent_t CudaInternal::constantMemReusable = nullptr; -std::mutex CudaInternal::constantMemMutex; - -//---------------------------------------------------------------------------- - -void CudaInternal::print_configuration(std::ostream &s) const { - const CudaInternalDevices &dev_info = CudaInternalDevices::singleton(); - -#if defined(KOKKOS_ENABLE_CUDA) - s << "macro KOKKOS_ENABLE_CUDA : defined\n"; -#endif -#if defined(CUDA_VERSION) - s << "macro CUDA_VERSION = " << CUDA_VERSION << " = version " - << CUDA_VERSION / 1000 << "." << (CUDA_VERSION % 1000) / 10 << '\n'; -#endif - - for (int i = 0; i < dev_info.m_cudaDevCount; ++i) { - s << "Kokkos::Cuda[ " << i << " ] " << dev_info.m_cudaProp[i].name - << " capability " << dev_info.m_cudaProp[i].major << "." - << dev_info.m_cudaProp[i].minor << ", Total Global Memory: " - << human_memory_size(dev_info.m_cudaProp[i].totalGlobalMem) - << ", Shared Memory per Block: " - << human_memory_size(dev_info.m_cudaProp[i].sharedMemPerBlock); - if (m_cudaDev == i) s << " : Selected"; - s << std::endl; - } -} - -//---------------------------------------------------------------------------- - -CudaInternal::~CudaInternal() { - if (m_stream || m_scratchSpace || m_scratchFlags || m_scratchUnified) { - std::cerr << "Kokkos::Cuda ERROR: Failed to call Kokkos::Cuda::finalize()" - << std::endl; - } - - m_cudaDev = -1; - m_cudaArch = -1; - m_multiProcCount = 0; - m_maxWarpCount = 0; - m_maxBlock = {0, 0, 0}; - m_maxSharedWords = 0; - m_maxConcurrency = 0; - m_scratchSpaceCount = 0; - m_scratchFlagsCount = 0; - m_scratchUnifiedCount = 0; - m_scratchUnifiedSupported = 0; - m_streamCount = 0; - m_scratchSpace = nullptr; - m_scratchFlags = nullptr; - m_scratchUnified = nullptr; - m_stream = nullptr; - for (int i = 0; i < m_n_team_scratch; ++i) { - m_team_scratch_current_size[i] = 0; - m_team_scratch_ptr[i] = nullptr; - } -} - -int CudaInternal::verify_is_initialized(const char *const label) const { - if (m_cudaDev < 0) { - Kokkos::abort((std::string("Kokkos::Cuda::") + label + - " : ERROR device not initialized\n") - .c_str()); - } - return 0 <= m_cudaDev; -} -uint32_t CudaInternal::impl_get_instance_id() const { return m_instance_id; } -CudaInternal &CudaInternal::singleton() { - static CudaInternal self; - return self; -} -void CudaInternal::fence(const std::string &name) const { - Impl::cuda_stream_synchronize(m_stream, this, name); -} -void CudaInternal::fence() const { - fence("Kokkos::CudaInternal::fence(): Unnamed Instance Fence"); -} - -void CudaInternal::initialize(int cuda_device_id, cudaStream_t stream, - bool manage_stream) { - if (was_finalized) - Kokkos::abort("Calling Cuda::initialize after Cuda::finalize is illegal\n"); - was_initialized = true; - if (is_initialized()) return; - - enum { WordSize = sizeof(size_type) }; - -#ifndef KOKKOS_IMPL_TURN_OFF_CUDA_HOST_INIT_CHECK - if (!HostSpace::execution_space::impl_is_initialized()) { - const std::string msg( - "Cuda::initialize ERROR : HostSpace::execution_space is not " - "initialized"); - throw_runtime_exception(msg); - } -#endif - - const CudaInternalDevices &dev_info = CudaInternalDevices::singleton(); - - const bool ok_init = nullptr == m_scratchSpace || nullptr == m_scratchFlags; - - const bool ok_id = - 0 <= cuda_device_id && cuda_device_id < dev_info.m_cudaDevCount; - - // Need device capability 3.0 or better - - const bool ok_dev = - ok_id && (3 <= dev_info.m_cudaProp[cuda_device_id].major && - 0 <= dev_info.m_cudaProp[cuda_device_id].minor); - - if (ok_init && ok_dev) { - const struct cudaDeviceProp &cudaProp = dev_info.m_cudaProp[cuda_device_id]; - - m_cudaDev = cuda_device_id; - m_deviceProp = cudaProp; - - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaSetDevice(m_cudaDev)); - Kokkos::Impl::cuda_device_synchronize( - "Kokkos::CudaInternal::initialize: Fence on space initialization"); - - // Query what compute capability architecture a kernel executes: - m_cudaArch = cuda_kernel_arch(); - - if (m_cudaArch == 0) { - std::stringstream ss; - ss << "Kokkos::Cuda::initialize ERROR: likely mismatch of architecture\n"; - std::string msg = ss.str(); - Kokkos::abort(msg.c_str()); - } - - int compiled_major = m_cudaArch / 100; - int compiled_minor = (m_cudaArch % 100) / 10; - - if (compiled_major != cudaProp.major || compiled_minor > cudaProp.minor) { - std::stringstream ss; - ss << "Kokkos::Cuda::initialize ERROR: running kernels compiled for " - "compute capability " - << compiled_major << "." << compiled_minor - << " on device with compute capability " << cudaProp.major << "." - << cudaProp.minor << " is not supported by CUDA!\n"; - std::string msg = ss.str(); - Kokkos::abort(msg.c_str()); - } - if (Kokkos::show_warnings() && (compiled_major != cudaProp.major || - compiled_minor != cudaProp.minor)) { - std::cerr << "Kokkos::Cuda::initialize WARNING: running kernels compiled " - "for compute capability " - << compiled_major << "." << compiled_minor - << " on device with compute capability " << cudaProp.major - << "." << cudaProp.minor - << " , this will likely reduce potential performance." - << std::endl; - } - - // number of multiprocessors - - m_multiProcCount = cudaProp.multiProcessorCount; - - //---------------------------------- - // Maximum number of warps, - // at most one warp per thread in a warp for reduction. - - m_maxWarpCount = cudaProp.maxThreadsPerBlock / Impl::CudaTraits::WarpSize; - - if (Impl::CudaTraits::WarpSize < m_maxWarpCount) { - m_maxWarpCount = Impl::CudaTraits::WarpSize; - } - - m_maxSharedWords = cudaProp.sharedMemPerBlock / WordSize; - - //---------------------------------- - // Maximum number of blocks: - - m_maxBlock[0] = cudaProp.maxGridSize[0]; - m_maxBlock[1] = cudaProp.maxGridSize[1]; - m_maxBlock[2] = cudaProp.maxGridSize[2]; - - m_shmemPerSM = cudaProp.sharedMemPerMultiprocessor; - m_maxShmemPerBlock = cudaProp.sharedMemPerBlock; - m_regsPerSM = cudaProp.regsPerMultiprocessor; - m_maxBlocksPerSM = - m_cudaArch < 500 - ? 16 - : (m_cudaArch < 750 ? 32 : (m_cudaArch == 750 ? 16 : 32)); - m_maxThreadsPerSM = cudaProp.maxThreadsPerMultiProcessor; - m_maxThreadsPerBlock = cudaProp.maxThreadsPerBlock; - - //---------------------------------- - - m_scratchUnifiedSupported = cudaProp.unifiedAddressing; - - if (Kokkos::show_warnings() && !m_scratchUnifiedSupported) { - std::cerr << "Kokkos::Cuda device " << cudaProp.name << " capability " - << cudaProp.major << "." << cudaProp.minor - << " does not support unified virtual address space" - << std::endl; - } - - //---------------------------------- - // Multiblock reduction uses scratch flags for counters - // and scratch space for partial reduction values. - // Allocate some initial space. This will grow as needed. - - { - const unsigned reduce_block_count = - m_maxWarpCount * Impl::CudaTraits::WarpSize; - - (void)scratch_unified(16 * sizeof(size_type)); - (void)scratch_flags(reduce_block_count * 2 * sizeof(size_type)); - (void)scratch_space(reduce_block_count * 16 * sizeof(size_type)); - } - //---------------------------------- - // Concurrent bitset for obtaining unique tokens from within - // an executing kernel. - { - m_maxConcurrency = m_maxThreadsPerSM * cudaProp.multiProcessorCount; - - const int32_t buffer_bound = - Kokkos::Impl::concurrent_bitset::buffer_bound(m_maxConcurrency); - - // Allocate and initialize uint32_t[ buffer_bound ] - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - Record *const r = - Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchBitset", - sizeof(uint32_t) * buffer_bound); - - Record::increment(r); - } - //---------------------------------- - - } else { - std::ostringstream msg; - msg << "Kokkos::Cuda::initialize(" << cuda_device_id << ") FAILED"; - - if (!ok_init) { - msg << " : Already initialized"; - } - if (!ok_id) { - msg << " : Device identifier out of range " - << "[0.." << dev_info.m_cudaDevCount << "]"; - } else if (!ok_dev) { - msg << " : Device "; - msg << dev_info.m_cudaProp[cuda_device_id].major; - msg << "."; - msg << dev_info.m_cudaProp[cuda_device_id].minor; - msg << " has insufficient capability, required 3.0 or better"; - } - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - -#ifdef KOKKOS_ENABLE_CUDA_UVM - const char *env_force_device_alloc = - getenv("CUDA_MANAGED_FORCE_DEVICE_ALLOC"); - bool force_device_alloc; - if (env_force_device_alloc == nullptr) - force_device_alloc = false; - else - force_device_alloc = std::stoi(env_force_device_alloc) != 0; - - const char *env_visible_devices = getenv("CUDA_VISIBLE_DEVICES"); - bool visible_devices_one = true; - if (env_visible_devices == nullptr) visible_devices_one = false; - - if (Kokkos::show_warnings() && - (!visible_devices_one && !force_device_alloc)) { - std::cerr << R"warning( -Kokkos::Cuda::initialize WARNING: Cuda is allocating into UVMSpace by default - without setting CUDA_MANAGED_FORCE_DEVICE_ALLOC=1 or - setting CUDA_VISIBLE_DEVICES. - This could on multi GPU systems lead to severe performance" - penalties.)warning" - << std::endl; - } -#endif - -#ifdef KOKKOS_ENABLE_PRE_CUDA_10_DEPRECATION_API - cudaThreadSetCacheConfig(cudaFuncCachePreferShared); -#else - cudaDeviceSetCacheConfig(cudaFuncCachePreferShared); -#endif - - // Init the array for used for arbitrarily sized atomics - if (stream == nullptr) Impl::initialize_host_cuda_lock_arrays(); - - // Allocate a staging buffer for constant mem in pinned host memory - // and an event to avoid overwriting driver for previous kernel launches - if (stream == nullptr) { - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMallocHost(reinterpret_cast(&constantMemHostStaging), - CudaTraits::ConstantMemoryUsage)); - - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaEventCreate(&constantMemReusable)); - } - - m_stream = stream; - m_manage_stream = manage_stream; - for (int i = 0; i < m_n_team_scratch; ++i) { - m_team_scratch_current_size[i] = 0; - m_team_scratch_ptr[i] = nullptr; - } - - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMalloc(&m_scratch_locks, sizeof(int32_t) * m_maxConcurrency)); - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMemset(m_scratch_locks, 0, sizeof(int32_t) * m_maxConcurrency)); -} - -//---------------------------------------------------------------------------- - -using ScratchGrain = Cuda::size_type[Impl::CudaTraits::WarpSize]; -enum { sizeScratchGrain = sizeof(ScratchGrain) }; - -Cuda::size_type *CudaInternal::scratch_flags(const std::size_t size) const { - if (verify_is_initialized("scratch_flags") && - m_scratchFlagsCount * sizeScratchGrain < size) { - m_scratchFlagsCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - if (m_scratchFlags) Record::decrement(Record::get_record(m_scratchFlags)); - - Record *const r = - Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchFlags", - (sizeof(ScratchGrain) * m_scratchFlagsCount)); - - Record::increment(r); - - m_scratchFlags = reinterpret_cast(r->data()); - - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMemset(m_scratchFlags, 0, m_scratchFlagsCount * sizeScratchGrain)); - } - - return m_scratchFlags; -} - -Cuda::size_type *CudaInternal::scratch_space(const std::size_t size) const { - if (verify_is_initialized("scratch_space") && - m_scratchSpaceCount * sizeScratchGrain < size) { - m_scratchSpaceCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - if (m_scratchSpace) Record::decrement(Record::get_record(m_scratchSpace)); - - Record *const r = - Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchSpace", - (sizeof(ScratchGrain) * m_scratchSpaceCount)); - - Record::increment(r); - - m_scratchSpace = reinterpret_cast(r->data()); - } - - return m_scratchSpace; -} - -Cuda::size_type *CudaInternal::scratch_unified(const std::size_t size) const { - if (verify_is_initialized("scratch_unified") && m_scratchUnifiedSupported && - m_scratchUnifiedCount * sizeScratchGrain < size) { - m_scratchUnifiedCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - if (m_scratchUnified) - Record::decrement(Record::get_record(m_scratchUnified)); - - Record *const r = Record::allocate( - Kokkos::CudaHostPinnedSpace(), "Kokkos::InternalScratchUnified", - (sizeof(ScratchGrain) * m_scratchUnifiedCount)); - - Record::increment(r); - - m_scratchUnified = reinterpret_cast(r->data()); - } - - return m_scratchUnified; -} - -Cuda::size_type *CudaInternal::scratch_functor(const std::size_t size) const { - if (verify_is_initialized("scratch_functor") && m_scratchFunctorSize < size) { - m_scratchFunctorSize = size; - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - if (m_scratchFunctor) - Record::decrement(Record::get_record(m_scratchFunctor)); - - Record *const r = - Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchFunctor", - m_scratchFunctorSize); - - Record::increment(r); - - m_scratchFunctor = reinterpret_cast(r->data()); - } - - return m_scratchFunctor; -} - -std::pair CudaInternal::resize_team_scratch_space( - std::int64_t bytes, bool force_shrink) { - // Multiple ParallelFor/Reduce Teams can call this function at the same time - // and invalidate the m_team_scratch_ptr. We use a pool to avoid any race - // condition. - - int current_team_scratch = 0; - int zero = 0; - int one = 1; - while (!m_team_scratch_pool[current_team_scratch].compare_exchange_weak( - zero, one, std::memory_order_release, std::memory_order_relaxed)) { - current_team_scratch = (current_team_scratch + 1) % m_n_team_scratch; - } - if (m_team_scratch_current_size[current_team_scratch] == 0) { - m_team_scratch_current_size[current_team_scratch] = bytes; - m_team_scratch_ptr[current_team_scratch] = - Kokkos::kokkos_malloc( - "Kokkos::CudaSpace::TeamScratchMemory", - m_team_scratch_current_size[current_team_scratch]); - } - if ((bytes > m_team_scratch_current_size[current_team_scratch]) || - ((bytes < m_team_scratch_current_size[current_team_scratch]) && - (force_shrink))) { - m_team_scratch_current_size[current_team_scratch] = bytes; - m_team_scratch_ptr[current_team_scratch] = - Kokkos::kokkos_realloc( - m_team_scratch_ptr[current_team_scratch], - m_team_scratch_current_size[current_team_scratch]); - } - return std::make_pair(m_team_scratch_ptr[current_team_scratch], - current_team_scratch); -} - -//---------------------------------------------------------------------------- - -void CudaInternal::finalize() { - // skip if finalize() has already been called - if (was_finalized) return; - - was_finalized = true; - - // Only finalize this if we're the singleton - if (this == &singleton()) { - (void)Impl::cuda_global_unique_token_locks(true); - Impl::finalize_host_cuda_lock_arrays(); - - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFreeHost(constantMemHostStaging)); - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaEventDestroy(constantMemReusable)); - auto &deep_copy_space = - Kokkos::Impl::cuda_get_deep_copy_space(/*initialize*/ false); - if (deep_copy_space) - deep_copy_space->impl_internal_space_instance()->finalize(); - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaStreamDestroy(cuda_get_deep_copy_stream())); - } - - if (nullptr != m_scratchSpace || nullptr != m_scratchFlags) { - using RecordCuda = Kokkos::Impl::SharedAllocationRecord; - using RecordHost = - Kokkos::Impl::SharedAllocationRecord; - - RecordCuda::decrement(RecordCuda::get_record(m_scratchFlags)); - RecordCuda::decrement(RecordCuda::get_record(m_scratchSpace)); - RecordHost::decrement(RecordHost::get_record(m_scratchUnified)); - if (m_scratchFunctorSize > 0) - RecordCuda::decrement(RecordCuda::get_record(m_scratchFunctor)); - } - - for (int i = 0; i < m_n_team_scratch; ++i) { - if (m_team_scratch_current_size[i] > 0) - Kokkos::kokkos_free(m_team_scratch_ptr[i]); - } - - if (m_manage_stream && m_stream != nullptr) - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaStreamDestroy(m_stream)); - - m_cudaDev = -1; - m_multiProcCount = 0; - m_maxWarpCount = 0; - m_maxBlock = {0, 0, 0}; - m_maxSharedWords = 0; - m_scratchSpaceCount = 0; - m_scratchFlagsCount = 0; - m_scratchUnifiedCount = 0; - m_streamCount = 0; - m_scratchSpace = nullptr; - m_scratchFlags = nullptr; - m_scratchUnified = nullptr; - m_stream = nullptr; - for (int i = 0; i < m_n_team_scratch; ++i) { - m_team_scratch_current_size[i] = 0; - m_team_scratch_ptr[i] = nullptr; - } - - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaFree(m_scratch_locks)); - m_scratch_locks = nullptr; -} - -//---------------------------------------------------------------------------- - -Cuda::size_type cuda_internal_multiprocessor_count() { - return CudaInternal::singleton().m_multiProcCount; -} - -CudaSpace::size_type cuda_internal_maximum_concurrent_block_count() { -#if defined(KOKKOS_ARCH_KEPLER) - // Compute capability 3.0 through 3.7 - enum : int { max_resident_blocks_per_multiprocessor = 16 }; -#else - // Compute capability 5.0 through 6.2 - enum : int { max_resident_blocks_per_multiprocessor = 32 }; -#endif - return CudaInternal::singleton().m_multiProcCount * - max_resident_blocks_per_multiprocessor; -}; - -Cuda::size_type cuda_internal_maximum_warp_count() { - return CudaInternal::singleton().m_maxWarpCount; -} - -std::array cuda_internal_maximum_grid_count() { - return CudaInternal::singleton().m_maxBlock; -} - -Cuda::size_type cuda_internal_maximum_shared_words() { - return CudaInternal::singleton().m_maxSharedWords; -} - -Cuda::size_type *cuda_internal_scratch_space(const Cuda &instance, - const std::size_t size) { - return instance.impl_internal_space_instance()->scratch_space(size); -} - -Cuda::size_type *cuda_internal_scratch_flags(const Cuda &instance, - const std::size_t size) { - return instance.impl_internal_space_instance()->scratch_flags(size); -} - -Cuda::size_type *cuda_internal_scratch_unified(const Cuda &instance, - const std::size_t size) { - return instance.impl_internal_space_instance()->scratch_unified(size); -} - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -Cuda::size_type Cuda::detect_device_count() { - return Impl::CudaInternalDevices::singleton().m_cudaDevCount; -} - -int Cuda::concurrency() { - return Impl::CudaInternal::singleton().m_maxConcurrency; -} - -int Cuda::impl_is_initialized() { - return Impl::CudaInternal::singleton().is_initialized(); -} - -void Cuda::impl_initialize(InitializationSettings const &settings) { - Impl::CudaInternal::singleton().initialize(Impl::get_gpu(settings)); - - // In order to support setting an atexit hook for Kokkos::finalize - // We need to ensure that the Cuda deep_copy instance is not destroyed - // before that atexit hook is getting called. - // Thus we create the static instance here, so that it will be deallocated - // after the potential atexit call. - // This is neccessary since we will access that instance in Kokkos::finalize - (void)::Kokkos::Impl::cuda_get_deep_copy_space(true); -} - -std::vector Cuda::detect_device_arch() { - const Impl::CudaInternalDevices &s = Impl::CudaInternalDevices::singleton(); - - std::vector output(s.m_cudaDevCount); - - for (int i = 0; i < s.m_cudaDevCount; ++i) { - output[i] = s.m_cudaProp[i].major * 100 + s.m_cudaProp[i].minor; - } - - return output; -} - -Cuda::size_type Cuda::device_arch() { - const int dev_id = Impl::CudaInternal::singleton().m_cudaDev; - - int dev_arch = 0; - - if (0 <= dev_id) { - const struct cudaDeviceProp &cudaProp = - Impl::CudaInternalDevices::singleton().m_cudaProp[dev_id]; - - dev_arch = cudaProp.major * 100 + cudaProp.minor; - } - - return dev_arch; -} - -void Cuda::impl_finalize() { Impl::CudaInternal::singleton().finalize(); } - -Cuda::Cuda() - : m_space_instance(&Impl::CudaInternal::singleton(), - [](Impl::CudaInternal *) {}) { - Impl::CudaInternal::singleton().verify_is_initialized( - "Cuda instance constructor"); -} - -Cuda::Cuda(cudaStream_t stream, bool manage_stream) - : m_space_instance(new Impl::CudaInternal, [](Impl::CudaInternal *ptr) { - ptr->finalize(); - delete ptr; - }) { - Impl::CudaInternal::singleton().verify_is_initialized( - "Cuda instance constructor"); - m_space_instance->initialize(Impl::CudaInternal::singleton().m_cudaDev, - stream, manage_stream); -} - -void Cuda::print_configuration(std::ostream &os, bool /*verbose*/) const { - os << "Device Execution Space:\n"; - os << " KOKKOS_ENABLE_CUDA: yes\n"; - - os << "Cuda Atomics:\n"; - os << " KOKKOS_ENABLE_CUDA_ATOMICS: "; -#ifdef KOKKOS_ENABLE_CUDA_ATOMICS - os << "yes\n"; -#else - os << "no\n"; -#endif - - os << "Cuda Options:\n"; - os << " KOKKOS_ENABLE_CUDA_LAMBDA: "; -#ifdef KOKKOS_ENABLE_CUDA_LAMBDA - os << "yes\n"; -#else - os << "no\n"; -#endif - os << " KOKKOS_ENABLE_CUDA_LDG_INTRINSIC: "; -#ifdef KOKKOS_ENABLE_CUDA_LDG_INTRINSIC - os << "yes\n"; -#else - os << "no\n"; -#endif - os << " KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE: "; -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE - os << "yes\n"; -#else - os << "no\n"; -#endif - os << " KOKKOS_ENABLE_CUDA_UVM: "; -#ifdef KOKKOS_ENABLE_CUDA_UVM - os << "yes\n"; -#else - os << "no\n"; -#endif - os << " KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA: "; -#ifdef KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA - os << "yes\n"; -#else - os << "no\n"; -#endif - - os << "\nCuda Runtime Configuration:\n"; - - m_space_instance->print_configuration(os); -} - -void Cuda::impl_static_fence(const std::string &name) { - Kokkos::Impl::cuda_device_synchronize(name); -} - -void Cuda::fence(const std::string &name) const { - m_space_instance->fence(name); -} - -const char *Cuda::name() { return "Cuda"; } -uint32_t Cuda::impl_instance_id() const noexcept { - return m_space_instance->impl_get_instance_id(); -} - -cudaStream_t Cuda::cuda_stream() const { return m_space_instance->m_stream; } -int Cuda::cuda_device() const { return m_space_instance->m_cudaDev; } -const cudaDeviceProp &Cuda::cuda_device_prop() const { - return m_space_instance->m_deviceProp; -} - -namespace Impl { - -int g_cuda_space_factory_initialized = - initialize_space_factory("150_Cuda"); - -} // namespace Impl - -#ifdef KOKKOS_ENABLE_CXX14 -namespace Tools { -namespace Experimental { -constexpr DeviceType DeviceTypeTraits::id; -} -} // namespace Tools -#endif - -} // namespace Kokkos - -#else - -void KOKKOS_CORE_SRC_CUDA_IMPL_PREVENT_LINK_ERROR() {} - -#endif // KOKKOS_ENABLE_CUDA diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Instance.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Instance.hpp deleted file mode 100644 index 62b1f09cd5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Instance.hpp +++ /dev/null @@ -1,251 +0,0 @@ -#ifndef KOKKOS_CUDA_INSTANCE_HPP_ -#define KOKKOS_CUDA_INSTANCE_HPP_ - -#include -#include -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -// These functions fulfill the purpose of allowing to work around -// a suspected system software issue, or to check for race conditions. -// They are not currently a fully officially supported capability. -#ifdef KOKKOS_IMPL_DEBUG_CUDA_SERIAL_EXECUTION -extern "C" void kokkos_impl_cuda_set_serial_execution(bool); -extern "C" bool kokkos_impl_cuda_use_serial_execution(); -#endif - -namespace Kokkos { -namespace Impl { - -struct CudaTraits { - static constexpr CudaSpace::size_type WarpSize = 32 /* 0x0020 */; - static constexpr CudaSpace::size_type WarpIndexMask = - 0x001f; /* Mask for warpindex */ - static constexpr CudaSpace::size_type WarpIndexShift = - 5; /* WarpSize == 1 << WarpShift */ - - static constexpr CudaSpace::size_type ConstantMemoryUsage = - 0x008000; /* 32k bytes */ - static constexpr CudaSpace::size_type ConstantMemoryCache = - 0x002000; /* 8k bytes */ - static constexpr CudaSpace::size_type KernelArgumentLimit = - 0x001000; /* 4k bytes */ - static constexpr CudaSpace::size_type MaxHierarchicalParallelism = - 1024; /* team_size * vector_length */ - using ConstantGlobalBufferType = - unsigned long[ConstantMemoryUsage / sizeof(unsigned long)]; - - static constexpr int ConstantMemoryUseThreshold = 0x000200 /* 512 bytes */; - - KOKKOS_INLINE_FUNCTION static CudaSpace::size_type warp_count( - CudaSpace::size_type i) { - return (i + WarpIndexMask) >> WarpIndexShift; - } - - KOKKOS_INLINE_FUNCTION static CudaSpace::size_type warp_align( - CudaSpace::size_type i) { - constexpr CudaSpace::size_type Mask = ~WarpIndexMask; - return (i + WarpIndexMask) & Mask; - } -}; - -//---------------------------------------------------------------------------- - -CudaSpace::size_type cuda_internal_multiprocessor_count(); -CudaSpace::size_type cuda_internal_maximum_warp_count(); -std::array cuda_internal_maximum_grid_count(); -CudaSpace::size_type cuda_internal_maximum_shared_words(); - -CudaSpace::size_type cuda_internal_maximum_concurrent_block_count(); - -CudaSpace::size_type* cuda_internal_scratch_flags(const Cuda&, - const std::size_t size); -CudaSpace::size_type* cuda_internal_scratch_space(const Cuda&, - const std::size_t size); -CudaSpace::size_type* cuda_internal_scratch_unified(const Cuda&, - const std::size_t size); - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -namespace Kokkos { -namespace Impl { - -class CudaInternal { - private: - CudaInternal(const CudaInternal&); - CudaInternal& operator=(const CudaInternal&); -#ifdef KOKKOS_IMPL_DEBUG_CUDA_SERIAL_EXECUTION - static bool kokkos_impl_cuda_use_serial_execution_v; -#endif - - public: - using size_type = Cuda::size_type; - - int m_cudaDev; - - // Device Properties - int m_cudaArch; - unsigned m_multiProcCount; - unsigned m_maxWarpCount; - std::array m_maxBlock; - unsigned m_maxSharedWords; - uint32_t m_maxConcurrency; - int m_shmemPerSM; - int m_maxShmemPerBlock; - int m_regsPerSM; - int m_maxBlocksPerSM; - int m_maxThreadsPerSM; - int m_maxThreadsPerBlock; - - cudaDeviceProp m_deviceProp; - - // Scratch Spaces for Reductions - mutable std::size_t m_scratchSpaceCount; - mutable std::size_t m_scratchFlagsCount; - mutable std::size_t m_scratchUnifiedCount; - mutable std::size_t m_scratchFunctorSize; - - size_type m_scratchUnifiedSupported; - size_type m_streamCount; - mutable size_type* m_scratchSpace; - mutable size_type* m_scratchFlags; - mutable size_type* m_scratchUnified; - mutable size_type* m_scratchFunctor; - cudaStream_t m_stream; - uint32_t m_instance_id; - bool m_manage_stream; - - // Team Scratch Level 1 Space - int m_n_team_scratch = 10; - mutable int64_t m_team_scratch_current_size[10]; - mutable void* m_team_scratch_ptr[10]; - mutable std::atomic_int m_team_scratch_pool[10]; - std::int32_t* m_scratch_locks; - - bool was_initialized = false; - bool was_finalized = false; - - // FIXME_CUDA: these want to be per-device, not per-stream... use of 'static' - // here will break once there are multiple devices though - static unsigned long* constantMemHostStaging; - static cudaEvent_t constantMemReusable; - static std::mutex constantMemMutex; - - static CudaInternal& singleton(); - - int verify_is_initialized(const char* const label) const; - - int is_initialized() const { - return nullptr != m_scratchSpace && nullptr != m_scratchFlags; - } - - void initialize(int cuda_device_id, cudaStream_t stream = nullptr, - bool manage_stream = false); - void finalize(); - - void print_configuration(std::ostream&) const; - -#ifdef KOKKOS_IMPL_DEBUG_CUDA_SERIAL_EXECUTION - static bool cuda_use_serial_execution(); - static void cuda_set_serial_execution(bool); -#endif - - void fence(const std::string&) const; - void fence() const; - - ~CudaInternal(); - - CudaInternal() - : m_cudaDev(-1), - m_cudaArch(-1), - m_multiProcCount(0), - m_maxWarpCount(0), - m_maxBlock({0, 0, 0}), - m_maxSharedWords(0), - m_maxConcurrency(0), - m_shmemPerSM(0), - m_maxShmemPerBlock(0), - m_regsPerSM(0), - m_maxBlocksPerSM(0), - m_maxThreadsPerSM(0), - m_maxThreadsPerBlock(0), - m_scratchSpaceCount(0), - m_scratchFlagsCount(0), - m_scratchUnifiedCount(0), - m_scratchFunctorSize(0), - m_scratchUnifiedSupported(0), - m_streamCount(0), - m_scratchSpace(nullptr), - m_scratchFlags(nullptr), - m_scratchUnified(nullptr), - m_scratchFunctor(nullptr), - m_stream(nullptr), - m_instance_id( - Kokkos::Tools::Experimental::Impl::idForInstance( - reinterpret_cast(this))) { - for (int i = 0; i < m_n_team_scratch; ++i) { - m_team_scratch_current_size[i] = 0; - m_team_scratch_ptr[i] = nullptr; - m_team_scratch_pool[i] = 0; - } - } - - // Resizing of reduction related scratch spaces - size_type* scratch_space(const std::size_t size) const; - size_type* scratch_flags(const std::size_t size) const; - size_type* scratch_unified(const std::size_t size) const; - size_type* scratch_functor(const std::size_t size) const; - uint32_t impl_get_instance_id() const; - // Resizing of team level 1 scratch - std::pair resize_team_scratch_space(std::int64_t bytes, - bool force_shrink = false); -}; - -} // Namespace Impl - -namespace Experimental { -// Partitioning an Execution Space: expects space and integer arguments for -// relative weight -// Customization point for backends -// Default behavior is to return the passed in instance - -namespace Impl { -inline void create_Cuda_instances(std::vector& instances) { - for (int s = 0; s < int(instances.size()); s++) { - cudaStream_t stream; - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaStreamCreate(&stream)); - instances[s] = Cuda(stream, true); - } -} -} // namespace Impl - -template -std::vector partition_space(const Cuda&, Args...) { -#ifdef __cpp_fold_expressions - static_assert( - (... && std::is_arithmetic_v), - "Kokkos Error: partitioning arguments must be integers or floats"); -#endif - std::vector instances(sizeof...(Args)); - Impl::create_Cuda_instances(instances); - return instances; -} - -template -std::vector partition_space(const Cuda&, std::vector& weights) { - static_assert( - std::is_arithmetic::value, - "Kokkos Error: partitioning arguments must be integers or floats"); - - std::vector instances(weights.size()); - Impl::create_Cuda_instances(instances); - return instances; -} -} // namespace Experimental - -} // Namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Locks.cpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Locks.cpp deleted file mode 100644 index 3796534816..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Locks.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#ifdef KOKKOS_ENABLE_CUDA -#include -#include - -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE -namespace Kokkos { -namespace Impl { -__device__ __constant__ CudaLockArrays g_device_cuda_lock_arrays = {nullptr, 0}; -} -} // namespace Kokkos -#endif - -namespace Kokkos { - -namespace { - -__global__ void init_lock_array_kernel_atomic() { - unsigned i = blockIdx.x * blockDim.x + threadIdx.x; - if (i < CUDA_SPACE_ATOMIC_MASK + 1) { - Kokkos::Impl::g_device_cuda_lock_arrays.atomic[i] = 0; - } -} - -} // namespace - -namespace Impl { - -CudaLockArrays g_host_cuda_lock_arrays = {nullptr, 0}; - -void initialize_host_cuda_lock_arrays() { -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS - desul::Impl::init_lock_arrays(); - desul::ensure_cuda_lock_arrays_on_device(); -#endif - if (g_host_cuda_lock_arrays.atomic != nullptr) return; - KOKKOS_IMPL_CUDA_SAFE_CALL( - cudaMalloc(&g_host_cuda_lock_arrays.atomic, - sizeof(int) * (CUDA_SPACE_ATOMIC_MASK + 1))); - Impl::cuda_device_synchronize( - "Kokkos::Impl::initialize_host_cuda_lock_arrays: Pre Init Lock Arrays"); - g_host_cuda_lock_arrays.n = Cuda::concurrency(); - copy_cuda_lock_arrays_to_device(); - init_lock_array_kernel_atomic<<<(CUDA_SPACE_ATOMIC_MASK + 1 + 255) / 256, - 256>>>(); - Impl::cuda_device_synchronize( - "Kokkos::Impl::initialize_host_cuda_lock_arrays: Post Init Lock Arrays"); -} - -void finalize_host_cuda_lock_arrays() { -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS - desul::Impl::finalize_lock_arrays(); -#endif - - if (g_host_cuda_lock_arrays.atomic == nullptr) return; - cudaFree(g_host_cuda_lock_arrays.atomic); - g_host_cuda_lock_arrays.atomic = nullptr; - g_host_cuda_lock_arrays.n = 0; -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE - copy_cuda_lock_arrays_to_device(); -#endif -} - -} // namespace Impl - -} // namespace Kokkos - -#else - -void KOKKOS_CORE_SRC_CUDA_CUDA_LOCKS_PREVENT_LINK_ERROR() {} - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Locks.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Locks.hpp deleted file mode 100644 index 244f142f0d..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Locks.hpp +++ /dev/null @@ -1,197 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_LOCKS_HPP -#define KOKKOS_CUDA_LOCKS_HPP - -#include - -#ifdef KOKKOS_ENABLE_CUDA - -#include - -#include - -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#include -#endif - -namespace Kokkos { -namespace Impl { - -struct CudaLockArrays { - std::int32_t* atomic; - std::int32_t n; -}; - -/// \brief This global variable in Host space is the central definition -/// of these arrays. -extern CudaLockArrays g_host_cuda_lock_arrays; - -/// \brief After this call, the g_host_cuda_lock_arrays variable has -/// valid, initialized arrays. -/// -/// This call is idempotent. -void initialize_host_cuda_lock_arrays(); - -/// \brief After this call, the g_host_cuda_lock_arrays variable has -/// all null pointers, and all array memory has been freed. -/// -/// This call is idempotent. -void finalize_host_cuda_lock_arrays(); - -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -/// \brief This global variable in CUDA space is what kernels use -/// to get access to the lock arrays. -/// -/// When relocatable device code is enabled, there can be one single -/// instance of this global variable for the entire executable, -/// whose definition will be in Kokkos_Cuda_Locks.cpp (and whose declaration -/// here must then be extern. -/// This one instance will be initialized by initialize_host_cuda_lock_arrays -/// and need not be modified afterwards. -/// -/// When relocatable device code is disabled, an instance of this variable -/// will be created in every translation unit that sees this header file -/// (we make this clear by marking it static, meaning no other translation -/// unit can link to it). -/// Since the Kokkos_Cuda_Locks.cpp translation unit cannot initialize the -/// instances in other translation units, we must update this CUDA global -/// variable based on the Host global variable prior to running any kernels -/// that will use it. -/// That is the purpose of the ensure_cuda_lock_arrays_on_device function. -__device__ -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE - __constant__ extern -#endif - CudaLockArrays g_device_cuda_lock_arrays; - -#define CUDA_SPACE_ATOMIC_MASK 0x1FFFF - -/// \brief Acquire a lock for the address -/// -/// This function tries to acquire the lock for the hash value derived -/// from the provided ptr. If the lock is successfully acquired the -/// function returns true. Otherwise it returns false. -__device__ inline bool lock_address_cuda_space(void* ptr) { - size_t offset = size_t(ptr); - offset = offset >> 2; - offset = offset & CUDA_SPACE_ATOMIC_MASK; - return (0 == atomicCAS(&g_device_cuda_lock_arrays.atomic[offset], 0, 1)); -} - -/// \brief Release lock for the address -/// -/// This function releases the lock for the hash value derived -/// from the provided ptr. This function should only be called -/// after previously successfully acquiring a lock with -/// lock_address. -__device__ inline void unlock_address_cuda_space(void* ptr) { - size_t offset = size_t(ptr); - offset = offset >> 2; - offset = offset & CUDA_SPACE_ATOMIC_MASK; - atomicExch(&g_device_cuda_lock_arrays.atomic[offset], 0); -} - -} // namespace Impl -} // namespace Kokkos - -// Make lock_array_copied an explicit translation unit scope thingy -namespace Kokkos { -namespace Impl { -namespace { -static int lock_array_copied = 0; -inline int eliminate_warning_for_lock_array() { return lock_array_copied; } -} // namespace - -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE -inline -#else -static -#endif - void - copy_cuda_lock_arrays_to_device() { - if (lock_array_copied == 0) { - KOKKOS_IMPL_CUDA_SAFE_CALL(cudaMemcpyToSymbol(g_device_cuda_lock_arrays, - &g_host_cuda_lock_arrays, - sizeof(CudaLockArrays))); - } - lock_array_copied = 1; -} - -#ifndef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS - -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE -inline void ensure_cuda_lock_arrays_on_device() {} -#else -inline static void ensure_cuda_lock_arrays_on_device() { - copy_cuda_lock_arrays_to_device(); -} -#endif - -#else - -#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE -inline void ensure_cuda_lock_arrays_on_device() {} -#else -// Still Need COPY_CUDA_LOCK_ARRAYS for team scratch etc. -inline static void ensure_cuda_lock_arrays_on_device() { - copy_cuda_lock_arrays_to_device(); - desul::ensure_cuda_lock_arrays_on_device(); -} -#endif - -#endif /* defined( KOKKOS_ENABLE_IMPL_DESUL_ATOMICS ) */ - -} // namespace Impl -} // namespace Kokkos - -#endif /* defined( KOKKOS_ENABLE_CUDA ) */ - -#endif /* #ifndef KOKKOS_CUDA_LOCKS_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_MDRangePolicy.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_MDRangePolicy.hpp deleted file mode 100644 index 12b7f70a97..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_MDRangePolicy.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef KOKKOS_CUDA_MDRANGEPOLICY_HPP_ -#define KOKKOS_CUDA_MDRANGEPOLICY_HPP_ - -#include - -namespace Kokkos { - -template <> -struct default_outer_direction { - using type = Iterate; - static constexpr Iterate value = Iterate::Left; -}; - -template <> -struct default_inner_direction { - using type = Iterate; - static constexpr Iterate value = Iterate::Left; -}; - -namespace Impl { - -// Settings for MDRangePolicy -template <> -inline TileSizeProperties get_tile_size_properties( - const Kokkos::Cuda& space) { - TileSizeProperties properties; - properties.max_threads = - space.impl_internal_space_instance()->m_maxThreadsPerSM; - properties.default_largest_tile_size = 16; - properties.default_tile_size = 2; - properties.max_total_tile_size = 512; - return properties; -} - -} // Namespace Impl -} // Namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Task.cpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Task.cpp deleted file mode 100644 index 8f05448b17..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_Task.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#if defined(KOKKOS_ENABLE_CUDA) && defined(KOKKOS_ENABLE_TASKDAG) - -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template class TaskQueue< - Kokkos::Cuda, - Impl::default_tasking_memory_space_for_execution_space_t >; -template class TaskQueueMultiple< - Kokkos::Cuda, - Impl::default_tasking_memory_space_for_execution_space_t >; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -#else -void KOKKOS_CORE_SRC_CUDA_KOKKOS_CUDA_TASK_PREVENT_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_CUDA ) && defined( KOKKOS_ENABLE_TASKDAG \ - ) */ diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_View.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_View.hpp deleted file mode 100644 index a175820825..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_View.hpp +++ /dev/null @@ -1,303 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_EXPERIMENTAL_CUDA_VIEW_HPP -#define KOKKOS_EXPERIMENTAL_CUDA_VIEW_HPP - -#include -#if defined(KOKKOS_ENABLE_CUDA) - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -// Cuda Texture fetches can be performed for 4, 8 and 16 byte objects -// (int,int2,int4) Via reinterpret_case this can be used to support all scalar -// types of those sizes. Any other scalar type falls back to either normal reads -// out of global memory, or using the __ldg intrinsic on Kepler GPUs or newer -// (Compute Capability >= 3.0) - -template -struct CudaTextureFetch { - ::cudaTextureObject_t m_obj; - const ValueType* m_ptr; - int m_offset; - - // Deference operator pulls through texture object and returns by value - template - KOKKOS_INLINE_FUNCTION ValueType operator[](const iType& i) const { - KOKKOS_IF_ON_DEVICE( - (AliasType v = tex1Dfetch(m_obj, i + m_offset); - return *(reinterpret_cast(&v));)) - KOKKOS_IF_ON_HOST((return m_ptr[i];)) - } - - // Pointer to referenced memory - KOKKOS_INLINE_FUNCTION - operator const ValueType*() const { return m_ptr; } - - KOKKOS_INLINE_FUNCTION - CudaTextureFetch() : m_obj(), m_ptr(), m_offset() {} - - KOKKOS_DEFAULTED_FUNCTION - ~CudaTextureFetch() = default; - - KOKKOS_INLINE_FUNCTION - CudaTextureFetch(const CudaTextureFetch& rhs) - : m_obj(rhs.m_obj), m_ptr(rhs.m_ptr), m_offset(rhs.m_offset) {} - - KOKKOS_INLINE_FUNCTION - CudaTextureFetch(CudaTextureFetch&& rhs) - : m_obj(rhs.m_obj), m_ptr(rhs.m_ptr), m_offset(rhs.m_offset) {} - - KOKKOS_INLINE_FUNCTION - CudaTextureFetch& operator=(const CudaTextureFetch& rhs) { - m_obj = rhs.m_obj; - m_ptr = rhs.m_ptr; - m_offset = rhs.m_offset; - return *this; - } - - KOKKOS_INLINE_FUNCTION - CudaTextureFetch& operator=(CudaTextureFetch&& rhs) { - m_obj = rhs.m_obj; - m_ptr = rhs.m_ptr; - m_offset = rhs.m_offset; - return *this; - } - - // Texture object spans the entire allocation. - // This handle may view a subset of the allocation, so an offset is required. - template - inline explicit CudaTextureFetch( - const ValueType* const arg_ptr, - Kokkos::Impl::SharedAllocationRecord* record) - : m_obj(record->template attach_texture_object()), - m_ptr(arg_ptr), - m_offset(record->attach_texture_object_offset( - reinterpret_cast(arg_ptr))) {} - - // Texture object spans the entire allocation. - // This handle may view a subset of the allocation, so an offset is required. - KOKKOS_INLINE_FUNCTION - CudaTextureFetch(const CudaTextureFetch& rhs, size_t offset) - : m_obj(rhs.m_obj), - m_ptr(rhs.m_ptr + offset), - m_offset(offset + rhs.m_offset) {} -}; - -#if defined(KOKKOS_ENABLE_CUDA_LDG_INTRINSIC) - -template -struct CudaLDGFetch { - const ValueType* m_ptr; - - template - KOKKOS_INLINE_FUNCTION ValueType operator[](const iType& i) const { -#if defined(KOKKOS_ARCH_KEPLER30) || defined(KOKKOS_ARCH_KEPLER32) - return m_ptr[i]; -#else - KOKKOS_IF_ON_DEVICE( - (AliasType v = __ldg(reinterpret_cast(&m_ptr[i])); - return *(reinterpret_cast(&v));)) - KOKKOS_IF_ON_HOST((return m_ptr[i];)) -#endif - } - - KOKKOS_INLINE_FUNCTION - operator const ValueType*() const { return m_ptr; } - - KOKKOS_INLINE_FUNCTION - CudaLDGFetch() : m_ptr() {} - - KOKKOS_DEFAULTED_FUNCTION - ~CudaLDGFetch() = default; - - KOKKOS_INLINE_FUNCTION - CudaLDGFetch(const CudaLDGFetch& rhs) : m_ptr(rhs.m_ptr) {} - - KOKKOS_INLINE_FUNCTION - CudaLDGFetch(CudaLDGFetch&& rhs) : m_ptr(rhs.m_ptr) {} - - KOKKOS_INLINE_FUNCTION - CudaLDGFetch& operator=(const CudaLDGFetch& rhs) { - m_ptr = rhs.m_ptr; - return *this; - } - - KOKKOS_INLINE_FUNCTION - CudaLDGFetch& operator=(CudaLDGFetch&& rhs) { - m_ptr = rhs.m_ptr; - return *this; - } - - template - inline explicit CudaLDGFetch( - const ValueType* const arg_ptr, - Kokkos::Impl::SharedAllocationRecord*) - : m_ptr(arg_ptr) {} - - KOKKOS_INLINE_FUNCTION - CudaLDGFetch(CudaLDGFetch const rhs, size_t offset) - : m_ptr(rhs.m_ptr + offset) {} -}; - -#endif - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -/** \brief Replace Default ViewDataHandle with Cuda texture fetch - * specialization if 'const' value type, CudaSpace and random access. - */ -template -class ViewDataHandle< - Traits, std::enable_if_t<( - // Is Cuda memory space - (std::is_same::value || - std::is_same::value) && - // Is a trivial const value of 4, 8, or 16 bytes - std::is_trivial::value && - std::is_same::value && - (sizeof(typename Traits::const_value_type) == 4 || - sizeof(typename Traits::const_value_type) == 8 || - sizeof(typename Traits::const_value_type) == 16) && - // Random access trait - (Traits::memory_traits::is_random_access != 0))>> { - public: - using track_type = Kokkos::Impl::SharedAllocationTracker; - - using value_type = typename Traits::const_value_type; - using return_type = typename Traits::const_value_type; // NOT a reference - - using alias_type = std::conditional_t< - (sizeof(value_type) == 4), int, - std::conditional_t< - (sizeof(value_type) == 8), ::int2, - std::conditional_t<(sizeof(value_type) == 16), ::int4, void>>>; - -#if defined(KOKKOS_ENABLE_CUDA_LDG_INTRINSIC) - using handle_type = Kokkos::Impl::CudaLDGFetch; -#else - using handle_type = Kokkos::Impl::CudaTextureFetch; -#endif - - KOKKOS_INLINE_FUNCTION - static handle_type const& assign(handle_type const& arg_handle, - track_type const& /* arg_tracker */) { - return arg_handle; - } - - KOKKOS_INLINE_FUNCTION - static handle_type const assign(handle_type const& arg_handle, - size_t offset) { - return handle_type(arg_handle, offset); - } - - KOKKOS_INLINE_FUNCTION - static handle_type assign(value_type* arg_data_ptr, - track_type const& arg_tracker) { - if (arg_data_ptr == nullptr) return handle_type(); - -#if !defined(KOKKOS_ENABLE_CUDA_LDG_INTRINSIC) - KOKKOS_IF_ON_HOST(( - // Assignment of texture = non-texture requires creation of a texture - // object which can only occur on the host. In addition, 'get_record' - // is only valid if called in a host execution space - - using memory_space = typename Traits::memory_space; - using record = - typename Impl::SharedAllocationRecord; - - record* const r = arg_tracker.template get_record(); - - if (0 == r) { - Kokkos::abort( - "Cuda const random access View using Cuda texture memory " - "requires " - "Kokkos to allocate the View's memory"); - } - - return handle_type(arg_data_ptr, r);)) -#else - KOKKOS_IF_ON_HOST(( - // Assignment of texture = non-texture requires creation of a texture - // object which can only occur on the host. In addition, 'get_record' - // is only valid if called in a host execution space - - using memory_space = typename Traits::memory_space; - using record = - typename Impl::SharedAllocationRecord; - - record* const r = arg_tracker.template get_record(); - - return handle_type(arg_data_ptr, r);)) -#endif - - KOKKOS_IF_ON_DEVICE( - ((void)arg_tracker; Kokkos::Impl::cuda_abort( - "Cannot create Cuda texture object from within a Cuda kernel"); - return handle_type();)) - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ -#endif /* #ifndef KOKKOS_CUDA_VIEW_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_abort.hpp b/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_abort.hpp deleted file mode 100644 index c0daa274f8..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Cuda/Kokkos_Cuda_abort.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_ABORT_HPP -#define KOKKOS_CUDA_ABORT_HPP - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -#include -#if defined(KOKKOS_ENABLE_CUDA) - -#include - -extern "C" { -/* Cuda runtime function, declared in - * Requires capability 2.x or better. - */ -extern __device__ void __assertfail(const void *message, const void *file, - unsigned int line, const void *function, - size_t charsize); -} - -namespace Kokkos { -namespace Impl { - -#if !defined(__APPLE__) -// required to workaround failures in random number generator unit tests with -// pre-volta architectures -#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) -__device__ inline void cuda_abort(const char *const message) { -#else -[[noreturn]] __device__ inline void cuda_abort(const char *const message) { -#endif - const char empty[] = ""; - - __assertfail((const void *)message, (const void *)empty, (unsigned int)0, - (const void *)empty, sizeof(char)); - - // This loop is never executed. It's intended to suppress warnings that the - // function returns, even though it does not. This is necessary because - // __assertfail is not marked as [[noreturn]], even though it does not return. - // Disable with KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK to workaround failures - // in random number generator unit tests with pre-volta architectures -#if !defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) - while (true) - ; -#endif -} -#else -__device__ inline void cuda_abort(const char *const message) { - // __assertfail is not supported on MAC -} -#endif - -} // namespace Impl -} // namespace Kokkos -#else -void KOKKOS_CORE_SRC_CUDA_ABORT_PREVENT_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ -#endif /* #ifndef KOKKOS_CUDA_ABORT_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Abort.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Abort.hpp deleted file mode 100644 index dcc5863721..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Abort.hpp +++ /dev/null @@ -1,85 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_ABORT_HPP -#define KOKKOS_HIP_ABORT_HPP - -#include -#if defined(KOKKOS_ENABLE_HIP) - -#include - -// FIXME_HIP ROCm 4.5 version header include would be -#if __has_include() -#include -#define KOKKOS_IMPL_ROCM_VERSION \ - ROCM_VERSION_MAJOR * 10000 + ROCM_VERSION_MINOR * 100 + ROCM_VERSION_PATCH -#endif - -// FIXME_HIP workaround for ROCm version less than 5.0.2 -#if KOKKOS_IMPL_ROCM_VERSION < 50002 -#define KOKKOS_IMPL_HIP_ABORT_DOES_NOT_PRINT_MESSAGE -#endif - -namespace Kokkos { -namespace Impl { - -// The two keywords below are not contradictory. `noinline` is a -// directive to the optimizer. -[[noreturn]] __device__ __attribute__((noinline)) inline void hip_abort( - char const *msg) { - const char empty[] = ""; - __assert_fail(msg, empty, 0, empty); - // This loop is never executed. It's intended to suppress warnings that the - // function returns, even though it does not. This is necessary because - // abort() is not marked as [[noreturn]], even though it does not return. - while (true) - ; -} - -} // namespace Impl -} // namespace Kokkos - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Atomic.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Atomic.hpp deleted file mode 100644 index 88bcab6264..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Atomic.hpp +++ /dev/null @@ -1,618 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_ATOMIC_HPP -#define KOKKOS_HIP_ATOMIC_HPP - -#include -#include -#include - -#if defined(KOKKOS_ENABLE_HIP_ATOMICS) -namespace Kokkos { -// HIP can do: -// Types int/unsigned int -// variants: -// atomic_exchange/compare_exchange/fetch_add/fetch_sub/fetch_max/fetch_min/fetch_and/fetch_or/fetch_xor/fetch_inc/fetch_dec - -// atomic_exchange ------------------------------------------------------------- - -__inline__ __device__ int atomic_exchange(volatile int *const dest, - const int val) { - return atomicExch(const_cast(dest), val); -} - -__inline__ __device__ unsigned int atomic_exchange( - volatile unsigned int *const dest, const unsigned int val) { - return atomicExch(const_cast(dest), val); -} - -__inline__ __device__ unsigned long long int atomic_exchange( - volatile unsigned long long int *const dest, - const unsigned long long int val) { - return atomicExch(const_cast(dest), val); -} - -__inline__ __device__ float atomic_exchange(volatile float *const dest, - const float val) { - return atomicExch(const_cast(dest), val); -} - -template -__inline__ __device__ T -atomic_exchange(volatile T *const dest, - std::enable_if_t val) { - int tmp = atomicExch(reinterpret_cast(const_cast(dest)), - *reinterpret_cast(const_cast(&val))); - return reinterpret_cast(tmp); -} - -template -__inline__ __device__ T atomic_exchange( - volatile T *const dest, - std::enable_if_t - val) { - using type = unsigned long long int; - - type tmp = atomicExch(reinterpret_cast(const_cast(dest)), - *reinterpret_cast(const_cast(&val))); - return reinterpret_cast(tmp); -} - -template -__inline__ __device__ T atomic_exchange( - volatile T *const dest, - std::enable_if_t &val) { - T return_val; - int done = 0; - unsigned int active = __ballot(1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip_space((void *)dest)) { - return_val = *dest; - *dest = val; - Impl::unlock_address_hip_space((void *)dest); - done = 1; - } - } - done_active = __ballot(done); - } - return return_val; -} - -// atomic_assign --------------------------------------------------------------- - -template -__inline__ __device__ void atomic_assign( - volatile T *const dest, - std::enable_if_t val) { - atomicExch(reinterpret_cast(const_cast(dest)), - *reinterpret_cast(const_cast(&val))); -} - -template -__inline__ __device__ void atomic_assign( - volatile T *const dest, - std::enable_if_t - val) { - using type = unsigned long long int; - atomicExch(reinterpret_cast(const_cast(dest)), - *reinterpret_cast(const_cast(&val))); -} - -template -__inline__ __device__ void atomic_assign( - volatile T *const dest, - std::enable_if_t - val) { - atomic_exchange(dest, val); -} - -// atomic_compare_exchange ----------------------------------------------------- - -inline __device__ int atomic_compare_exchange(volatile int *dest, int compare, - const int &val) { - return atomicCAS(const_cast(dest), compare, val); -} - -inline __device__ unsigned int atomic_compare_exchange( - volatile unsigned int *dest, unsigned int compare, - const unsigned int &val) { - return atomicCAS(const_cast(dest), compare, val); -} - -inline __device__ unsigned long long int atomic_compare_exchange( - volatile unsigned long long int *dest, unsigned long long int compare, - const unsigned long long int &val) { - return atomicCAS(const_cast(dest), compare, val); -} - -template -__inline__ __device__ T atomic_compare_exchange( - volatile T *dest, T compare, - std::enable_if_t val) { - // FIXME_HIP UB - union U { - int i; - T f; - __inline__ __device__ U() {} - } idest, icompare, ival; - icompare.f = compare; - ival.f = val; - idest.i = atomicCAS(reinterpret_cast(const_cast(dest)), - icompare.i, ival.i); - return idest.f; -} - -template -__inline__ __device__ T atomic_compare_exchange( - volatile T *dest, T compare, - std::enable_if_t - val) { - // FIXME_HIP UB - union U { - unsigned long long int i; - T f; - __inline__ __device__ U() {} - } idest, icompare, ival; - icompare.f = compare; - ival.f = val; - idest.i = atomicCAS( - reinterpret_cast(const_cast(dest)), - icompare.i, ival.i); - return idest.f; -} - -template -__inline__ __device__ T atomic_compare_exchange( - volatile T *const dest, const T &compare, - std::enable_if_t &val) { - T return_val; - int done = 0; - unsigned int active = __ballot(1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip_space((void *)dest)) { - return_val = *dest; - if (return_val == compare) *dest = val; - Impl::unlock_address_hip_space((void *)dest); - done = 1; - } - } - done_active = __ballot(done); - } - return return_val; -} - -// atomic_fetch_add ------------------------------------------------------------ - -inline __device__ int atomic_fetch_add(volatile int *dest, const int &val) { - return atomicAdd(const_cast(dest), val); -} - -inline __device__ unsigned int atomic_fetch_add(volatile unsigned int *dest, - const unsigned int &val) { - return atomicAdd(const_cast(dest), val); -} - -inline __device__ unsigned long long atomic_fetch_add( - volatile unsigned long long *dest, const unsigned long long &val) { - return atomicAdd(const_cast(dest), val); -} - -inline __device__ float atomic_fetch_add(volatile float *dest, - const float &val) { - return atomicAdd(const_cast(dest), val); -} - -template -inline __device__ T -atomic_fetch_add(volatile T *const dest, - std::enable_if_t val) { - // FIXME_HIP UB - union U { - int i; - T t; - __inline__ __device__ U() {} - } assume, oldval, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = atomicCAS(reinterpret_cast(const_cast(dest)), - assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -inline __device__ T atomic_fetch_add( - volatile T *const dest, - std::enable_if_t val) { - // FIXME_HIP UB - union U { - unsigned long long i; - T t; - __inline__ __device__ U() {} - } assume, oldval, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = atomic_compare_exchange( - reinterpret_cast(dest), assume.i, - newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -__inline__ __device__ char atomic_fetch_add(volatile char *dest, - const char &val) { - unsigned int oldval, newval, assume; - oldval = *reinterpret_cast(&dest); - - do { - assume = oldval; - newval = assume & 0x7fffff00 + ((assume & 0xff) + val) & 0xff; - oldval = - atomicCAS(reinterpret_cast(const_cast(dest)), - assume, newval); - } while (assume != oldval); - - return oldval; -} - -__inline__ __device__ short atomic_fetch_add(volatile short *dest, - const short &val) { - unsigned int oldval, newval, assume; - oldval = *reinterpret_cast(&dest); - - do { - assume = oldval; - newval = assume & 0x7fff0000 + ((assume & 0xffff) + val) & 0xffff; - oldval = - atomicCAS(reinterpret_cast(const_cast(dest)), - assume, newval); - } while (assume != oldval); - - return oldval; -} - -__inline__ __device__ long long atomic_fetch_add(volatile long long *dest, - const long long &val) { - return atomicAdd( - reinterpret_cast(const_cast(dest)), - val); -} - -template -__inline__ __device__ T atomic_fetch_add( - volatile T *dest, - std::enable_if_t - val) { - T return_val; - int done = 0; - unsigned int active = __ballot(1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Kokkos::Impl::lock_address_hip_space((void *)dest)) { - return_val = *dest; - *dest = return_val + val; - Kokkos::Impl::unlock_address_hip_space((void *)dest); - done = 1; - } - } - done_active = __ballot(done); - } - return return_val; -} - -// atmic_fetch_sub ------------------------------------------------------------- - -__inline__ __device__ int atomic_fetch_sub(volatile int *dest, int const &val) { - return atomicSub(const_cast(dest), val); -} - -__inline__ __device__ unsigned int atomic_fetch_sub(volatile unsigned int *dest, - unsigned int const &val) { - return atomicSub(const_cast(dest), val); -} - -__inline__ __device__ unsigned long long atomic_fetch_sub( - unsigned long long *dest, int64_t const &val) { - return atomicAdd(reinterpret_cast(dest), - -reinterpret_cast(val)); -} - -__inline__ __device__ char atomic_fetch_sub(volatile char *dest, - const char &val) { - unsigned int oldval, newval, assume; - oldval = *reinterpret_cast(dest); - - do { - assume = oldval; - newval = assume & 0x7fffff00 + ((assume & 0xff) - val) & 0xff; - oldval = - atomicCAS(reinterpret_cast(const_cast(dest)), - assume, newval); - } while (assume != oldval); - - return oldval; -} - -__inline__ __device__ short atomic_fetch_sub(volatile short *dest, - const short &val) { - unsigned int oldval, newval, assume; - oldval = *reinterpret_cast(dest); - - do { - assume = oldval; - newval = assume & 0x7fff0000 + ((assume & 0xffff) - val) & 0xffff; - oldval = - atomicCAS(reinterpret_cast(const_cast(dest)), - assume, newval); - } while (assume != oldval); - - return oldval; -} - -__inline__ __device__ long long atomic_fetch_sub(volatile long long *dest, - const long long &val) { - return static_cast(atomicAdd( - reinterpret_cast(const_cast(dest)), - -reinterpret_cast(val))); -} - -template -__inline__ __device__ T atomic_fetch_sub( - volatile T *dest, std::enable_if_t val) { - // FIXME_HIP UB - union U { - int i; - T t; - __inline__ __device__ U() {} - } assume, oldval, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t - val; - oldval.i = atomic_compare_exchange(reinterpret_cast(dest), - assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -inline __device__ T atomic_fetch_sub( - volatile T *const dest, - std::enable_if_t val) { - // FIXME_HIP UB - union U { - unsigned long long i; - T t; - __inline__ __device__ U() {} - } assume, oldval, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t - val; - oldval.i = atomic_compare_exchange( - reinterpret_cast(dest), assume.i, - newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -__inline__ __device__ T atomic_fetch_sub( - volatile T *dest, std::enable_if_t val) { - unsigned int oldval, newval, assume; - oldval = *reinterpret_cast(dest); - - do { - assume = oldval; - newval = assume & 0x7fffff00 + ((assume & 0xff) - val) & 0xff; - oldval = atomicCAS(reinterpret_cast(dest), assume, newval); - } while (assume != oldval); - - return reinterpret_cast(oldval) & 0xff; -} - -template -__inline__ __device__ T atomic_fetch_sub( - volatile T *dest, std::enable_if_t val) { - unsigned int oldval, newval, assume; - oldval = *reinterpret_cast(dest); - - do { - assume = oldval; - newval = assume & 0x7fff0000 + ((assume & 0xffff) - val) & 0xffff; - oldval = atomicCAS(reinterpret_cast(dest), assume, newval); - } while (assume != oldval); - - return reinterpret_cast(oldval) & 0xffff; -} - -template -__inline__ __device__ T atomic_fetch_sub( - volatile T *const dest, - std::enable_if_t &val) { - T return_val; - int done = 0; - unsigned int active = __ballot(1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip_space((void *)dest)) { - return_val = *dest; - *dest = return_val - val; - Impl::unlock_address_hip_space((void *)dest); - done = 1; - } - } - done_active = __ballot(done); - } - return return_val; -} - -// atomic_fetch_or ------------------------------------------------------------- - -__inline__ __device__ int atomic_fetch_or(volatile int *const dest, - int const val) { - return atomicOr(const_cast(dest), val); -} - -__inline__ __device__ unsigned int atomic_fetch_or( - volatile unsigned int *const dest, unsigned int const val) { - return atomicOr(const_cast(dest), val); -} - -__inline__ __device__ unsigned long long int atomic_fetch_or( - volatile unsigned long long int *const dest, - unsigned long long int const val) { - return atomicOr(const_cast(dest), val); -} - -// atomic_fetch_and ------------------------------------------------------------ - -__inline__ __device__ int atomic_fetch_and(volatile int *const dest, - int const val) { - return atomicAnd(const_cast(dest), val); -} - -__inline__ __device__ unsigned int atomic_fetch_and( - volatile unsigned int *const dest, unsigned int const val) { - return atomicAnd(const_cast(dest), val); -} - -__inline__ __device__ unsigned long long int atomic_fetch_and( - volatile unsigned long long int *const dest, - unsigned long long int const val) { - return atomicAnd(const_cast(dest), val); -} - -namespace Impl { - -template -__inline__ __device__ void _atomic_store(T *ptr, T val, - memory_order_relaxed_t) { - (void)atomic_exchange(ptr, val); -} - -template -__inline__ __device__ void _atomic_store(T *ptr, T val, - memory_order_seq_cst_t) { - memory_fence(); - atomic_store(ptr, val, memory_order_relaxed); - memory_fence(); -} - -template -__inline__ __device__ void _atomic_store(T *ptr, T val, - memory_order_release_t) { - memory_fence(); - atomic_store(ptr, val, memory_order_relaxed); -} - -template -__inline__ __device__ void _atomic_store(T *ptr, T val) { - atomic_store(ptr, val, memory_order_relaxed); -} - -template -__inline__ __device__ T _atomic_load(T *ptr, memory_order_relaxed_t) { - T dummy{}; - return atomic_compare_exchange(ptr, dummy, dummy); -} - -template -__inline__ __device__ T _atomic_load(T *ptr, memory_order_seq_cst_t) { - memory_fence(); - T rv = atomic_load(ptr, memory_order_relaxed); - memory_fence(); - return rv; -} - -template -__inline__ __device__ T _atomic_load(T *ptr, memory_order_acquire_t) { - T rv = atomic_load(ptr, memory_order_relaxed); - memory_fence(); - return rv; -} - -template -__inline__ __device__ T _atomic_load(T *ptr) { - return atomic_load(ptr, memory_order_relaxed); -} - -} // namespace Impl -} // namespace Kokkos -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Error.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Error.hpp deleted file mode 100644 index a75e7a4a6c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Error.hpp +++ /dev/null @@ -1,128 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_ERROR_HPP -#define KOKKOS_HIP_ERROR_HPP - -#include -#include - -#include - -#include - -namespace Kokkos { -namespace Impl { - -void hip_internal_error_throw(hipError_t e, const char* name, - const char* file = nullptr, const int line = 0); - -inline void hip_internal_safe_call(hipError_t e, const char* name, - const char* file = nullptr, - const int line = 0) { - if (hipSuccess != e) { - hip_internal_error_throw(e, name, file, line); - } -} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - -KOKKOS_DEPRECATED -inline void hip_internal_safe_call_deprecated(hipError_t e, const char* name, - const char* file = nullptr, - const int line = 0) { - hip_internal_safe_call(e, name, file, line); -} - -#endif - -} // namespace Impl -} // namespace Kokkos - -#define KOKKOS_IMPL_HIP_SAFE_CALL(call) \ - Kokkos::Impl::hip_internal_safe_call(call, #call, __FILE__, __LINE__) - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -#define HIP_SAFE_CALL(call) \ - Kokkos::Impl::hip_internal_safe_call_deprecated(call, #call, __FILE__, \ - __LINE__) - -#endif - -namespace Kokkos { -namespace Experimental { - -class HIPRawMemoryAllocationFailure : public RawMemoryAllocationFailure { - private: - hipError_t m_error_code = hipSuccess; - - static FailureMode get_failure_mode(hipError_t error_code) { - switch (error_code) { - case hipErrorMemoryAllocation: return FailureMode::OutOfMemoryError; - case hipErrorInvalidValue: return FailureMode::InvalidAllocationSize; - default: return FailureMode::Unknown; - } - } - - public: - HIPRawMemoryAllocationFailure(size_t arg_attempted_size, - hipError_t arg_error_code, - AllocationMechanism arg_mechanism) noexcept - : RawMemoryAllocationFailure( - arg_attempted_size, /* HIPSpace doesn't handle alignment? */ 1, - get_failure_mode(arg_error_code), arg_mechanism), - m_error_code(arg_error_code) {} - - void append_additional_error_information(std::ostream& o) const override { - if (m_error_code != hipSuccess) { - o << " The HIP allocation returned the error code \"\"" - << hipGetErrorName(m_error_code) << "\"."; - } - } -}; - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Half_Impl_Type.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Half_Impl_Type.hpp deleted file mode 100644 index 728a229c1a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Half_Impl_Type.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_HALF_IMPL_TYPE_HPP_ -#define KOKKOS_HIP_HALF_IMPL_TYPE_HPP_ - -#include - -#ifndef KOKKOS_IMPL_HALF_TYPE_DEFINED -// Make sure no one else tries to define half_t -#define KOKKOS_IMPL_HALF_TYPE_DEFINED -#define KOKKOS_IMPL_HIP_HALF_TYPE_DEFINED - -namespace Kokkos { -namespace Impl { -struct half_impl_t { - using type = __half; -}; -} // namespace Impl -} // namespace Kokkos -#endif // KOKKOS_IMPL_HALF_TYPE_DEFINED -#endif // KOKKOS_ENABLE_HIP diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Instance.cpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Instance.cpp deleted file mode 100644 index 3785cfe80b..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Instance.cpp +++ /dev/null @@ -1,569 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/*--------------------------------------------------------------------------*/ -/* Kokkos interfaces */ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include -#include - -/*--------------------------------------------------------------------------*/ -/* Standard 'C' libraries */ -#include - -/* Standard 'C++' libraries */ -#include -#include -#include -#include - -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE -__device__ __constant__ unsigned long kokkos_impl_hip_constant_memory_buffer - [Kokkos::Experimental::Impl::HIPTraits::ConstantMemoryUsage / - sizeof(unsigned long)]; -#endif - -namespace Kokkos { -namespace Impl { -Kokkos::View -hip_global_unique_token_locks(bool deallocate) { - static Kokkos::View locks = - Kokkos::View(); - if (!deallocate && locks.extent(0) == 0) - locks = Kokkos::View( - "Kokkos::UniqueToken::m_locks", - Kokkos::Experimental::HIP().concurrency()); - if (deallocate) - locks = Kokkos::View(); - return locks; -} -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Experimental { -namespace { -class HIPInternalDevices { - public: - enum { MAXIMUM_DEVICE_COUNT = 64 }; - struct hipDeviceProp_t m_hipProp[MAXIMUM_DEVICE_COUNT]; - int m_hipDevCount; - - HIPInternalDevices(); - - static HIPInternalDevices const &singleton(); -}; - -HIPInternalDevices::HIPInternalDevices() { - KOKKOS_IMPL_HIP_SAFE_CALL(hipGetDeviceCount(&m_hipDevCount)); - - if (m_hipDevCount > MAXIMUM_DEVICE_COUNT) { - Kokkos::abort( - "Sorry, you have more GPUs per node than we thought anybody would ever " - "have. Please report this to github.com/kokkos/kokkos."); - } - for (int i = 0; i < m_hipDevCount; ++i) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipGetDeviceProperties(m_hipProp + i, i)); - } -} - -const HIPInternalDevices &HIPInternalDevices::singleton() { - static HIPInternalDevices self; - return self; -} -} // namespace - -unsigned long *Impl::HIPInternal::constantMemHostStaging = nullptr; -hipEvent_t Impl::HIPInternal::constantMemReusable = nullptr; -std::mutex Impl::HIPInternal::constantMemMutex; - -namespace Impl { - -//---------------------------------------------------------------------------- - -void HIPInternal::print_configuration(std::ostream &s) const { - const HIPInternalDevices &dev_info = HIPInternalDevices::singleton(); - - s << "macro KOKKOS_ENABLE_HIP : defined" << '\n'; -#if defined(HIP_VERSION) - s << "macro HIP_VERSION = " << HIP_VERSION << " = version " - << HIP_VERSION_MAJOR << '.' << HIP_VERSION_MINOR << '.' << HIP_VERSION_PATCH - << '\n'; -#endif - - for (int i = 0; i < dev_info.m_hipDevCount; ++i) { - s << "Kokkos::Experimental::HIP[ " << i << " ] " - << dev_info.m_hipProp[i].name << " version " - << (dev_info.m_hipProp[i].major) << "." << dev_info.m_hipProp[i].minor - << ", Total Global Memory: " - << ::Kokkos::Impl::human_memory_size(dev_info.m_hipProp[i].totalGlobalMem) - << ", Shared Memory per Block: " - << ::Kokkos::Impl::human_memory_size( - dev_info.m_hipProp[i].sharedMemPerBlock); - if (m_hipDev == i) s << " : Selected"; - s << '\n'; - } -} - -//---------------------------------------------------------------------------- - -HIPInternal::~HIPInternal() { - if (m_scratchSpace || m_scratchFlags) { - std::cerr << "Kokkos::Experimental::HIP ERROR: Failed to call " - "Kokkos::Experimental::HIP::finalize()" - << std::endl; - std::cerr.flush(); - } - - m_hipDev = -1; - m_hipArch = -1; - m_multiProcCount = 0; - m_maxWarpCount = 0; - m_maxSharedWords = 0; - m_maxShmemPerBlock = 0; - m_scratchSpaceCount = 0; - m_scratchFlagsCount = 0; - m_scratchSpace = nullptr; - m_scratchFlags = nullptr; - m_stream = nullptr; -} - -int HIPInternal::verify_is_initialized(const char *const label) const { - if (m_hipDev < 0) { - Kokkos::abort((std::string("Kokkos::Experimental::HIP::") + label + - " : ERROR device not initialized\n") - .c_str()); - } - return 0 <= m_hipDev; -} - -uint32_t HIPInternal::impl_get_instance_id() const noexcept { - return m_instance_id; -} -HIPInternal &HIPInternal::singleton() { - static HIPInternal *self = nullptr; - if (!self) { - self = new HIPInternal(); - } - return *self; -} - -void HIPInternal::fence() const { - fence("Kokkos::HIPInternal::fence: Unnamed Internal Fence"); -} -void HIPInternal::fence(const std::string &name) const { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::HIP>( - name, - Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{ - impl_get_instance_id()}, - [&]() { - KOKKOS_IMPL_HIP_SAFE_CALL(hipStreamSynchronize(m_stream)); - // can reset our cycle id now as well - m_cycleId = 0; - }); -} - -void HIPInternal::initialize(int hip_device_id, hipStream_t stream, - bool manage_stream) { - if (was_finalized) - Kokkos::abort("Calling HIP::initialize after HIP::finalize is illegal\n"); - - if (is_initialized()) return; - - int constexpr WordSize = sizeof(size_type); - - if (!HostSpace::execution_space::impl_is_initialized()) { - const std::string msg( - "HIP::initialize ERROR : HostSpace::execution_space " - "is not initialized"); - Kokkos::Impl::throw_runtime_exception(msg); - } - - const HIPInternalDevices &dev_info = HIPInternalDevices::singleton(); - - const bool ok_init = nullptr == m_scratchSpace || nullptr == m_scratchFlags; - - // Need at least a GPU device - const bool ok_id = - 0 <= hip_device_id && hip_device_id < dev_info.m_hipDevCount; - - if (ok_init && ok_id) { - const struct hipDeviceProp_t &hipProp = dev_info.m_hipProp[hip_device_id]; - - m_hipDev = hip_device_id; - m_deviceProp = hipProp; - - KOKKOS_IMPL_HIP_SAFE_CALL(hipSetDevice(m_hipDev)); - - m_stream = stream; - m_manage_stream = manage_stream; - m_team_scratch_current_size = 0; - m_team_scratch_ptr = nullptr; - - // number of multiprocessors - m_multiProcCount = hipProp.multiProcessorCount; - - //---------------------------------- - // Maximum number of warps, - // at most one warp per thread in a warp for reduction. - m_maxWarpCount = hipProp.maxThreadsPerBlock / Impl::HIPTraits::WarpSize; - if (HIPTraits::WarpSize < m_maxWarpCount) { - m_maxWarpCount = Impl::HIPTraits::WarpSize; - } - m_maxSharedWords = hipProp.sharedMemPerBlock / WordSize; - - //---------------------------------- - // Maximum number of blocks - m_maxBlock[0] = hipProp.maxGridSize[0]; - m_maxBlock[1] = hipProp.maxGridSize[1]; - m_maxBlock[2] = hipProp.maxGridSize[2]; - - // theoretically, we can get 40 WF's / CU, but only can sustain 32 - // see - // https://github.com/ROCm-Developer-Tools/HIP/blob/a0b5dfd625d99af7e288629747b40dd057183173/vdi/hip_platform.cpp#L742 - m_maxWavesPerCU = 32; - // FIXME_HIP - Nick to implement this upstream - // Register count comes from Sec. 2.2. "Data Sharing" of the - // Vega 7nm ISA document (see the diagram) - // https://developer.amd.com/wp-content/resources/Vega_7nm_Shader_ISA.pdf - // VGPRS = 4 (SIMD/CU) * 256 VGPR/SIMD * 64 registers / VGPR = - // 65536 VGPR/CU - m_regsPerSM = 65536; - m_shmemPerSM = hipProp.maxSharedMemoryPerMultiProcessor; - m_maxShmemPerBlock = hipProp.sharedMemPerBlock; - m_maxThreadsPerSM = m_maxWavesPerCU * HIPTraits::WarpSize; - //---------------------------------- - // Multiblock reduction uses scratch flags for counters - // and scratch space for partial reduction values. - // Allocate some initial space. This will grow as needed. - { - const unsigned reduce_block_count = - m_maxWarpCount * Impl::HIPTraits::WarpSize; - - (void)scratch_flags(reduce_block_count * 2 * sizeof(size_type)); - (void)scratch_space(reduce_block_count * 16 * sizeof(size_type)); - } - //---------------------------------- - // Concurrent bitset for obtaining unique tokens from within - // an executing kernel. - { - const int32_t buffer_bound = - Kokkos::Impl::concurrent_bitset::buffer_bound(HIP::concurrency()); - - // Allocate and initialize uint32_t[ buffer_bound ] - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - Record *const r = Record::allocate(Kokkos::Experimental::HIPSpace(), - "Kokkos::InternalScratchBitset", - sizeof(uint32_t) * buffer_bound); - - Record::increment(r); - } - //---------------------------------- - - } else { - std::ostringstream msg; - msg << "Kokkos::Experimental::HIP::initialize(" << hip_device_id - << ") FAILED"; - - if (!ok_init) { - msg << " : Already initialized"; - } - if (!ok_id) { - msg << " : Device identifier out of range " - << "[0.." << dev_info.m_hipDevCount - 1 << "]"; - } - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - - // Init the array for used for arbitrarily sized atomics - if (m_stream == nullptr) ::Kokkos::Impl::initialize_host_hip_lock_arrays(); - - // Allocate a staging buffer for constant mem in pinned host memory - // and an event to avoid overwriting driver for previous kernel launches - if (m_stream == nullptr) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipHostMalloc((void **)&constantMemHostStaging, - HIPTraits::ConstantMemoryUsage)); - - KOKKOS_IMPL_HIP_SAFE_CALL(hipEventCreate(&constantMemReusable)); - } - - KOKKOS_IMPL_HIP_SAFE_CALL( - hipMalloc(&m_scratch_locks, sizeof(int32_t) * HIP::concurrency())); - KOKKOS_IMPL_HIP_SAFE_CALL( - hipMemset(m_scratch_locks, 0, sizeof(int32_t) * HIP::concurrency())); -} - -//---------------------------------------------------------------------------- - -using ScratchGrain = - Kokkos::Experimental::HIP::size_type[Impl::HIPTraits::WarpSize]; -enum { sizeScratchGrain = sizeof(ScratchGrain) }; - -Kokkos::Experimental::HIP::size_type *HIPInternal::scratch_space( - const std::size_t size) { - if (verify_is_initialized("scratch_space") && - m_scratchSpaceCount * sizeScratchGrain < size) { - m_scratchSpaceCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - if (m_scratchSpace) Record::decrement(Record::get_record(m_scratchSpace)); - - Record *const r = Record::allocate( - Kokkos::Experimental::HIPSpace(), "Kokkos::InternalScratchSpace", - (sizeScratchGrain * m_scratchSpaceCount)); - - Record::increment(r); - - m_scratchSpace = reinterpret_cast(r->data()); - } - - return m_scratchSpace; -} - -Kokkos::Experimental::HIP::size_type *HIPInternal::scratch_flags( - const std::size_t size) { - if (verify_is_initialized("scratch_flags") && - m_scratchFlagsCount * sizeScratchGrain < size) { - m_scratchFlagsCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = - Kokkos::Impl::SharedAllocationRecord; - - if (m_scratchFlags) Record::decrement(Record::get_record(m_scratchFlags)); - - Record *const r = Record::allocate( - Kokkos::Experimental::HIPSpace(), "Kokkos::InternalScratchFlags", - (sizeScratchGrain * m_scratchFlagsCount)); - - Record::increment(r); - - m_scratchFlags = reinterpret_cast(r->data()); - - KOKKOS_IMPL_HIP_SAFE_CALL( - hipMemset(m_scratchFlags, 0, m_scratchFlagsCount * sizeScratchGrain)); - } - - return m_scratchFlags; -} - -void *HIPInternal::resize_team_scratch_space(std::int64_t bytes, - bool force_shrink) { - if (m_team_scratch_current_size == 0) { - m_team_scratch_current_size = bytes; - m_team_scratch_ptr = Kokkos::kokkos_malloc( - "Kokkos::HIPSpace::TeamScratchMemory", m_team_scratch_current_size); - } - if ((bytes > m_team_scratch_current_size) || - ((bytes < m_team_scratch_current_size) && (force_shrink))) { - m_team_scratch_current_size = bytes; - m_team_scratch_ptr = Kokkos::kokkos_realloc( - m_team_scratch_ptr, m_team_scratch_current_size); - } - return m_team_scratch_ptr; -} - -//---------------------------------------------------------------------------- - -void HIPInternal::finalize() { - this->fence("Kokkos::HIPInternal::finalize: fence on finalization"); - was_finalized = true; - - if (this == &singleton()) { - (void)Kokkos::Impl::hip_global_unique_token_locks(true); - KOKKOS_IMPL_HIP_SAFE_CALL(hipHostFree(constantMemHostStaging)); - KOKKOS_IMPL_HIP_SAFE_CALL(hipEventDestroy(constantMemReusable)); - } - - if (nullptr != m_scratchSpace || nullptr != m_scratchFlags) { - using RecordHIP = - Kokkos::Impl::SharedAllocationRecord; - - RecordHIP::decrement(RecordHIP::get_record(m_scratchFlags)); - RecordHIP::decrement(RecordHIP::get_record(m_scratchSpace)); - - if (m_team_scratch_current_size > 0) - Kokkos::kokkos_free(m_team_scratch_ptr); - - if (m_manage_stream && m_stream != nullptr) - KOKKOS_IMPL_HIP_SAFE_CALL(hipStreamDestroy(m_stream)); - } - - m_hipDev = -1; - m_hipArch = -1; - m_multiProcCount = 0; - m_maxWarpCount = 0; - m_maxBlock = {0, 0, 0}; - m_maxSharedWords = 0; - m_maxShmemPerBlock = 0; - m_scratchSpaceCount = 0; - m_scratchFlagsCount = 0; - m_scratchSpace = nullptr; - m_scratchFlags = nullptr; - m_stream = nullptr; - m_team_scratch_current_size = 0; - m_team_scratch_ptr = nullptr; - - KOKKOS_IMPL_HIP_SAFE_CALL(hipFree(m_scratch_locks)); - m_scratch_locks = nullptr; - - if (nullptr != d_driverWorkArray) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipHostFree(d_driverWorkArray)); - d_driverWorkArray = nullptr; - } -} - -char *HIPInternal::get_next_driver(size_t driverTypeSize) const { - if (d_driverWorkArray == nullptr) { - KOKKOS_IMPL_HIP_SAFE_CALL( - hipHostMalloc(&d_driverWorkArray, - m_maxDriverCycles * m_maxDriverTypeSize * sizeof(char), - hipHostMallocNonCoherent)); - } - if (driverTypeSize > m_maxDriverTypeSize) { - // fence handles the cycle id reset for us - fence( - "Kokkos::HIPInternal::get_next_driver: fence before reallocating " - "resources"); - KOKKOS_IMPL_HIP_SAFE_CALL(hipHostFree(d_driverWorkArray)); - m_maxDriverTypeSize = driverTypeSize; - if (m_maxDriverTypeSize % 128 != 0) - m_maxDriverTypeSize = - m_maxDriverTypeSize + 128 - m_maxDriverTypeSize % 128; - KOKKOS_IMPL_HIP_SAFE_CALL( - hipHostMalloc(&d_driverWorkArray, - m_maxDriverCycles * m_maxDriverTypeSize * sizeof(char), - hipHostMallocNonCoherent)); - } else { - m_cycleId = (m_cycleId + 1) % m_maxDriverCycles; - if (m_cycleId == 0) { - // ensure any outstanding kernels are completed before we wrap around - fence( - "Kokkos::HIPInternal::get_next_driver: fence before reusing first " - "driver"); - } - } - return &d_driverWorkArray[m_maxDriverTypeSize * m_cycleId]; -} - -//---------------------------------------------------------------------------- - -Kokkos::Experimental::HIP::size_type hip_internal_multiprocessor_count() { - return HIPInternal::singleton().m_multiProcCount; -} - -Kokkos::Experimental::HIP::size_type hip_internal_maximum_warp_count() { - return HIPInternal::singleton().m_maxWarpCount; -} - -std::array -hip_internal_maximum_grid_count() { - return HIPInternal::singleton().m_maxBlock; -} - -Kokkos::Experimental::HIP::size_type *hip_internal_scratch_space( - const HIP &instance, const std::size_t size) { - return instance.impl_internal_space_instance()->scratch_space(size); -} - -Kokkos::Experimental::HIP::size_type *hip_internal_scratch_flags( - const HIP &instance, const std::size_t size) { - return instance.impl_internal_space_instance()->scratch_flags(size); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { -void hip_device_synchronize(const std::string &name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::HIP>( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - [&]() { KOKKOS_IMPL_HIP_SAFE_CALL(hipDeviceSynchronize()); }); -} - -void hip_internal_error_throw(hipError_t e, const char *name, const char *file, - const int line) { - std::ostringstream out; - out << name << " error( " << hipGetErrorName(e) - << "): " << hipGetErrorString(e); - if (file) { - out << " " << file << ":" << line; - } - throw_runtime_exception(out.str()); -} -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Experimental { -HIP::size_type HIP::detect_device_count() { - return HIPInternalDevices::singleton().m_hipDevCount; -} -} // namespace Experimental -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Instance.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Instance.hpp deleted file mode 100644 index d8f265834a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Instance.hpp +++ /dev/null @@ -1,221 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/*--------------------------------------------------------------------------*/ - -#ifndef KOKKOS_HIP_INSTANCE_HPP -#define KOKKOS_HIP_INSTANCE_HPP - -#include -#include - -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -struct HIPTraits { - static int constexpr WarpSize = 64; - static int constexpr WarpIndexMask = 0x003f; /* hexadecimal for 63 */ - static int constexpr WarpIndexShift = 6; /* WarpSize == 1 << WarpShift*/ - static int constexpr ConservativeThreadsPerBlock = - 256; // conservative fallback blocksize in case of spills - static int constexpr MaxThreadsPerBlock = - 1024; // the maximum we can fit in a block - static int constexpr ConstantMemoryUsage = 0x008000; /* 32k bytes */ - static int constexpr KernelArgumentLimit = 0x001000; /* 4k bytes */ - static int constexpr ConstantMemoryUseThreshold = 0x000200; /* 512 bytes */ -}; - -//---------------------------------------------------------------------------- - -HIP::size_type hip_internal_maximum_warp_count(); -std::array hip_internal_maximum_grid_count(); -HIP::size_type hip_internal_multiprocessor_count(); - -HIP::size_type *hip_internal_scratch_space(const HIP &instance, - const std::size_t size); -HIP::size_type *hip_internal_scratch_flags(const HIP &instance, - const std::size_t size); - -//---------------------------------------------------------------------------- - -class HIPInternal { - private: - HIPInternal(const HIPInternal &); - HIPInternal &operator=(const HIPInternal &); - - public: - using size_type = ::Kokkos::Experimental::HIP::size_type; - - int m_hipDev = -1; - int m_hipArch = -1; - unsigned m_multiProcCount = 0; - unsigned m_maxWarpCount = 0; - std::array m_maxBlock = {0, 0, 0}; - unsigned m_maxWavesPerCU = 0; - unsigned m_maxSharedWords = 0; - int m_regsPerSM; - int m_shmemPerSM = 0; - int m_maxShmemPerBlock = 0; - int m_maxThreadsPerSM = 0; - - // array of DriverTypes to be allocated in host-pinned memory for async - // kernel launches - mutable char *d_driverWorkArray = nullptr; - // number of kernel launches that can be in-flight w/o synchronization - const int m_maxDriverCycles = 100; - // max size of a DriverType [bytes] - mutable size_t m_maxDriverTypeSize = 1024 * 10; - // the current index in the driverWorkArray - mutable int m_cycleId = 0; - // mutex to access d_driverWorkArray - mutable std::mutex m_mutexWorkArray; - // mutex to access shared memory - mutable std::mutex m_mutexSharedMemory; - - // Scratch Spaces for Reductions - std::size_t m_scratchSpaceCount = 0; - std::size_t m_scratchFlagsCount = 0; - - size_type *m_scratchSpace = nullptr; - size_type *m_scratchFlags = nullptr; - - hipDeviceProp_t m_deviceProp; - - hipStream_t m_stream = nullptr; - uint32_t m_instance_id = Kokkos::Tools::Experimental::Impl::idForInstance< - Kokkos::Experimental::HIP>(reinterpret_cast(this)); - bool m_manage_stream = false; - - // Team Scratch Level 1 Space - mutable int64_t m_team_scratch_current_size = 0; - mutable void *m_team_scratch_ptr = nullptr; - mutable std::mutex m_team_scratch_mutex; - std::int32_t *m_scratch_locks; - - bool was_finalized = false; - - // FIXME_HIP: these want to be per-device, not per-stream... use of 'static' - // here will break once there are multiple devices though - static unsigned long *constantMemHostStaging; - static hipEvent_t constantMemReusable; - static std::mutex constantMemMutex; - - static HIPInternal &singleton(); - - int verify_is_initialized(const char *const label) const; - - int is_initialized() const { return m_hipDev >= 0; } - - void initialize(int hip_device_id, hipStream_t stream = nullptr, - bool manage_stream = false); - void finalize(); - - void print_configuration(std::ostream &) const; - - void fence() const; - void fence(const std::string &) const; - - // returns the next driver type pointer in our work array - char *get_next_driver(size_t driverTypeSize) const; - - ~HIPInternal(); - - HIPInternal() = default; - - // Resizing of reduction related scratch spaces - size_type *scratch_space(const std::size_t size); - size_type *scratch_flags(const std::size_t size); - uint32_t impl_get_instance_id() const noexcept; - // Resizing of team level 1 scratch - void *resize_team_scratch_space(std::int64_t bytes, - bool force_shrink = false); -}; - -} // namespace Impl - -// Partitioning an Execution Space: expects space and integer arguments for -// relative weight -// Customization point for backends -// Default behavior is to return the passed in instance - -namespace Impl { -inline void create_HIP_instances(std::vector &instances) { - for (int s = 0; s < int(instances.size()); s++) { - hipStream_t stream; - KOKKOS_IMPL_HIP_SAFE_CALL(hipStreamCreate(&stream)); - instances[s] = HIP(stream, true); - } -} -} // namespace Impl - -template -std::vector partition_space(const HIP &, Args...) { -#ifdef __cpp_fold_expressions - static_assert( - (... && std::is_arithmetic_v), - "Kokkos Error: partitioning arguments must be integers or floats"); -#endif - - std::vector instances(sizeof...(Args)); - Impl::create_HIP_instances(instances); - return instances; -} - -template -std::vector partition_space(const HIP &, std::vector &weights) { - static_assert( - std::is_arithmetic::value, - "Kokkos Error: partitioning arguments must be integers or floats"); - - std::vector instances(weights.size()); - Impl::create_HIP_instances(instances); - return instances; -} -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Locks.cpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Locks.cpp deleted file mode 100644 index f1ffaf3753..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Locks.cpp +++ /dev/null @@ -1,117 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include - -#include - -#include - -namespace Kokkos { - -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE -namespace Impl { -__device__ __constant__ HIPLockArrays g_device_hip_lock_arrays = {nullptr, 0}; -} -#endif - -namespace { - -__global__ void init_lock_array_kernel_atomic() { - unsigned i = blockIdx.x * blockDim.x + threadIdx.x; - if (i < KOKKOS_IMPL_HIP_SPACE_ATOMIC_MASK + 1) { - Kokkos::Impl::g_device_hip_lock_arrays.atomic[i] = 0; - } -} - -} // namespace - -namespace Impl { - -HIPLockArrays g_host_hip_lock_arrays = {nullptr, 0}; - -void initialize_host_hip_lock_arrays() { -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS - desul::Impl::init_lock_arrays(); - - DESUL_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE(); -#endif - - if (g_host_hip_lock_arrays.atomic != nullptr) return; - KOKKOS_IMPL_HIP_SAFE_CALL(hipMalloc( - &g_host_hip_lock_arrays.atomic, - sizeof(std::int32_t) * (KOKKOS_IMPL_HIP_SPACE_ATOMIC_MASK + 1))); - - g_host_hip_lock_arrays.n = ::Kokkos::Experimental::HIP::concurrency(); - - KOKKOS_COPY_HIP_LOCK_ARRAYS_TO_DEVICE(); - init_lock_array_kernel_atomic<<< - (KOKKOS_IMPL_HIP_SPACE_ATOMIC_MASK + 1 + 255) / 256, 256, 0, nullptr>>>(); -} - -void finalize_host_hip_lock_arrays() { -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS - desul::Impl::finalize_lock_arrays(); -#endif - - if (g_host_hip_lock_arrays.atomic == nullptr) return; - KOKKOS_IMPL_HIP_SAFE_CALL(hipFree(g_host_hip_lock_arrays.atomic)); - g_host_hip_lock_arrays.atomic = nullptr; - g_host_hip_lock_arrays.n = 0; -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE - KOKKOS_COPY_HIP_LOCK_ARRAYS_TO_DEVICE(); -#endif -} - -} // namespace Impl - -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Locks.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Locks.hpp deleted file mode 100644 index c72616dab1..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Locks.hpp +++ /dev/null @@ -1,185 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_LOCKS_HPP -#define KOKKOS_HIP_LOCKS_HPP - -#include - -#include - -#include - -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#include -#endif - -namespace Kokkos { -namespace Impl { - -struct HIPLockArrays { - std::int32_t* atomic; - std::int32_t n; -}; - -/// \brief This global variable in Host space is the central definition -/// of these arrays. -extern HIPLockArrays g_host_hip_lock_arrays; - -/// \brief After this call, the g_host_hip_lock_arrays variable has -/// valid, initialized arrays. -/// -/// This call is idempotent. -void initialize_host_hip_lock_arrays(); - -/// \brief After this call, the g_host_hip_lock_arrays variable has -/// all null pointers, and all array memory has been freed. -/// -/// This call is idempotent. -void finalize_host_hip_lock_arrays(); - -#if defined(__HIPCC__) - -/// \brief This global variable in HIP space is what kernels use -/// to get access to the lock arrays. -/// -/// When relocatable device code is enabled, there can be one single -/// instance of this global variable for the entire executable, -/// whose definition will be in Kokkos_HIP_Locks.cpp (and whose declaration -/// here must then be extern). -/// This one instance will be initialized by initialize_host_HIP_lock_arrays -/// and need not be modified afterwards. -/// -/// When relocatable device code is disabled, an instance of this variable -/// will be created in every translation unit that sees this header file. -/// Since the Kokkos_HIP_Locks.cpp translation unit cannot initialize the -/// instances in other translation units, we must update this HIP global -/// variable based on the Host global variable prior to running any kernels -/// that will use it. -/// That is the purpose of the KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE macro. -__device__ -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE - __constant__ extern -#endif - HIPLockArrays g_device_hip_lock_arrays; - -#define KOKKOS_IMPL_HIP_SPACE_ATOMIC_MASK 0x1FFFF - -/// \brief Acquire a lock for the address -/// -/// This function tries to acquire the lock for the hash value derived -/// from the provided ptr. If the lock is successfully acquired the -/// function returns true. Otherwise it returns false. -__device__ inline bool lock_address_hip_space(void* ptr) { - auto offset = reinterpret_cast(ptr); - offset = offset >> 2; - offset = offset & KOKKOS_IMPL_HIP_SPACE_ATOMIC_MASK; - return (0 == atomicCAS(&g_device_hip_lock_arrays.atomic[offset], 0, 1)); -} - -/// \brief Release lock for the address -/// -/// This function releases the lock for the hash value derived -/// from the provided ptr. This function should only be called -/// after previously successfully aquiring a lock with -/// lock_address. -__device__ inline void unlock_address_hip_space(void* ptr) { - auto offset = reinterpret_cast(ptr); - offset = offset >> 2; - offset = offset & KOKKOS_IMPL_HIP_SPACE_ATOMIC_MASK; - atomicExch(&g_device_hip_lock_arrays.atomic[offset], 0); -} - -} // namespace Impl -} // namespace Kokkos - -// Make lock_array_copied an explicit translation unit scope thingy -namespace Kokkos { -namespace Impl { -namespace { -static int lock_array_copied = 0; -inline int eliminate_warning_for_lock_array() { return lock_array_copied; } -} // namespace -} // namespace Impl -} // namespace Kokkos - -/* Dan Ibanez: it is critical that this code be a macro, so that it will - capture the right address for g_device_hip_lock_arrays! - putting this in an inline function will NOT do the right thing! */ -#define KOKKOS_COPY_HIP_LOCK_ARRAYS_TO_DEVICE() \ - { \ - if (::Kokkos::Impl::lock_array_copied == 0) { \ - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemcpyToSymbol( \ - HIP_SYMBOL(::Kokkos::Impl::g_device_hip_lock_arrays), \ - &::Kokkos::Impl::g_host_hip_lock_arrays, \ - sizeof(::Kokkos::Impl::HIPLockArrays))); \ - } \ - ::Kokkos::Impl::lock_array_copied = 1; \ - } - -#ifndef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS - -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE -#define KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE() -#else -#define KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE() \ - KOKKOS_COPY_HIP_LOCK_ARRAYS_TO_DEVICE() -#endif - -#else - -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE -#define KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE() -#else -// Still Need COPY_CUDA_LOCK_ARRAYS for team scratch etc. -#define KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE() \ - KOKKOS_COPY_HIP_LOCK_ARRAYS_TO_DEVICE() \ - DESUL_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE() -#endif - -#endif /* defined( KOKKOS_ENABLE_IMPL_DESUL_ATOMICS ) */ - -#endif /* defined( __HIPCC__ ) */ - -#endif /* #ifndef KOKKOS_HIP_LOCKS_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_MDRangePolicy.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_MDRangePolicy.hpp deleted file mode 100644 index acb538e1cb..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_MDRangePolicy.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef KOKKOS_HIP_MDRANGEPOLICY_HPP_ -#define KOKKOS_HIP_MDRANGEPOLICY_HPP_ - -#include - -namespace Kokkos { - -template <> -struct default_outer_direction { - using type = Iterate; - static constexpr Iterate value = Iterate::Left; -}; - -template <> -struct default_inner_direction { - using type = Iterate; - static constexpr Iterate value = Iterate::Left; -}; - -namespace Impl { - -// Settings for MDRangePolicy -template <> -inline TileSizeProperties get_tile_size_properties( - const Kokkos::Experimental::HIP& space) { - TileSizeProperties properties; - properties.max_threads = - space.impl_internal_space_instance()->m_maxThreadsPerSM; - properties.default_largest_tile_size = 16; - properties.default_tile_size = 4; - properties.max_total_tile_size = - Kokkos::Experimental::Impl::HIPTraits::MaxThreadsPerBlock; - return properties; -} - -} // Namespace Impl -} // Namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp deleted file mode 100644 index 212bbb9ecd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp +++ /dev/null @@ -1,451 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_PARALLEL_MDRANGE_HPP -#define KOKKOS_HIP_PARALLEL_MDRANGE_HPP - -#include -#include -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { -// ParallelFor -template -class ParallelFor, - Kokkos::Experimental::HIP> { - public: - using Policy = Kokkos::MDRangePolicy; - - private: - using array_index_type = typename Policy::array_index_type; - using index_type = typename Policy::index_type; - using LaunchBounds = typename Policy::launch_bounds; - - const FunctorType m_functor; - const Policy m_policy; - - ParallelFor() = delete; - ParallelFor& operator=(ParallelFor const&) = delete; - - public: - inline __device__ void operator()() const { - Kokkos::Impl::DeviceIterateTile(m_policy, - m_functor) - .exec_range(); - } - - inline void execute() const { - using ClosureType = - ParallelFor; - if (m_policy.m_num_tiles == 0) return; - auto const maxblocks = - Kokkos::Experimental::Impl::hip_internal_maximum_grid_count(); - if (Policy::rank == 2) { - dim3 const block(m_policy.m_tile[0], m_policy.m_tile[1], 1); - dim3 const grid( - std::min( - (m_policy.m_upper[0] - m_policy.m_lower[0] + block.x - 1) / - block.x, - maxblocks[0]), - std::min( - (m_policy.m_upper[1] - m_policy.m_lower[1] + block.y - 1) / - block.y, - maxblocks[1]), - 1); - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, 0, - m_policy.space().impl_internal_space_instance(), false); - } else if (Policy::rank == 3) { - dim3 const block(m_policy.m_tile[0], m_policy.m_tile[1], - m_policy.m_tile[2]); - dim3 const grid( - std::min( - (m_policy.m_upper[0] - m_policy.m_lower[0] + block.x - 1) / - block.x, - maxblocks[0]), - std::min( - (m_policy.m_upper[1] - m_policy.m_lower[1] + block.y - 1) / - block.y, - maxblocks[1]), - std::min( - (m_policy.m_upper[2] - m_policy.m_lower[2] + block.z - 1) / - block.z, - maxblocks[2])); - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, 0, - m_policy.space().impl_internal_space_instance(), false); - } else if (Policy::rank == 4) { - // id0,id1 encoded within threadIdx.x; id2 to threadIdx.y; id3 to - // threadIdx.z - dim3 const block(m_policy.m_tile[0] * m_policy.m_tile[1], - m_policy.m_tile[2], m_policy.m_tile[3]); - dim3 const grid( - std::min( - m_policy.m_tile_end[0] * m_policy.m_tile_end[1], maxblocks[0]), - std::min( - (m_policy.m_upper[2] - m_policy.m_lower[2] + block.y - 1) / - block.y, - maxblocks[1]), - std::min( - (m_policy.m_upper[3] - m_policy.m_lower[3] + block.z - 1) / - block.z, - maxblocks[2])); - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, 0, - m_policy.space().impl_internal_space_instance(), false); - } else if (Policy::rank == 5) { - // id0,id1 encoded within threadIdx.x; id2,id3 to threadIdx.y; id4 - // to threadIdx.z - dim3 const block(m_policy.m_tile[0] * m_policy.m_tile[1], - m_policy.m_tile[2] * m_policy.m_tile[3], - m_policy.m_tile[4]); - dim3 const grid( - std::min( - m_policy.m_tile_end[0] * m_policy.m_tile_end[1], maxblocks[0]), - std::min( - m_policy.m_tile_end[2] * m_policy.m_tile_end[3], maxblocks[1]), - std::min( - (m_policy.m_upper[4] - m_policy.m_lower[4] + block.z - 1) / - block.z, - maxblocks[2])); - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, 0, - m_policy.space().impl_internal_space_instance(), false); - } else if (Policy::rank == 6) { - // id0,id1 encoded within threadIdx.x; id2,id3 to threadIdx.y; - // id4,id5 to threadIdx.z - dim3 const block(m_policy.m_tile[0] * m_policy.m_tile[1], - m_policy.m_tile[2] * m_policy.m_tile[3], - m_policy.m_tile[4] * m_policy.m_tile[5]); - dim3 const grid( - std::min( - m_policy.m_tile_end[0] * m_policy.m_tile_end[1], maxblocks[0]), - std::min( - m_policy.m_tile_end[2] * m_policy.m_tile_end[3], maxblocks[1]), - std::min( - m_policy.m_tile_end[4] * m_policy.m_tile_end[5], maxblocks[2])); - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, 0, - m_policy.space().impl_internal_space_instance(), false); - } else { - Kokkos::abort("Kokkos::MDRange Error: Exceeded rank bounds with HIP\n"); - } - - } // end execute - - ParallelFor(FunctorType const& arg_functor, Policy const& arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} - - template - static int max_tile_size_product(const Policy&, const Functor&) { - using closure_type = - ParallelFor, - Kokkos::Experimental::HIP>; - unsigned block_size = - Kokkos::Experimental::Impl::hip_get_max_blocksize(); - if (block_size == 0) - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelFor< HIP > could not find a valid " - "tile size.")); - return block_size; - } -}; - -// ParallelReduce -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::HIP> { - public: - using Policy = Kokkos::MDRangePolicy; - - private: - using array_index_type = typename Policy::array_index_type; - using index_type = typename Policy::index_type; - - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using LaunchBounds = typename Policy::launch_bounds; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - typename Kokkos::Impl::if_c::value, - WorkTag, void>::type; - - using Analysis = - Kokkos::Impl::FunctorAnalysis; - - public: - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - using functor_type = FunctorType; - using size_type = Experimental::HIP::size_type; - - // Algorithmic constraints: blockSize is a power of two AND blockDim.y == - // blockDim.z == 1 - - const FunctorType m_functor; - const Policy m_policy; // used for workrange and nwork - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const bool m_result_ptr_device_accessible; - size_type* m_scratch_space; - size_type* m_scratch_flags; - // Only let one Parallel/Scan modify the shared memory. The - // constructor acquires the mutex which is released in the destructor. - std::lock_guard m_shared_memory_lock; - - using DeviceIteratePattern = typename Kokkos::Impl::Reduce::DeviceIterateTile< - Policy::rank, Policy, FunctorType, WorkTag, reference_type>; - - public: - inline __device__ void exec_range(reference_type update) const { - DeviceIteratePattern(m_policy, m_functor, update).exec_range(); - } - - inline __device__ void operator()() const { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - const integral_nonzero_constant - word_count(Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)) / - sizeof(size_type)); - - { - reference_type value = final_reducer.init(reinterpret_cast( - Experimental::kokkos_impl_hip_shared_memory() + - threadIdx.y * word_count.value)); - - // Number of blocks is bounded so that the reduction can be limited to two - // passes. Each thread block is given an approximately equal amount of - // work to perform. Accumulate the values for this block. The accumulation - // ordering does not match the final pass, but is arithmatically - // equivalent. - - this->exec_range(value); - } - - // Reduce with final value at blockDim.y - 1 location. - // Problem: non power-of-two blockDim - if (::Kokkos::Impl::hip_single_inter_block_reduce_scan( - final_reducer, blockIdx.x, gridDim.x, - Experimental::kokkos_impl_hip_shared_memory(), - m_scratch_space, m_scratch_flags)) { - // This is the final block with the final result at the final threads' - // location - size_type* const shared = - Experimental::kokkos_impl_hip_shared_memory() + - (blockDim.y - 1) * word_count.value; - size_type* const global = m_result_ptr_device_accessible - ? reinterpret_cast(m_result_ptr) - : m_scratch_space; - - if (threadIdx.y == 0) { - final_reducer.final(reinterpret_cast(shared)); - } - - if (Experimental::Impl::HIPTraits::WarpSize < word_count.value) { - __syncthreads(); - } - - for (unsigned i = threadIdx.y; i < word_count.value; i += blockDim.y) { - global[i] = shared[i]; - } - } - } - - // Determine block size constrained by shared memory: - // This is copy/paste from Kokkos_HIP_Parallel_Range - inline unsigned local_block_size(const FunctorType& f) { - const auto& instance = m_policy.space().impl_internal_space_instance(); - auto shmem_functor = [&f](unsigned n) { - return hip_single_inter_block_reduce_scan_shmem(f, n); - }; - using closure_type = ParallelReduce; - - unsigned block_size = - Kokkos::Experimental::Impl::hip_get_preferred_blocksize( - instance, shmem_functor); - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > could not find a " - "valid tile size.")); - } - return block_size; - } - - inline void execute() { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - using ClosureType = ParallelReduce; - const auto nwork = m_policy.m_num_tiles; - if (nwork) { - int block_size = m_policy.m_prod_tile_dims; - // CONSTRAINT: Algorithm requires block_size >= product of tile dimensions - // Nearest power of two - int exponent_pow_two = std::ceil(std::log2(block_size)); - block_size = std::pow(2, exponent_pow_two); - int suggested_blocksize = local_block_size(m_functor); - - block_size = (block_size > suggested_blocksize) - ? block_size - : suggested_blocksize; // Note: block_size must be less - // than or equal to 512 - - m_scratch_space = - ::Kokkos::Experimental::Impl::hip_internal_scratch_space( - m_policy.space(), - Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)) * - block_size /* block_size == max block_count */); - m_scratch_flags = - ::Kokkos::Experimental::Impl::hip_internal_scratch_flags( - m_policy.space(), sizeof(size_type)); - - // REQUIRED ( 1 , N , 1 ) - const dim3 block(1, block_size, 1); - // Required grid.x <= block.y - const dim3 grid(std::min(static_cast(block.y), - static_cast(nwork)), - 1, 1); - - const int shmem = - ::Kokkos::Impl::hip_single_inter_block_reduce_scan_shmem< - false, FunctorType, WorkTag>(m_functor, block.y); - - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, shmem, - m_policy.space().impl_internal_space_instance(), - false); // copy to device and execute - - if (!m_result_ptr_device_accessible && m_result_ptr) { - const int size = Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)); - DeepCopy( - m_policy.space(), m_result_ptr, m_scratch_space, size); - } - } else { - if (m_result_ptr) { - final_reducer.init(m_result_ptr); - } - } - } - - template - ParallelReduce( - const FunctorType& arg_functor, const Policy& arg_policy, - const ViewType& arg_result, - std::enable_if_t::value, void*> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_scratch_space(nullptr), - m_scratch_flags(nullptr), - m_shared_memory_lock(m_policy.space() - .impl_internal_space_instance() - ->m_mutexSharedMemory) {} - - ParallelReduce(const FunctorType& arg_functor, const Policy& arg_policy, - const ReducerType& reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_scratch_space(nullptr), - m_scratch_flags(nullptr), - m_shared_memory_lock(m_policy.space() - .impl_internal_space_instance() - ->m_mutexSharedMemory) {} - - template - static int max_tile_size_product(const Policy&, const Functor&) { - using closure_type = - ParallelReduce, - ReducerType, Kokkos::Experimental::HIP>; - unsigned block_size = - Kokkos::Experimental::Impl::hip_get_max_blocksize(); - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > could not find a " - "valid tile size.")); - } - return block_size; - } -}; -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp deleted file mode 100644 index 5c871e0d61..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp +++ /dev/null @@ -1,771 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKO_HIP_PARALLEL_RANGE_HPP -#define KOKKO_HIP_PARALLEL_RANGE_HPP - -#include - -#if defined(__HIPCC__) - -#include -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Experimental::HIP> { - public: - using Policy = Kokkos::RangePolicy; - - private: - using Member = typename Policy::member_type; - using WorkTag = typename Policy::work_tag; - using LaunchBounds = typename Policy::launch_bounds; - - const FunctorType m_functor; - const Policy m_policy; - - ParallelFor() = delete; - ParallelFor& operator=(const ParallelFor&) = delete; - - template - inline __device__ std::enable_if_t::value> exec_range( - const Member i) const { - m_functor(i); - } - - template - inline __device__ std::enable_if_t::value> exec_range( - const Member i) const { - m_functor(TagType(), i); - } - - public: - using functor_type = FunctorType; - - inline __device__ void operator()() const { - const Member work_stride = blockDim.y * gridDim.x; - const Member work_end = m_policy.end(); - - for (Member iwork = - m_policy.begin() + threadIdx.y + blockDim.y * blockIdx.x; - iwork < work_end; - iwork = iwork < work_end - work_stride ? iwork + work_stride - : work_end) { - this->template exec_range(iwork); - } - } - - inline void execute() const { - const typename Policy::index_type nwork = m_policy.end() - m_policy.begin(); - - using DriverType = - ParallelFor; - const int block_size = - Kokkos::Experimental::Impl::hip_get_preferred_blocksize(); - const dim3 block(1, block_size, 1); - const dim3 grid( - typename Policy::index_type((nwork + block.y - 1) / block.y), 1, 1); - - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelFor< HIP > could not find a " - "valid execution configuration.")); - } - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, 0, m_policy.space().impl_internal_space_instance(), - false); - } - - ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} -}; - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::HIP> { - public: - using Policy = Kokkos::RangePolicy; - - private: - using WorkRange = typename Policy::WorkRange; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using LaunchBounds = typename Policy::launch_bounds; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - typename Kokkos::Impl::if_c::value, - WorkTag, void>::type; - - using Analysis = - Kokkos::Impl::FunctorAnalysis; - - public: - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - using functor_type = FunctorType; - using size_type = Kokkos::Experimental::HIP::size_type; - using index_type = typename Policy::index_type; - - // Algorithmic constraints: blockSize is a power of two AND blockDim.y == - // blockDim.z == 1 - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const bool m_result_ptr_device_accessible; - const bool m_result_ptr_host_accessible; - size_type* m_scratch_space = nullptr; - size_type* m_scratch_flags = nullptr; - // Only let one ParallelReduce/Scan modify the shared memory. The - // constructor acquires the mutex which is released in the destructor. - std::lock_guard m_shared_memory_lock; - - static bool constexpr UseShflReduction = - static_cast(Analysis::StaticValueSize); - - private: - struct ShflReductionTag {}; - struct SHMEMReductionTag {}; - - // Make the exec_range calls call to Reduce::DeviceIterateTile - template - __device__ inline std::enable_if_t::value> exec_range( - const Member& i, reference_type update) const { - m_functor(i, update); - } - - template - __device__ inline std::enable_if_t::value> exec_range( - const Member& i, reference_type update) const { - m_functor(TagType(), i, update); - } - - public: - __device__ inline void operator()() const { - using ReductionTag = std::conditional_t; - run(ReductionTag{}); - } - - __device__ inline void run(SHMEMReductionTag) const { - const integral_nonzero_constant - word_count(Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)) / - sizeof(size_type)); - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - { - reference_type value = final_reducer.init(reinterpret_cast( - ::Kokkos::Experimental::kokkos_impl_hip_shared_memory() + - threadIdx.y * word_count.value)); - - // Number of blocks is bounded so that the reduction can be limited to two - // passes. Each thread block is given an approximately equal amount of - // work to perform. Accumulate the values for this block. The accumulation - // ordering does not match the final pass, but is arithmetically - // equivalent. - - const WorkRange range(m_policy, blockIdx.x, gridDim.x); - - for (Member iwork = range.begin() + threadIdx.y, iwork_end = range.end(); - iwork < iwork_end; iwork += blockDim.y) { - this->template exec_range(iwork, value); - } - } - - // Reduce with final value at blockDim.y - 1 location. - // Shortcut for length zero reduction - bool do_final_reduction = m_policy.begin() == m_policy.end(); - if (!do_final_reduction) - do_final_reduction = hip_single_inter_block_reduce_scan( - final_reducer, blockIdx.x, gridDim.x, - ::Kokkos::Experimental::kokkos_impl_hip_shared_memory(), - m_scratch_space, m_scratch_flags); - if (do_final_reduction) { - // This is the final block with the final result at the final threads' - // location - - size_type* const shared = - ::Kokkos::Experimental::kokkos_impl_hip_shared_memory() + - (blockDim.y - 1) * word_count.value; - size_type* const global = m_result_ptr_device_accessible - ? reinterpret_cast(m_result_ptr) - : m_scratch_space; - - if (threadIdx.y == 0) { - final_reducer.final(reinterpret_cast(shared)); - } - - if (::Kokkos::Experimental::Impl::HIPTraits::WarpSize < - word_count.value) { - __syncthreads(); - } - - for (unsigned i = threadIdx.y; i < word_count.value; i += blockDim.y) { - global[i] = shared[i]; - } - } - } - - __device__ inline void run(ShflReductionTag) const { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - value_type value; - final_reducer.init(&value); - // Number of blocks is bounded so that the reduction can be limited to two - // passes. Each thread block is given an approximately equal amount of work - // to perform. Accumulate the values for this block. The accumulation - // ordering does not match the final pass, but is arithmetically equivalent. - - WorkRange const range(m_policy, blockIdx.x, gridDim.x); - - for (Member iwork = range.begin() + threadIdx.y, iwork_end = range.end(); - iwork < iwork_end; iwork += blockDim.y) { - this->template exec_range(iwork, value); - } - - pointer_type const result = reinterpret_cast(m_scratch_space); - - int max_active_thread = static_cast(range.end() - range.begin()) < - static_cast(blockDim.y) - ? range.end() - range.begin() - : blockDim.y; - - max_active_thread = - (max_active_thread == 0) ? blockDim.y : max_active_thread; - - value_type init; - final_reducer.init(&init); - if (m_policy.begin() == m_policy.end()) { - final_reducer.final(&value); - pointer_type const final_result = - m_result_ptr_device_accessible ? m_result_ptr : result; - *final_result = value; - } else if (Impl::hip_inter_block_shuffle_reduction<>( - value, init, final_reducer, m_scratch_space, result, - m_scratch_flags, max_active_thread)) { - unsigned int const id = threadIdx.y * blockDim.x + threadIdx.x; - if (id == 0) { - final_reducer.final(&value); - pointer_type const final_result = - m_result_ptr_device_accessible ? m_result_ptr : result; - *final_result = value; - } - } - } - - // Determine block size constrained by shared memory: - inline unsigned local_block_size(const FunctorType& f) { - const auto& instance = m_policy.space().impl_internal_space_instance(); - auto shmem_functor = [&f](unsigned n) { - return hip_single_inter_block_reduce_scan_shmem(f, n); - }; - using DriverType = ParallelReduce; - return Kokkos::Experimental::Impl::hip_get_preferred_blocksize< - DriverType, LaunchBounds>(instance, shmem_functor); - } - - inline void execute() { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - const index_type nwork = m_policy.end() - m_policy.begin(); - const bool need_device_set = Analysis::has_init_member_function || - Analysis::has_final_member_function || - !m_result_ptr_host_accessible || - !std::is_same::value; - if ((nwork > 0) || need_device_set) { - const int block_size = local_block_size(m_functor); - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > could not find a " - "valid execution configuration.")); - } - - m_scratch_space = - ::Kokkos::Experimental::Impl::hip_internal_scratch_space( - m_policy.space(), - Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)) * - block_size /* block_size == max block_count */); - m_scratch_flags = - ::Kokkos::Experimental::Impl::hip_internal_scratch_flags( - m_policy.space(), sizeof(size_type)); - - // REQUIRED ( 1 , N , 1 ) - dim3 block(1, block_size, 1); - // Required grid.x <= block.y - dim3 grid(std::min(block.y, static_cast((nwork + block.y - 1) / - block.y)), - 1, 1); - - if (nwork == 0) { - block = dim3(1, 1, 1); - grid = dim3(1, 1, 1); - } - const int shmem = - UseShflReduction - ? 0 - : hip_single_inter_block_reduce_scan_shmem(m_functor, - block.y); - - using DriverType = ParallelReduce; - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, shmem, - m_policy.space().impl_internal_space_instance(), - false); // copy to device and execute - - if (!m_result_ptr_device_accessible && m_result_ptr) { - const int size = Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)); - DeepCopy(m_policy.space(), m_result_ptr, - m_scratch_space, size); - } - } else { - if (m_result_ptr) { - final_reducer.init(m_result_ptr); - } - } - } - - template - ParallelReduce( - const FunctorType& arg_functor, const Policy& arg_policy, - const ViewType& arg_result, - std::enable_if_t::value, void*> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_result_ptr_host_accessible( - MemorySpaceAccess::accessible), - m_shared_memory_lock(m_policy.space() - .impl_internal_space_instance() - ->m_mutexSharedMemory) {} - - ParallelReduce(const FunctorType& arg_functor, const Policy& arg_policy, - const ReducerType& reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_result_ptr_host_accessible( - MemorySpaceAccess::accessible), - m_shared_memory_lock(m_policy.space() - .impl_internal_space_instance() - ->m_mutexSharedMemory) {} -}; - -template -class ParallelScanHIPBase { - public: - using Policy = Kokkos::RangePolicy; - - protected: - using Member = typename Policy::member_type; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using LaunchBounds = typename Policy::launch_bounds; - - using Analysis = Kokkos::Impl::FunctorAnalysis; - - public: - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - using functor_type = FunctorType; - using size_type = Kokkos::Experimental::HIP::size_type; - using index_type = typename Policy::index_type; - - protected: - // Algorithmic constraints: - // (a) blockDim.y is a power of two - // (b) blockDim.x == blockDim.z == 1 - // (c) gridDim.x <= blockDim.y * blockDim.y - // (d) gridDim.y == gridDim.z == 1 - - const FunctorType m_functor; - const Policy m_policy; - size_type* m_scratch_space = nullptr; - size_type* m_scratch_flags = nullptr; - size_type m_final = false; - int m_grid_x = 0; - // Only let one ParallelReduce/Scan modify the shared memory. The - // constructor acquires the mutex which is released in the destructor. - std::lock_guard m_shared_memory_lock; - - private: - template - __device__ inline std::enable_if_t::value> exec_range( - const Member& i, reference_type update, const bool final_result) const { - m_functor(i, update, final_result); - } - - template - __device__ inline std::enable_if_t::value> exec_range( - const Member& i, reference_type update, const bool final_result) const { - m_functor(TagType(), i, update, final_result); - } - - //---------------------------------------- - - __device__ inline void initial() const { - typename Analysis::Reducer final_reducer(&m_functor); - - const integral_nonzero_constant - word_count(Analysis::value_size(m_functor) / sizeof(size_type)); - - pointer_type const shared_value = reinterpret_cast( - Kokkos::Experimental::kokkos_impl_hip_shared_memory() + - word_count.value * threadIdx.y); - - final_reducer.init(shared_value); - - // Number of blocks is bounded so that the reduction can be limited to two - // passes. Each thread block is given an approximately equal amount of work - // to perform. Accumulate the values for this block. The accumulation - // ordering does not match the final pass, but is arithmetically equivalent. - - const WorkRange range(m_policy, blockIdx.x, gridDim.x); - - for (Member iwork = range.begin() + threadIdx.y, iwork_end = range.end(); - iwork < iwork_end; iwork += blockDim.y) { - this->template exec_range( - iwork, final_reducer.reference(shared_value), false); - } - - // Reduce and scan, writing out scan of blocks' totals and block-groups' - // totals. Blocks' scan values are written to 'blockIdx.x' location. - // Block-groups' scan values are at: i = ( j * blockDim.y - 1 ) for i < - // gridDim.x - hip_single_inter_block_reduce_scan( - final_reducer, blockIdx.x, gridDim.x, - Kokkos::Experimental::kokkos_impl_hip_shared_memory(), - m_scratch_space, m_scratch_flags); - } - - //---------------------------------------- - - __device__ inline void final() const { - typename Analysis::Reducer final_reducer(&m_functor); - - const integral_nonzero_constant - word_count(Analysis::value_size(m_functor) / sizeof(size_type)); - - // Use shared memory as an exclusive scan: { 0 , value[0] , value[1] , - // value[2] , ... } - size_type* const shared_data = - Kokkos::Experimental::kokkos_impl_hip_shared_memory(); - size_type* const shared_prefix = - shared_data + word_count.value * threadIdx.y; - size_type* const shared_accum = - shared_data + word_count.value * (blockDim.y + 1); - - // Starting value for this thread block is the previous block's total. - if (blockIdx.x) { - size_type* const block_total = - m_scratch_space + word_count.value * (blockIdx.x - 1); - for (unsigned i = threadIdx.y; i < word_count.value; ++i) { - shared_accum[i] = block_total[i]; - } - } else if (0 == threadIdx.y) { - final_reducer.init(reinterpret_cast(shared_accum)); - } - - const WorkRange range(m_policy, blockIdx.x, gridDim.x); - - for (typename Policy::member_type iwork_base = range.begin(); - iwork_base < range.end(); iwork_base += blockDim.y) { - const typename Policy::member_type iwork = iwork_base + threadIdx.y; - - __syncthreads(); // Don't overwrite previous iteration values until they - // are used - - final_reducer.init( - reinterpret_cast(shared_prefix + word_count.value)); - - // Copy previous block's accumulation total into thread[0] prefix and - // inclusive scan value of this block - for (unsigned i = threadIdx.y; i < word_count.value; ++i) { - shared_data[i + word_count.value] = shared_data[i] = shared_accum[i]; - } - - // Make sure the write is seen by all threads - __threadfence_block(); - - // Call functor to accumulate inclusive scan value for this work item - const bool doWork = (iwork < range.end()); - if (doWork) { - this->template exec_range( - iwork, - final_reducer.reference(reinterpret_cast( - shared_prefix + word_count.value)), - false); - } - - // Scan block values into locations shared_data[1..blockDim.y] - hip_intra_block_reduce_scan( - final_reducer, - typename Analysis::pointer_type(shared_data + word_count.value)); - - { - size_type* const block_total = - shared_data + word_count.value * blockDim.y; - for (unsigned i = threadIdx.y; i < word_count.value; ++i) { - shared_accum[i] = block_total[i]; - } - } - - // Call functor with exclusive scan value - if (doWork) { - this->template exec_range( - iwork, - final_reducer.reference( - reinterpret_cast(shared_prefix)), - true); - } - } - } - - public: - //---------------------------------------- - - __device__ inline void operator()() const { - if (!m_final) { - initial(); - } else { - final(); - } - } - - // Determine block size constrained by shared memory: - virtual inline unsigned local_block_size(const FunctorType& f) = 0; - - inline void impl_execute() { - const index_type nwork = m_policy.end() - m_policy.begin(); - if (nwork) { - // FIXME_HIP we cannot choose it larger for large work sizes to work - // correctly, the unit tests fail with wrong results - const int gridMaxComputeCapability_2x = 0x01fff; - - const int block_size = static_cast(local_block_size(m_functor)); - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelScan< HIP > could not find a " - "valid execution configuration.")); - } - - const int grid_max = - std::min(block_size * block_size, gridMaxComputeCapability_2x); - - // At most 'max_grid' blocks: - const int max_grid = - std::min(grid_max, (nwork + block_size - 1) / block_size); - - // How much work per block: - const int work_per_block = (nwork + max_grid - 1) / max_grid; - - // How many block are really needed for this much work: - m_grid_x = (nwork + work_per_block - 1) / work_per_block; - - m_scratch_space = Kokkos::Experimental::Impl::hip_internal_scratch_space( - m_policy.space(), Analysis::value_size(m_functor) * m_grid_x); - m_scratch_flags = Kokkos::Experimental::Impl::hip_internal_scratch_flags( - m_policy.space(), sizeof(size_type) * 1); - - dim3 grid(m_grid_x, 1, 1); - dim3 block(1, block_size, 1); // REQUIRED DIMENSIONS ( 1 , N , 1 ) - const int shmem = Analysis::value_size(m_functor) * (block_size + 2); - - m_final = false; - // these ones are OK to be just the base because the specializations - // do not modify the kernel at all - using DriverType = ParallelScanHIPBase; - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, shmem, - m_policy.space().impl_internal_space_instance(), - false); // copy to device and execute - - m_final = true; - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, shmem, - m_policy.space().impl_internal_space_instance(), - false); // copy to device and execute - } - } - - ParallelScanHIPBase(const FunctorType& arg_functor, const Policy& arg_policy) - : m_functor(arg_functor), - m_policy(arg_policy), - m_shared_memory_lock(m_policy.space() - .impl_internal_space_instance() - ->m_mutexSharedMemory) {} -}; - -template -class ParallelScan, - Kokkos::Experimental::HIP> - : public ParallelScanHIPBase { - public: - using Base = ParallelScanHIPBase; - using Base::operator(); - - inline void execute() { Base::impl_execute(); } - - ParallelScan(const FunctorType& arg_functor, - const typename Base::Policy& arg_policy) - : Base(arg_functor, arg_policy) {} - - inline unsigned local_block_size(const FunctorType& f) { - // blockDim.y must be power of two = 128 (2 warps) or 256 (4 warps) or - // 512 (8 warps) gridDim.x <= blockDim.y * blockDim.y - - const auto& instance = - Base::m_policy.space().impl_internal_space_instance(); - auto shmem_functor = [&f](unsigned n) { - return hip_single_inter_block_reduce_scan_shmem( - f, n); - }; - using DriverType = ParallelScan; - return Kokkos::Experimental::Impl::hip_get_preferred_blocksize< - DriverType, typename Base::LaunchBounds>(instance, shmem_functor); - } -}; - -//---------------------------------------------------------------------------- - -template -class ParallelScanWithTotal, - ReturnType, Kokkos::Experimental::HIP> - : public ParallelScanHIPBase { - public: - using Base = ParallelScanHIPBase; - using Base::operator(); - - ReturnType& m_returnvalue; - - inline void execute() { - Base::impl_execute(); - - const auto nwork = Base::m_policy.end() - Base::m_policy.begin(); - if (nwork) { - const int size = Base::Analysis::value_size(Base::m_functor); - DeepCopy( - Base::m_policy.space(), &m_returnvalue, - Base::m_scratch_space + (Base::m_grid_x - 1) * size / sizeof(int), - size); - } - } - - ParallelScanWithTotal(const FunctorType& arg_functor, - const typename Base::Policy& arg_policy, - ReturnType& arg_returnvalue) - : Base(arg_functor, arg_policy), m_returnvalue(arg_returnvalue) {} - - inline unsigned local_block_size(const FunctorType& f) { - // blockDim.y must be power of two = 128 (2 warps) or 256 (4 warps) or - // 512 (8 warps) gridDim.x <= blockDim.y * blockDim.y - - const auto& instance = - Base::m_policy.space().impl_internal_space_instance(); - auto shmem_functor = [&f](unsigned n) { - return hip_single_inter_block_reduce_scan_shmem( - f, n); - }; - using DriverType = - ParallelScanWithTotal; - return Kokkos::Experimental::Impl::hip_get_preferred_blocksize< - DriverType, typename Base::LaunchBounds>(instance, shmem_functor); - } -}; - -} // namespace Impl -} // namespace Kokkos - -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp deleted file mode 100644 index 69ced48a9b..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp +++ /dev/null @@ -1,1082 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKO_HIP_PARALLEL_TEAM_HPP -#define KOKKO_HIP_PARALLEL_TEAM_HPP - -#include - -#if defined(__HIPCC__) - -#include -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class TeamPolicyInternal - : public PolicyTraits { - public: - using execution_policy = TeamPolicyInternal; - - using traits = PolicyTraits; - using BlockType = Kokkos::Experimental::Impl::BlockType; - - template - friend class TeamPolicyInternal; - - private: - typename traits::execution_space m_space; - int m_league_size; - int m_team_size; - int m_vector_length; - size_t m_team_scratch_size[2]; - size_t m_thread_scratch_size[2]; - int m_chunk_size; - bool m_tune_team_size; - bool m_tune_vector_length; - - public: - using execution_space = Kokkos::Experimental::HIP; - - template - TeamPolicyInternal(TeamPolicyInternal const& p) { - m_league_size = p.m_league_size; - m_team_size = p.m_team_size; - m_vector_length = p.m_vector_length; - m_team_scratch_size[0] = p.m_team_scratch_size[0]; - m_team_scratch_size[1] = p.m_team_scratch_size[1]; - m_thread_scratch_size[0] = p.m_thread_scratch_size[0]; - m_thread_scratch_size[1] = p.m_thread_scratch_size[1]; - m_chunk_size = p.m_chunk_size; - m_space = p.m_space; - m_tune_team_size = p.m_tune_team_size; - m_tune_vector_length = p.m_tune_vector_length; - } - - template - int team_size_max(FunctorType const& f, ParallelForTag const&) const { - using closure_type = - Impl::ParallelFor>; - - return internal_team_size_common(f); - } - - template - inline int team_size_max(const FunctorType& f, - const ParallelReduceTag&) const { - using functor_analysis_type = - Impl::FunctorAnalysis; - using reducer_type = typename Impl::ParallelReduceReturnValue< - void, typename functor_analysis_type::value_type, - FunctorType>::reducer_type; - using closure_type = - Impl::ParallelReduce, - reducer_type>; - return internal_team_size_max(f); - } - - template - inline int team_size_max(const FunctorType& f, const ReducerType&, - const ParallelReduceTag&) const { - using closure_type = - Impl::ParallelReduce, - ReducerType>; - return internal_team_size_max(f); - } - - template - int team_size_recommended(FunctorType const& f, ParallelForTag const&) const { - using closure_type = - Impl::ParallelFor>; - - return internal_team_size_common(f); - } - - template - inline int team_size_recommended(FunctorType const& f, - ParallelReduceTag const&) const { - using functor_analysis_type = - Impl::FunctorAnalysis; - using reducer_type = typename Impl::ParallelReduceReturnValue< - void, typename functor_analysis_type::value_type, - FunctorType>::reducer_type; - using closure_type = - Impl::ParallelReduce, - reducer_type>; - return internal_team_size_recommended(f); - } - - template - int team_size_recommended(FunctorType const& f, ReducerType const&, - ParallelReduceTag const&) const { - using closure_type = - Impl::ParallelReduce, - ReducerType>; - return internal_team_size_recommended(f); - } - - inline bool impl_auto_vector_length() const { return m_tune_vector_length; } - inline bool impl_auto_team_size() const { return m_tune_team_size; } - static int vector_length_max() { - return ::Kokkos::Experimental::Impl::HIPTraits::WarpSize; - } - - static int verify_requested_vector_length(int requested_vector_length) { - int test_vector_length = - std::min(requested_vector_length, vector_length_max()); - - // Allow only power-of-two vector_length - if (!(is_integral_power_of_two(test_vector_length))) { - int test_pow2 = 1; - int constexpr warp_size = Experimental::Impl::HIPTraits::WarpSize; - while (test_pow2 < warp_size) { - test_pow2 <<= 1; - if (test_pow2 > test_vector_length) { - break; - } - } - test_vector_length = test_pow2 >> 1; - } - - return test_vector_length; - } - - static int scratch_size_max(int level) { - return ( - level == 0 ? 1024 * 40 : // FIXME_HIP arbitrarily setting this to 48kB - 20 * 1024 * 1024); // FIXME_HIP arbitrarily setting this to 20MB - } - inline void impl_set_vector_length(size_t size) { m_vector_length = size; } - inline void impl_set_team_size(size_t size) { m_team_size = size; } - int impl_vector_length() const { return m_vector_length; } - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - KOKKOS_DEPRECATED int vector_length() const { return impl_vector_length(); } -#endif - - int team_size() const { return m_team_size; } - - int league_size() const { return m_league_size; } - - size_t scratch_size(int level, int team_size_ = -1) const { - if (team_size_ < 0) team_size_ = m_team_size; - return m_team_scratch_size[level] + - team_size_ * m_thread_scratch_size[level]; - } - - size_t team_scratch_size(int level) const { - return m_team_scratch_size[level]; - } - - size_t thread_scratch_size(int level) const { - return m_thread_scratch_size[level]; - } - - typename traits::execution_space space() const { return m_space; } - - TeamPolicyInternal() - : m_space(typename traits::execution_space()), - m_league_size(0), - m_team_size(-1), - m_vector_length(0), - m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(::Kokkos::Experimental::Impl::HIPTraits::WarpSize), - m_tune_team_size(false), - m_tune_vector_length(false) {} - - /** \brief Specify league size, request team size */ - TeamPolicyInternal(const execution_space space_, int league_size_, - int team_size_request, int vector_length_request = 1) - : m_space(space_), - m_league_size(league_size_), - m_team_size(team_size_request), - m_vector_length( - (vector_length_request > 0) - ? verify_requested_vector_length(vector_length_request) - : (verify_requested_vector_length(1))), - m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(::Kokkos::Experimental::Impl::HIPTraits::WarpSize), - m_tune_team_size(bool(team_size_request <= 0)), - m_tune_vector_length(bool(vector_length_request <= 0)) { - // Make sure league size is permissible - if (league_size_ >= - static_cast( - ::Kokkos::Experimental::Impl::hip_internal_maximum_grid_count()[0])) - Impl::throw_runtime_exception( - "Requested too large league_size for TeamPolicy on HIP execution " - "space."); - - // Make sure total block size is permissible - if (m_team_size * m_vector_length > - ::Kokkos::Experimental::Impl::HIPTraits::MaxThreadsPerBlock) { - Impl::throw_runtime_exception( - std::string("Kokkos::TeamPolicy< HIP > the team size is too large. " - "Team size x vector length must be smaller than 1024.")); - } - } - - /** \brief Specify league size, request team size */ - TeamPolicyInternal(const execution_space space_, int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - int vector_length_request = 1) - : TeamPolicyInternal(space_, league_size_, -1, vector_length_request) {} - // FLAG - /** \brief Specify league size and team size, request vector length*/ - TeamPolicyInternal(const execution_space space_, int league_size_, - int team_size_request, - const Kokkos::AUTO_t& /* vector_length_request */ - ) - : TeamPolicyInternal(space_, league_size_, team_size_request, -1) - - {} - - /** \brief Specify league size, request team size and vector length*/ - TeamPolicyInternal(const execution_space space_, int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - const Kokkos::AUTO_t& /* vector_length_request */ - - ) - : TeamPolicyInternal(space_, league_size_, -1, -1) - - {} - - TeamPolicyInternal(int league_size_, int team_size_request, - int vector_length_request = 1) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, - team_size_request, vector_length_request) {} - - TeamPolicyInternal(int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - int vector_length_request = 1) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, -1, - vector_length_request) {} - - /** \brief Specify league size and team size, request vector length*/ - TeamPolicyInternal(int league_size_, int team_size_request, - const Kokkos::AUTO_t& /* vector_length_request */ - - ) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, - team_size_request, -1) - - {} - - /** \brief Specify league size, request team size and vector length*/ - TeamPolicyInternal(int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - const Kokkos::AUTO_t& /* vector_length_request */ - - ) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, -1, - -1) {} - - int chunk_size() const { return m_chunk_size; } - - TeamPolicyInternal& set_chunk_size(typename traits::index_type chunk_size_) { - m_chunk_size = chunk_size_; - return *this; - } - - /** \brief set per team scratch size for a specific level of the scratch - * hierarchy */ - TeamPolicyInternal& set_scratch_size(int level, - PerTeamValue const& per_team) { - m_team_scratch_size[level] = per_team.value; - return *this; - } - - /** \brief set per thread scratch size for a specific level of the scratch - * hierarchy */ - TeamPolicyInternal& set_scratch_size(int level, - PerThreadValue const& per_thread) { - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - /** \brief set per thread and per team scratch size for a specific level of - * the scratch hierarchy */ - TeamPolicyInternal& set_scratch_size(int level, PerTeamValue const& per_team, - PerThreadValue const& per_thread) { - m_team_scratch_size[level] = per_team.value; - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - using member_type = Kokkos::Impl::HIPTeamMember; - - protected: - template - int internal_team_size_common(const FunctorType& f) const { - // FIXME_HIP: this could be unified with the - // internal_team_size_common_reduce - // once we can turn c++17 constexpr on by default. - // The problem right now is that we can't turn off the evaluation - // of the Analysis' valuesize / StaticValueSize - - const unsigned shmem_block = team_scratch_size(0) + 2 * sizeof(double); - const unsigned shmem_thread = thread_scratch_size(0) + sizeof(double); - const int vector_length = impl_vector_length(); - - const auto functor = [&f, shmem_block, shmem_thread, vector_length]( - const hipFuncAttributes& attr, int block_size) { - int functor_shmem = - ::Kokkos::Impl::FunctorTeamShmemSize::value( - f, block_size / vector_length); - return shmem_block + shmem_thread * (block_size / vector_length) + - functor_shmem + attr.sharedSizeBytes; - }; - int block_size; - // FIXME_HIP - could be if constexpr for c++17 - if (BlockSize == BlockType::Max) { - block_size = ::Kokkos::Experimental::Impl::hip_get_max_team_blocksize< - ClosureType, typename traits::launch_bounds>( - space().impl_internal_space_instance(), functor); - } else { - block_size = - ::Kokkos::Experimental::Impl::hip_get_preferred_team_blocksize< - ClosureType, typename traits::launch_bounds>( - space().impl_internal_space_instance(), functor); - } - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelFor< HIP > could not find a valid " - "team size.")); - } - return block_size / impl_vector_length(); - } - - template - int internal_team_size_common_reduce(const FunctorType& f) const { - using Interface = - typename Impl::DeduceFunctorPatternInterface::type; - using Analysis = - Impl::FunctorAnalysis; - - const unsigned shmem_block = team_scratch_size(0) + 2 * sizeof(double); - const unsigned shmem_thread = - thread_scratch_size(0) + sizeof(double) + - ((Analysis::StaticValueSize != 0) ? 0 : Analysis::value_size(f)); - const int vector_length = impl_vector_length(); - - const auto functor = [&f, shmem_block, shmem_thread, vector_length]( - const hipFuncAttributes& attr, int block_size) { - int functor_shmem = - ::Kokkos::Impl::FunctorTeamShmemSize::value( - f, block_size / vector_length); - return shmem_block + shmem_thread * (block_size / vector_length) + - functor_shmem + attr.sharedSizeBytes; - }; - int block_size; - // FIXME_HIP - could be if constexpr for c++17 - if (BlockSize == BlockType::Max) { - block_size = ::Kokkos::Experimental::Impl::hip_get_max_team_blocksize< - ClosureType, typename traits::launch_bounds>( - space().impl_internal_space_instance(), functor); - } else { - block_size = - ::Kokkos::Experimental::Impl::hip_get_preferred_team_blocksize< - ClosureType, typename traits::launch_bounds>( - space().impl_internal_space_instance(), functor); - } - - if (block_size == 0) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > could not find a " - "valid team size.")); - } - // Currently we require Power-of-2 team size for reductions. - int p2 = 1; - while (p2 <= block_size) p2 *= 2; - p2 /= 2; - return p2 / impl_vector_length(); - } - - template - int internal_team_size_max(const FunctorType& f) const { - return internal_team_size_common_reduce(f); - } - - template - int internal_team_size_recommended(const FunctorType& f) const { - return internal_team_size_common_reduce( - f); - } -}; - -__device__ inline int64_t hip_get_scratch_index( - Experimental::HIP::size_type league_size, int32_t* scratch_locks) { - int64_t threadid = 0; - __shared__ int64_t base_thread_id; - if (threadIdx.x == 0 && threadIdx.y == 0) { - int64_t const wraparound_len = - Kokkos::min(int64_t(league_size), - (int64_t(Kokkos::Impl::g_device_hip_lock_arrays.n)) / - (blockDim.x * blockDim.y)); - threadid = (blockIdx.x * blockDim.z + threadIdx.z) % wraparound_len; - threadid *= blockDim.x * blockDim.y; - int done = 0; - while (!done) { - done = (0 == atomicCAS(&scratch_locks[threadid], 0, 1)); - if (!done) { - threadid += blockDim.x * blockDim.y; - if (int64_t(threadid + blockDim.x * blockDim.y) >= - wraparound_len * blockDim.x * blockDim.y) - threadid = 0; - } - } - base_thread_id = threadid; - } - __syncthreads(); - threadid = base_thread_id; - return threadid; -} - -__device__ inline void hip_release_scratch_index(int32_t* scratch_locks, - int64_t threadid) { - __syncthreads(); - if (threadIdx.x == 0 && threadIdx.y == 0) { - scratch_locks[threadid] = 0; - } -} - -template -class ParallelFor, - Kokkos::Experimental::HIP> { - public: - using Policy = TeamPolicyInternal; - using functor_type = FunctorType; - using size_type = ::Kokkos::Experimental::HIP::size_type; - - private: - using member_type = typename Policy::member_type; - using work_tag = typename Policy::work_tag; - using launch_bounds = typename Policy::launch_bounds; - - // Algorithmic constraints: blockDim.y is a power of two AND - // blockDim.y == blockDim.z == 1 shared memory utilization: - // - // [ team reduce space ] - // [ team shared space ] - - FunctorType const m_functor; - Policy const m_policy; - size_type const m_league_size; - int m_team_size; - size_type const m_vector_size; - int m_shmem_begin; - int m_shmem_size; - void* m_scratch_ptr[2]; - size_t m_scratch_size[2]; - int32_t* m_scratch_locks; - // Only let one ParallelFor/Reduce modify the team scratch memory. The - // constructor acquires the mutex which is released in the destructor. - std::lock_guard m_scratch_lock_guard; - - template - __device__ inline std::enable_if_t::value> exec_team( - const member_type& member) const { - m_functor(member); - } - - template - __device__ inline std::enable_if_t::value> exec_team( - const member_type& member) const { - m_functor(TagType(), member); - } - - public: - __device__ inline void operator()() const { - // Iterate this block through the league - int64_t threadid = 0; - if (m_scratch_size[1] > 0) { - threadid = hip_get_scratch_index(m_league_size, m_scratch_locks); - } - - int const int_league_size = static_cast(m_league_size); - for (int league_rank = blockIdx.x; league_rank < int_league_size; - league_rank += gridDim.x) { - this->template exec_team(typename Policy::member_type( - ::Kokkos::Experimental::kokkos_impl_hip_shared_memory(), - m_shmem_begin, m_shmem_size, - static_cast(static_cast(m_scratch_ptr[1]) + - ptrdiff_t(threadid / (blockDim.x * blockDim.y)) * - m_scratch_size[1]), - m_scratch_size[1], league_rank, m_league_size)); - } - if (m_scratch_size[1] > 0) { - hip_release_scratch_index(m_scratch_locks, threadid); - } - } - - inline void execute() const { - int64_t const shmem_size_total = m_shmem_begin + m_shmem_size; - dim3 const grid(static_cast(m_league_size), 1, 1); - dim3 const block(static_cast(m_vector_size), - static_cast(m_team_size), 1); - - using closure_type = - ParallelFor, - Kokkos::Experimental::HIP>; - ::Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, shmem_size_total, - m_policy.space().impl_internal_space_instance(), - true); // copy to device and execute - } - - ParallelFor(FunctorType const& arg_functor, Policy const& arg_policy) - : m_functor(arg_functor), - m_policy(arg_policy), - m_league_size(arg_policy.league_size()), - m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()), - m_scratch_lock_guard(m_policy.space() - .impl_internal_space_instance() - ->m_team_scratch_mutex) { - m_team_size = m_team_size >= 0 ? m_team_size - : arg_policy.team_size_recommended( - arg_functor, ParallelForTag()); - - m_shmem_begin = (sizeof(double) * (m_team_size + 2)); - m_shmem_size = - (m_policy.scratch_size(0, m_team_size) + - FunctorTeamShmemSize::value(m_functor, m_team_size)); - m_scratch_size[0] = m_policy.scratch_size(0, m_team_size); - m_scratch_size[1] = m_policy.scratch_size(1, m_team_size); - m_scratch_locks = - m_policy.space().impl_internal_space_instance()->m_scratch_locks; - - // Functor's reduce memory, team scan memory, and team shared memory depend - // upon team size. - m_scratch_ptr[0] = nullptr; - m_scratch_ptr[1] = - m_team_size <= 0 - ? nullptr - : m_policy.space() - .impl_internal_space_instance() - ->resize_team_scratch_space( - static_cast(m_scratch_size[1]) * - (std::min(static_cast( - Kokkos::Experimental::HIP::concurrency() / - (m_team_size * m_vector_size)), - static_cast(m_league_size)))); - - int const shmem_size_total = m_shmem_begin + m_shmem_size; - if (m_policy.space().impl_internal_space_instance()->m_maxShmemPerBlock < - shmem_size_total) { - Kokkos::Impl::throw_runtime_exception(std::string( - "Kokkos::Impl::ParallelFor< HIP > insufficient shared memory")); - } - - size_t max_size = arg_policy.team_size_max(arg_functor, ParallelForTag()); - if (static_cast(m_team_size) > static_cast(max_size)) { - Kokkos::Impl::throw_runtime_exception(std::string( - "Kokkos::Impl::ParallelFor< HIP > requested too large team size.")); - } - } -}; - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -class ParallelReduce, - ReducerType, Kokkos::Experimental::HIP> { - public: - using Policy = TeamPolicyInternal; - - private: - using member_type = typename Policy::member_type; - using work_tag = typename Policy::work_tag; - using launch_bounds = typename Policy::launch_bounds; - - using reducer_conditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using reducer_type_fwd = typename reducer_conditional::type; - using work_tag_fwd = - typename Kokkos::Impl::if_c::value, - work_tag, void>::type; - - using analysis = Impl::FunctorAnalysis; - - using pointer_type = typename analysis::pointer_type; - using reference_type = typename analysis::reference_type; - using value_type = typename analysis::value_type; - - public: - using functor_type = FunctorType; - using size_type = Kokkos::Experimental::HIP::size_type; - - static int constexpr UseShflReduction = (analysis::StaticValueSize != 0); - - private: - struct ShflReductionTag {}; - struct SHMEMReductionTag {}; - - // Algorithmic constraints: blockDim.y is a power of two AND - // blockDim.y == blockDim.z == 1 shared memory utilization: - // - // [ global reduce space ] - // [ team reduce space ] - // [ team shared space ] - // - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const bool m_result_ptr_device_accessible; - const bool m_result_ptr_host_accessible; - size_type* m_scratch_space; - size_type* m_scratch_flags; - size_type m_team_begin; - size_type m_shmem_begin; - size_type m_shmem_size; - void* m_scratch_ptr[2]; - size_t m_scratch_size[2]; - int32_t* m_scratch_locks; - const size_type m_league_size; - int m_team_size; - const size_type m_vector_size; - // Only let one ParallelFor/Reduce modify the team scratch memory. The - // constructor acquires the mutex which is released in the destructor. - std::lock_guard m_scratch_lock_guard; - - template - __device__ inline std::enable_if_t::value> exec_team( - member_type const& member, reference_type update) const { - m_functor(member, update); - } - - template - __device__ inline std::enable_if_t::value> exec_team( - member_type const& member, reference_type update) const { - m_functor(TagType(), member, update); - } - - __device__ inline void iterate_through_league(int const threadid, - reference_type value) const { - int const int_league_size = static_cast(m_league_size); - for (int league_rank = blockIdx.x; league_rank < int_league_size; - league_rank += gridDim.x) { - this->template exec_team( - member_type( - Kokkos::Experimental::kokkos_impl_hip_shared_memory() + - m_team_begin, - m_shmem_begin, m_shmem_size, - reinterpret_cast( - reinterpret_cast(m_scratch_ptr[1]) + - static_cast(threadid / (blockDim.x * blockDim.y)) * - m_scratch_size[1]), - m_scratch_size[1], league_rank, m_league_size), - value); - } - } - - public: - __device__ inline void operator()() const { - int64_t threadid = 0; - if (m_scratch_size[1] > 0) { - threadid = hip_get_scratch_index(m_league_size, m_scratch_locks); - } - - using ReductionTag = std::conditional_t; - run(ReductionTag{}, threadid); - - if (m_scratch_size[1] > 0) { - hip_release_scratch_index(m_scratch_locks, threadid); - } - } - - __device__ inline void run(SHMEMReductionTag, int const threadid) const { - typename analysis::Reducer final_reducer( - &reducer_conditional::select(m_functor, m_reducer)); - - integral_nonzero_constant const - word_count(analysis::value_size( - reducer_conditional::select(m_functor, m_reducer)) / - sizeof(size_type)); - - reference_type value = final_reducer.init( - Kokkos::Experimental::kokkos_impl_hip_shared_memory() + - threadIdx.y * word_count.value); - - // Iterate this block through the league - iterate_through_league(threadid, value); - - // Reduce with final value at blockDim.y - 1 location. - bool do_final_reduce = (m_league_size == 0); - if (!do_final_reduce) - do_final_reduce = - hip_single_inter_block_reduce_scan( - reducer_conditional::select(m_functor, m_reducer), blockIdx.x, - gridDim.x, - Kokkos::Experimental::kokkos_impl_hip_shared_memory(), - m_scratch_space, m_scratch_flags); - if (do_final_reduce) { - // This is the final block with the final result at the final threads' - // location - - size_type* const shared = - Kokkos::Experimental::kokkos_impl_hip_shared_memory() + - (blockDim.y - 1) * word_count.value; - size_type* const global = m_result_ptr_device_accessible - ? reinterpret_cast(m_result_ptr) - : m_scratch_space; - - if (threadIdx.y == 0) { - final_reducer.final(reinterpret_cast(shared)); - } - - if (Kokkos::Experimental::Impl::HIPTraits::WarpSize < word_count.value) { - __syncthreads(); - } - - for (unsigned i = threadIdx.y; i < word_count.value; i += blockDim.y) { - global[i] = shared[i]; - } - } - } - - __device__ inline void run(ShflReductionTag, int const threadid) const { - typename analysis::Reducer final_reducer( - &reducer_conditional::select(m_functor, m_reducer)); - - value_type value; - final_reducer.init(&value); - - // Iterate this block through the league - iterate_through_league(threadid, value); - - pointer_type const result = - m_result_ptr_device_accessible - ? m_result_ptr - : reinterpret_cast(m_scratch_space); - - value_type init; - final_reducer.init(&init); - if (m_league_size == 0) { - final_reducer.final(&value); - *result = value; - } else if (Impl::hip_inter_block_shuffle_reduction( - value, init, final_reducer, m_scratch_space, result, - m_scratch_flags, blockDim.y)) { - unsigned int const id = threadIdx.y * blockDim.x + threadIdx.x; - if (id == 0) { - final_reducer.final(&value); - *result = value; - } - } - } - - inline void execute() { - typename analysis::Reducer final_reducer( - &reducer_conditional::select(m_functor, m_reducer)); - - const bool is_empty_range = m_league_size == 0 || m_team_size == 0; - const bool need_device_set = analysis::has_init_member_function || - analysis::has_final_member_function || - !m_result_ptr_host_accessible || - !std::is_same::value; - if (!is_empty_range || need_device_set) { - const int block_count = - UseShflReduction - ? std::min( - m_league_size, - size_type(1024 * - Kokkos::Experimental::Impl::HIPTraits::WarpSize)) - : std::min(static_cast(m_league_size), m_team_size); - - m_scratch_space = Kokkos::Experimental::Impl::hip_internal_scratch_space( - m_policy.space(), analysis::value_size(reducer_conditional::select( - m_functor, m_reducer)) * - block_count); - m_scratch_flags = Kokkos::Experimental::Impl::hip_internal_scratch_flags( - m_policy.space(), sizeof(size_type)); - - dim3 block(m_vector_size, m_team_size, 1); - dim3 grid(block_count, 1, 1); - if (is_empty_range) { - block = dim3(1, 1, 1); - grid = dim3(1, 1, 1); - } - const int shmem_size_total = m_team_begin + m_shmem_begin + m_shmem_size; - - using closure_type = - ParallelReduce, - ReducerType, Kokkos::Experimental::HIP>; - Kokkos::Experimental::Impl::hip_parallel_launch( - *this, grid, block, shmem_size_total, - m_policy.space().impl_internal_space_instance(), - true); // copy to device and execute - - if (!m_result_ptr_device_accessible) { - m_policy.space().impl_internal_space_instance()->fence(); - - if (m_result_ptr) { - const int size = analysis::value_size( - reducer_conditional::select(m_functor, m_reducer)); - DeepCopy( - m_result_ptr, m_scratch_space, size); - } - } - } else { - if (m_result_ptr) { - final_reducer.init(m_result_ptr); - } - } - } - - template - ParallelReduce( - FunctorType const& arg_functor, Policy const& arg_policy, - ViewType const& arg_result, - std::enable_if_t::value, void*> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_result_ptr_host_accessible( - MemorySpaceAccess::accessible), - m_scratch_space(nullptr), - m_scratch_flags(nullptr), - m_team_begin(0), - m_shmem_begin(0), - m_shmem_size(0), - m_scratch_ptr{nullptr, nullptr}, - m_league_size(arg_policy.league_size()), - m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()), - m_scratch_lock_guard(m_policy.space() - .impl_internal_space_instance() - ->m_team_scratch_mutex) { - m_team_size = m_team_size >= 0 ? m_team_size - : arg_policy.team_size_recommended( - arg_functor, ParallelReduceTag()); - - m_team_begin = - UseShflReduction - ? 0 - : hip_single_inter_block_reduce_scan_shmem(arg_functor, - m_team_size); - m_shmem_begin = sizeof(double) * (m_team_size + 2); - m_shmem_size = - m_policy.scratch_size(0, m_team_size) + - FunctorTeamShmemSize::value(arg_functor, m_team_size); - m_scratch_size[0] = m_shmem_size; - m_scratch_size[1] = m_policy.scratch_size(1, m_team_size); - m_scratch_locks = - m_policy.space().impl_internal_space_instance()->m_scratch_locks; - m_scratch_ptr[1] = - m_team_size <= 0 - ? nullptr - : m_policy.space() - .impl_internal_space_instance() - ->resize_team_scratch_space( - static_cast(m_scratch_size[1]) * - (std::min(static_cast( - Kokkos::Experimental::HIP::concurrency() / - (m_team_size * m_vector_size)), - static_cast(m_league_size)))); - - // The global parallel_reduce does not support vector_length other than 1 at - // the moment - if ((arg_policy.impl_vector_length() > 1) && !UseShflReduction) - Impl::throw_runtime_exception( - "Kokkos::parallel_reduce with a TeamPolicy using a vector length of " - "greater than 1 is not currently supported for HIP for dynamic " - "sized reduction types."); - - if ((m_team_size < Kokkos::Experimental::Impl::HIPTraits::WarpSize) && - !UseShflReduction) - Impl::throw_runtime_exception( - "Kokkos::parallel_reduce with a TeamPolicy using a team_size smaller " - "than 64 is not currently supported with HIP for dynamic sized " - "reduction types."); - - // Functor's reduce memory, team scan memory, and team shared memory depend - // upon team size. - - const int shmem_size_total = m_team_begin + m_shmem_begin + m_shmem_size; - - if (!Kokkos::Impl::is_integral_power_of_two(m_team_size) && - !UseShflReduction) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > bad team size")); - } - - if (m_policy.space().impl_internal_space_instance()->m_maxShmemPerBlock < - shmem_size_total) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > requested too much " - "L0 scratch memory")); - } - - size_t max_size = - arg_policy.team_size_max(arg_functor, ParallelReduceTag()); - if (static_cast(m_team_size) > static_cast(max_size)) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > requested too " - "large team size.")); - } - } - - ParallelReduce(FunctorType const& arg_functor, Policy const& arg_policy, - ReducerType const& reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_result_ptr_host_accessible( - MemorySpaceAccess::accessible), - m_scratch_space(nullptr), - m_scratch_flags(nullptr), - m_team_begin(0), - m_shmem_begin(0), - m_shmem_size(0), - m_scratch_ptr{nullptr, nullptr}, - m_league_size(arg_policy.league_size()), - m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()), - m_scratch_lock_guard(m_policy.space() - .impl_internal_space_instance() - ->m_team_scratch_mutex) { - m_team_size = m_team_size >= 0 - ? m_team_size - : arg_policy.team_size_recommended(arg_functor, reducer, - ParallelReduceTag()); - m_team_begin = - UseShflReduction - ? 0 - : hip_single_inter_block_reduce_scan_shmem(arg_functor, - m_team_size); - m_shmem_begin = sizeof(double) * (m_team_size + 2); - m_shmem_size = - m_policy.scratch_size(0, m_team_size) + - FunctorTeamShmemSize::value(arg_functor, m_team_size); - m_scratch_size[0] = m_shmem_size; - m_scratch_size[1] = m_policy.scratch_size(1, m_team_size); - m_scratch_locks = - m_policy.space().impl_internal_space_instance()->m_scratch_locks; - m_scratch_ptr[1] = - m_team_size <= 0 - ? nullptr - : m_policy.space() - .impl_internal_space_instance() - ->resize_team_scratch_space( - static_cast(m_scratch_size[1]) * - (std::min(static_cast( - Kokkos::Experimental::HIP::concurrency() / - (m_team_size * m_vector_size)), - static_cast(m_league_size)))); - - // The global parallel_reduce does not support vector_length other than 1 at - // the moment - if ((arg_policy.impl_vector_length() > 1) && !UseShflReduction) - Impl::throw_runtime_exception( - "Kokkos::parallel_reduce with a TeamPolicy using a vector length of " - "greater than 1 is not currently supported for HIP for dynamic " - "sized reduction types."); - - if ((m_team_size < Kokkos::Experimental::Impl::HIPTraits::WarpSize) && - !UseShflReduction) - Impl::throw_runtime_exception( - "Kokkos::parallel_reduce with a TeamPolicy using a team_size smaller " - "than 64 is not currently supported with HIP for dynamic sized " - "reduction types."); - - // Functor's reduce memory, team scan memory, and team shared memory depend - // upon team size. - - const int shmem_size_total = m_team_begin + m_shmem_begin + m_shmem_size; - if ((!Kokkos::Impl::is_integral_power_of_two(m_team_size) && - !UseShflReduction) || - m_policy.space().impl_internal_space_instance()->m_maxShmemPerBlock < - shmem_size_total) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > bad team size")); - } - - size_t max_size = - arg_policy.team_size_max(arg_functor, reducer, ParallelReduceTag()); - if (static_cast(m_team_size) > static_cast(max_size)) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::ParallelReduce< HIP > requested too " - "large team size.")); - } - } -}; -} // namespace Impl -} // namespace Kokkos - -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Space.cpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Space.cpp deleted file mode 100644 index aee9756af0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_Space.cpp +++ /dev/null @@ -1,634 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ -namespace { - -static std::atomic is_first_hip_managed_allocation(true); - -bool hip_driver_check_page_migration(int deviceId) { - // check with driver if page migrating memory is available - // this driver query is copied from the hip documentation - int hasManagedMemory = 0; // false by default - KOKKOS_IMPL_HIP_SAFE_CALL(hipDeviceGetAttribute( - &hasManagedMemory, hipDeviceAttributeManagedMemory, deviceId)); - return static_cast(hasManagedMemory); -} -} // namespace -namespace Kokkos { -namespace Impl { - -namespace { -hipStream_t get_deep_copy_stream() { - static hipStream_t s = nullptr; - if (s == nullptr) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipStreamCreate(&s)); - } - return s; -} -} // namespace - -void DeepCopyHIP(void* dst, void const* src, size_t n) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemcpyAsync(dst, src, n, hipMemcpyDefault)); -} - -void DeepCopyAsyncHIP(const Kokkos::Experimental::HIP& instance, void* dst, - void const* src, size_t n) { - KOKKOS_IMPL_HIP_SAFE_CALL( - hipMemcpyAsync(dst, src, n, hipMemcpyDefault, instance.hip_stream())); -} - -void DeepCopyAsyncHIP(void* dst, void const* src, size_t n) { - hipStream_t s = get_deep_copy_stream(); - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemcpyAsync(dst, src, n, hipMemcpyDefault, s)); - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::HIP>( - "Kokkos::Impl::DeepCopyAsyncHIP: Post Deep Copy Fence on Deep-Copy " - "stream", - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - DeepCopyResourceSynchronization, - [&]() { KOKKOS_IMPL_HIP_SAFE_CALL(hipStreamSynchronize(s)); }); -} - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -namespace Kokkos { - -KOKKOS_DEPRECATED void Experimental::HIPSpace::access_error() { - const std::string msg( - "Kokkos::Experimental::HIPSpace::access_error attempt to execute " - "Experimental::HIP function from non-HIP space"); - Kokkos::Impl::throw_runtime_exception(msg); -} - -KOKKOS_DEPRECATED void Experimental::HIPSpace::access_error(const void* const) { - const std::string msg( - "Kokkos::Experimental::HIPSpace::access_error attempt to execute " - "Experimental::HIP function from non-HIP space"); - Kokkos::Impl::throw_runtime_exception(msg); -} - -} // namespace Kokkos -#endif - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Experimental { - -HIPSpace::HIPSpace() : m_device(HIP().hip_device()) {} - -HIPHostPinnedSpace::HIPHostPinnedSpace() {} - -HIPManagedSpace::HIPManagedSpace() : m_device(HIP().hip_device()) {} - -void* HIPSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void* HIPSpace::allocate( - - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void* HIPSpace::impl_allocate( - - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - void* ptr = nullptr; - - auto const error_code = hipMalloc(&ptr, arg_alloc_size); - if (error_code != hipSuccess) { - // This is the only way to clear the last error, which we should do here - // since we're turning it into an exception here - (void)hipGetLastError(); - throw HIPRawMemoryAllocationFailure( - arg_alloc_size, error_code, - RawMemoryAllocationFailure::AllocationMechanism::HIPMalloc); - } - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void* HIPHostPinnedSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void* HIPHostPinnedSpace::allocate(const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void* HIPHostPinnedSpace::impl_allocate( - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - void* ptr = nullptr; - - auto const error_code = - hipHostMalloc(&ptr, arg_alloc_size, hipHostMallocNonCoherent); - if (error_code != hipSuccess) { - // This is the only way to clear the last error, which we should do here - // since we're turning it into an exception here - (void)hipGetLastError(); - throw HIPRawMemoryAllocationFailure( - arg_alloc_size, error_code, - RawMemoryAllocationFailure::AllocationMechanism::HIPHostMalloc); - } - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void* HIPManagedSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void* HIPManagedSpace::allocate(const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void* HIPManagedSpace::impl_allocate( - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - void* ptr = nullptr; - - if (arg_alloc_size > 0) { - if (is_first_hip_managed_allocation.exchange(false) && - Kokkos::show_warnings()) { - if (!hip_driver_check_page_migration(m_device)) { - std::cerr << R"warning( -Kokkos::HIP::allocation WARNING: The combination of device and system configuration - does not support page migration between device and host. - HIPManagedSpace might not work as expected. - Please refer to the ROCm documentation on unified/managed memory.)warning" - << std::endl; - } - - // check for correct runtime environment - const char* hsa_xnack = std::getenv("HSA_XNACK"); - if (!hsa_xnack) - std::cerr << R"warning( -Kokkos::HIP::runtime WARNING: Kokkos did not find an environment variable 'HSA_XNACK' - for the current process. - Nevertheless, xnack is enabled for all processes if - amdgpu.noretry=0 was set in the Linux kernel boot line. - Without xnack enabled, Kokkos::HIPManaged might not behave - as expected.)warning" - << std::endl; - else if (Kokkos::Impl::strcmp(hsa_xnack, "1") != 0) - std::cerr << "Kokkos::HIP::runtime WARNING: Kokkos detected the " - "environement variable " - << "'HSA_XNACK=" << hsa_xnack << "\n" - << "Kokkos advises to set it to '1' to enable it per process." - << std::endl; - } - auto const error_code = hipMallocManaged(&ptr, arg_alloc_size); - if (error_code != hipSuccess) { - // This is the only way to clear the last error, which we should do here - // since we're turning it into an exception here - (void)hipGetLastError(); - throw HIPRawMemoryAllocationFailure( - arg_alloc_size, error_code, - RawMemoryAllocationFailure::AllocationMechanism::HIPMallocManaged); - } - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemAdvise( - ptr, arg_alloc_size, hipMemAdviseSetCoarseGrain, m_device)); - } - - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void HIPSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} -void HIPSpace::deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void HIPSpace::impl_deallocate( - const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - KOKKOS_IMPL_HIP_SAFE_CALL(hipFree(arg_alloc_ptr)); -} - -void HIPHostPinnedSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void HIPHostPinnedSpace::deallocate(const char* arg_label, - void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void HIPHostPinnedSpace::impl_deallocate( - const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - KOKKOS_IMPL_HIP_SAFE_CALL(hipHostFree(arg_alloc_ptr)); -} - -void HIPManagedSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void HIPManagedSpace::deallocate(const char* arg_label, - void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void HIPManagedSpace::impl_deallocate( - const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - // We have to unset the CoarseGrain property manually as hipFree does not take - // care of it. Otherwise, the allocation would continue to linger in the - // kernel mem page table. - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemAdvise( - arg_alloc_ptr, arg_alloc_size, hipMemAdviseUnsetCoarseGrain, m_device)); - KOKKOS_IMPL_HIP_SAFE_CALL(hipFree(arg_alloc_ptr)); -} - -} // namespace Experimental -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -#ifdef KOKKOS_ENABLE_DEBUG -SharedAllocationRecord - SharedAllocationRecord::s_root_record; - -SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::HIPHostPinnedSpace, void>::s_root_record; - -SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::HIPManagedSpace, void>::s_root_record; -#endif - -SharedAllocationRecord::~SharedAllocationRecord() { - auto alloc_size = SharedAllocationRecord::m_alloc_size; - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - alloc_size, (alloc_size - sizeof(SharedAllocationHeader))); -} - -SharedAllocationRecord::~SharedAllocationRecord() { - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size); -} - -SharedAllocationRecord::~SharedAllocationRecord() { - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, arg_label); - - // Copy to device memory - Kokkos::Experimental::HIP exec; - Kokkos::Impl::DeepCopy( - exec, RecordBase::m_alloc_ptr, &header, sizeof(SharedAllocationHeader)); - exec.fence( - "SharedAllocationRecord::SharedAllocationRecord(): fence after copying header from " - "HostSpace"); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::HIP& arg_exec_space, - const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, arg_label); - - // Copy to device memory - Kokkos::Impl::DeepCopy( - arg_exec_space, RecordBase::m_alloc_ptr, &header, - sizeof(SharedAllocationHeader)); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::HIPHostPinnedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - // Fill in the Header information, directly accessible via host pinned memory - this->base_t::_fill_host_accessible_header_info(*RecordBase::m_alloc_ptr, - arg_label); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::HIPManagedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - // Fill in the Header information, directly accessible via managed memory - this->base_t::_fill_host_accessible_header_info(*RecordBase::m_alloc_ptr, - arg_label); -} - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ -namespace Kokkos { -namespace Experimental { - -int HIP::concurrency() { - auto const& prop = hip_device_prop(); - return prop.maxThreadsPerMultiProcessor * prop.multiProcessorCount; -} -int HIP::impl_is_initialized() { - return Impl::HIPInternal::singleton().is_initialized(); -} - -void HIP::impl_initialize(InitializationSettings const& settings) { - Impl::HIPInternal::singleton().initialize(::Kokkos::Impl::get_gpu(settings)); -} - -void HIP::impl_finalize() { Impl::HIPInternal::singleton().finalize(); } - -HIP::HIP() - : m_space_instance(&Impl::HIPInternal::singleton(), - [](Impl::HIPInternal*) {}) { - Impl::HIPInternal::singleton().verify_is_initialized( - "HIP instance constructor"); -} - -HIP::HIP(hipStream_t const stream, bool manage_stream) - : m_space_instance(new Impl::HIPInternal, [](Impl::HIPInternal* ptr) { - ptr->finalize(); - delete ptr; - }) { - Impl::HIPInternal::singleton().verify_is_initialized( - "HIP instance constructor"); - m_space_instance->initialize(Impl::HIPInternal::singleton().m_hipDev, stream, - manage_stream); -} - -void HIP::print_configuration(std::ostream& os, bool /*verbose*/) const { - os << "Device Execution Space:\n"; - os << " KOKKOS_ENABLE_HIP: yes\n"; - - os << "HIP Options:\n"; - os << " KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE: "; -#ifdef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE - os << "yes\n"; -#else - os << "no\n"; -#endif - - os << "\nRuntime Configuration:\n"; - - m_space_instance->print_configuration(os); -} - -uint32_t HIP::impl_instance_id() const noexcept { - return m_space_instance->impl_get_instance_id(); -} -void HIP::impl_static_fence(const std::string& name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::HIP>( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - [&]() { KOKKOS_IMPL_HIP_SAFE_CALL(hipDeviceSynchronize()); }); -} - -void HIP::fence(const std::string& name) const { - m_space_instance->fence(name); -} - -hipStream_t HIP::hip_stream() const { return m_space_instance->m_stream; } - -int HIP::hip_device() const { return impl_internal_space_instance()->m_hipDev; } - -hipDeviceProp_t const& HIP::hip_device_prop() { - return Impl::HIPInternal::singleton().m_deviceProp; -} - -const char* HIP::name() { return "HIP"; } - -} // namespace Experimental - -namespace Impl { - -int g_hip_space_factory_initialized = - initialize_space_factory<::Kokkos::Experimental::HIP>("150_HIP"); - -} // namespace Impl - -#ifdef KOKKOS_ENABLE_CXX14 -namespace Tools { -namespace Experimental { -constexpr DeviceType DeviceTypeTraits::id; -} -} // namespace Tools -#endif - -} // namespace Kokkos - -//============================================================================== -// {{{1 - -#include - -namespace Kokkos { -namespace Impl { - -// To avoid additional compilation cost for something that's (mostly?) not -// performance sensitive, we explicity instantiate these CRTP base classes here, -// where we have access to the associated *_timpl.hpp header files. -template class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::HIPSpace>; -template class SharedAllocationRecordCommon; -template class SharedAllocationRecordCommon< - Kokkos::Experimental::HIPHostPinnedSpace>; -template class SharedAllocationRecordCommon< - Kokkos::Experimental::HIPManagedSpace>; - -} // end namespace Impl -} // end namespace Kokkos - -// end Explicit instantiations of CRTP Base classes }}}1 -//============================================================================== diff --git a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_WorkGraphPolicy.hpp b/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_WorkGraphPolicy.hpp deleted file mode 100644 index 081f6f4047..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HIP/Kokkos_HIP_WorkGraphPolicy.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_WORKGRAPHPOLICY_HPP -#define KOKKOS_HIP_WORKGRAPHPOLICY_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Experimental::HIP> { - public: - using Policy = Kokkos::WorkGraphPolicy; - using Self = ParallelFor; - - private: - Policy m_policy; - FunctorType m_functor; - - template - __device__ inline std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - m_functor(w); - } - - template - __device__ inline std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - const TagType t{}; - m_functor(t, w); - } - - public: - __device__ inline void operator()() const noexcept { - // Spin until COMPLETED_TOKEN. - // END_TOKEN indicates no work is currently available. - for (std::int32_t w = Policy::END_TOKEN; - Policy::COMPLETED_TOKEN != (w = m_policy.pop_work());) { - if (Policy::END_TOKEN != w) { - exec_one(w); - m_policy.completed_work(w); - } - } - } - - inline void execute() { - const int warps_per_block = 4; - const dim3 grid( - Kokkos::Experimental::Impl::hip_internal_multiprocessor_count(), 1, 1); - const dim3 block(1, Kokkos::Experimental::Impl::HIPTraits::WarpSize, - warps_per_block); - const int shared = 0; - - Kokkos::Experimental::Impl::HIPParallelLaunch( - *this, grid, block, shared, - Experimental::HIP().impl_internal_space_instance(), false); - } - - inline ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_policy(arg_policy), m_functor(arg_functor) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif /* #define KOKKOS_HIP_WORKGRAPHPOLICY_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX.cpp b/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX.cpp deleted file mode 100644 index 6027ead01b..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX.cpp +++ /dev/null @@ -1,158 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#ifdef KOKKOS_ENABLE_HPX -#include - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace Kokkos { -namespace Experimental { - -bool HPX::m_hpx_initialized = false; -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) -std::atomic HPX::m_next_instance_id{HPX::impl_default_instance_id() + - 1}; -uint32_t HPX::m_active_parallel_region_count{0}; -hpx::spinlock HPX::m_active_parallel_region_count_mutex; -hpx::condition_variable_any HPX::m_active_parallel_region_count_cond; -HPX::instance_data HPX::m_default_instance_data; -#else -Kokkos::Impl::thread_buffer HPX::m_default_buffer; -#endif - -int HPX::concurrency() { - hpx::runtime *rt = hpx::get_runtime_ptr(); - if (rt == nullptr) { - return hpx::threads::hardware_concurrency(); - } else { - if (hpx::threads::get_self_ptr() == nullptr) { - return hpx::resource::get_thread_pool(0).get_os_thread_count(); - } else { - return hpx::this_thread::get_pool()->get_os_thread_count(); - } - } -} - -void HPX::impl_initialize(InitializationSettings const &settings) { - hpx::runtime *rt = hpx::get_runtime_ptr(); - if (rt == nullptr) { - hpx::local::init_params i; - i.cfg = { -#ifdef KOKKOS_ENABLE_DEBUG - "--hpx:attach-debugger=exception", -#endif - }; - if (settings.has_num_threads()) { - i.cfg.emplace_back("hpx.os_threads=" + - std::to_string(settings.get_num_threads())); - } - int argc_hpx = 1; - char name[] = "kokkos_hpx"; - char *argv_hpx[] = {name, nullptr}; - hpx::local::start(nullptr, argc_hpx, argv_hpx, i); - - m_hpx_initialized = true; - } -} - -bool HPX::impl_is_initialized() noexcept { - hpx::runtime *rt = hpx::get_runtime_ptr(); - return rt != nullptr; -} - -void HPX::impl_finalize() { - if (m_hpx_initialized) { - hpx::runtime *rt = hpx::get_runtime_ptr(); - if (rt != nullptr) { - hpx::apply([]() { hpx::local::finalize(); }); - hpx::local::stop(); - } else { - Kokkos::abort( - "Kokkos::Experimental::HPX::impl_finalize: Kokkos started " - "HPX but something else already stopped HPX\n"); - } - } -} - -} // namespace Experimental - -namespace Impl { - -int g_hpx_space_factory_initialized = - initialize_space_factory("060_HPX"); - -} // namespace Impl - -#ifdef KOKKOS_ENABLE_CXX14 -namespace Tools { -namespace Experimental { -constexpr DeviceType DeviceTypeTraits::id; -} -} // namespace Tools -#endif - -} // namespace Kokkos - -#else -void KOKKOS_CORE_SRC_IMPL_HPX_PREVENT_LINK_ERROR() {} -#endif //#ifdef KOKKOS_ENABLE_HPX diff --git a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_Task.cpp b/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_Task.cpp deleted file mode 100644 index e61ac728a9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_Task.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#if defined(KOKKOS_ENABLE_HPX) && defined(KOKKOS_ENABLE_TASKDAG) - -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template class TaskQueue; - -} // namespace Impl -} // namespace Kokkos - -#else -void KOKKOS_CORE_SRC_IMPL_HPX_TASK_PREVENT_LINK_ERROR() {} -#endif // #if defined( KOKKOS_ENABLE_HPX ) && defined( KOKKOS_ENABLE_TASKDAG ) diff --git a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_Task.hpp b/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_Task.hpp deleted file mode 100644 index 67765a6ae0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_Task.hpp +++ /dev/null @@ -1,306 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HPX_TASK_HPP -#define KOKKOS_HPX_TASK_HPP - -#include -#if defined(KOKKOS_ENABLE_HPX) && defined(KOKKOS_ENABLE_TASKDAG) - -#include - -#include - -#include -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class TaskQueueSpecialization< - SimpleTaskScheduler> { - public: - using execution_space = Kokkos::Experimental::HPX; - using scheduler_type = - SimpleTaskScheduler; - using member_type = - TaskTeamMemberAdapter; - using memory_space = Kokkos::HostSpace; - - static void execute(scheduler_type const &scheduler) { - // NOTE: We create an instance so that we can use dispatch_execute_task. - // This is not necessarily the most efficient, but can be improved later. - TaskQueueSpecialization task_queue; - task_queue.scheduler = &scheduler; - Kokkos::Impl::dispatch_execute_task(&task_queue, - Kokkos::Experimental::HPX()); - Kokkos::Experimental::HPX().fence( - "Kokkos::Impl::TaskQueueSpecialization::execute: fence " - "after task execution"); - } - - // Must provide task queue execution function - void execute_task() const { - // See [note 1] in Kokkos_HPX.hpp for an explanation. The work graph policy - // does not store an execution space instance, so we only need to reset the - // parallel region count here. - Kokkos::Experimental::HPX::reset_count_on_exit_parallel reset_count_on_exit; - - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - using task_base_type = typename scheduler_type::task_base_type; - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - - thread_buffer &buffer = Kokkos::Experimental::HPX().impl_get_buffer(); - buffer.resize(num_worker_threads, 512); - - auto &queue = scheduler->queue(); - - for_loop(par.with(static_chunk_size(1)), 0, num_worker_threads, - [this, &queue, &buffer, num_worker_threads](int) { - // NOTE: This implementation has been simplified based on the - // assumption that team_size = 1. The HPX backend currently only - // supports a team size of 1. - std::size_t t = - Kokkos::Experimental::HPX::impl_hardware_thread_id(); - - buffer.get(t); - HPXTeamMember member( - TeamPolicyInternal( - Kokkos::Experimental::HPX(), num_worker_threads, 1), - 0, t, buffer.get(t), 512); - - member_type single_exec(*scheduler, member); - member_type &team_exec = single_exec; - - auto &team_scheduler = team_exec.scheduler(); - auto current_task = OptionalRef(nullptr); - - while (!queue.is_done()) { - current_task = - queue.pop_ready_task(team_scheduler.team_scheduler_info()); - - if (current_task) { - KOKKOS_ASSERT(current_task->is_single_runnable() || - current_task->is_team_runnable()); - current_task->as_runnable_task().run(single_exec); - queue.complete((*std::move(current_task)).as_runnable_task(), - team_scheduler.team_scheduler_info()); - } - } - }); - } - - static uint32_t get_max_team_count(execution_space const &espace) { - return static_cast(espace.concurrency()); - } - - template - static void get_function_pointer(typename TaskType::function_type &ptr, - typename TaskType::destroy_type &dtor) { - ptr = TaskType::apply; - dtor = TaskType::destroy; - } - - private: - const scheduler_type *scheduler; -}; - -template -class TaskQueueSpecializationConstrained< - Scheduler, - std::enable_if_t::value>> { - public: - using execution_space = Kokkos::Experimental::HPX; - using scheduler_type = Scheduler; - using member_type = - TaskTeamMemberAdapter; - using memory_space = Kokkos::HostSpace; - - static void iff_single_thread_recursive_execute( - scheduler_type const &scheduler) { - using task_base_type = typename scheduler_type::task_base; - using queue_type = typename scheduler_type::queue_type; - - if (1 == Kokkos::Experimental::HPX::concurrency()) { - task_base_type *const end = (task_base_type *)task_base_type::EndTag; - task_base_type *task = end; - - HPXTeamMember member(TeamPolicyInternal( - Kokkos::Experimental::HPX(), 1, 1), - 0, 0, nullptr, 0); - member_type single_exec(scheduler, member); - - do { - task = end; - - // Loop by priority and then type - for (int i = 0; i < queue_type::NumQueue && end == task; ++i) { - for (int j = 0; j < 2 && end == task; ++j) { - task = - queue_type::pop_ready_task(&scheduler.m_queue->m_ready[i][j]); - } - } - - if (end == task) break; - - (*task->m_apply)(task, &single_exec); - - scheduler.m_queue->complete(task); - - } while (true); - } - } - - static void execute(scheduler_type const &scheduler) { - // NOTE: We create an instance so that we can use dispatch_execute_task. - // This is not necessarily the most efficient, but can be improved later. - TaskQueueSpecializationConstrained task_queue; - task_queue.scheduler = &scheduler; - Kokkos::Impl::dispatch_execute_task(&task_queue, - Kokkos::Experimental::HPX()); - Kokkos::Experimental::HPX().fence( - "Kokkos::Impl::TaskQueueSpecialization::execute: fence " - "after task execution"); - } - - // Must provide task queue execution function - void execute_task() const { - // See [note 1] in Kokkos_HPX.hpp for an explanation. The work graph policy - // does not store an execution space instance, so we only need to reset the - // parallel region count here. - Kokkos::Experimental::HPX::reset_count_on_exit_parallel reset_count_on_exit; - - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - - using task_base_type = typename scheduler_type::task_base; - using queue_type = typename scheduler_type::queue_type; - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - static task_base_type *const end = (task_base_type *)task_base_type::EndTag; - constexpr task_base_type *no_more_tasks_sentinel = nullptr; - - thread_buffer &buffer = Kokkos::Experimental::HPX().impl_get_buffer(); - buffer.resize(num_worker_threads, 512); - - auto &queue = scheduler->queue(); - queue.initialize_team_queues(num_worker_threads); - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [this, &buffer, num_worker_threads](int t) { - // NOTE: This implementation has been simplified based on the - // assumption that team_size = 1. The HPX backend currently only - // supports a team size of 1. - buffer.get(Kokkos::Experimental::HPX::impl_hardware_thread_id()); - HPXTeamMember member( - TeamPolicyInternal( - Kokkos::Experimental::HPX(), num_worker_threads, 1), - 0, t, buffer.get(t), 512); - - member_type single_exec(*scheduler, member); - member_type &team_exec = single_exec; - - auto &team_queue = team_exec.scheduler().queue(); - task_base_type *task = no_more_tasks_sentinel; - - do { - if (task != no_more_tasks_sentinel && task != end) { - team_queue.complete(task); - } - - if (*((volatile int *)&team_queue.m_ready_count) > 0) { - task = end; - for (int i = 0; i < queue_type::NumQueue && end == task; ++i) { - for (int j = 0; j < 2 && end == task; ++j) { - task = queue_type::pop_ready_task(&team_queue.m_ready[i][j]); - } - } - } else { - task = team_queue.attempt_to_steal_task(); - } - - if (task != no_more_tasks_sentinel && task != end) { - (*task->m_apply)(task, &single_exec); - } - } while (task != no_more_tasks_sentinel); - }); - } - - template - static void get_function_pointer(typename TaskType::function_type &ptr, - typename TaskType::destroy_type &dtor) { - ptr = TaskType::apply; - dtor = TaskType::destroy; - } - - private: - const scheduler_type *scheduler; -}; - -extern template class TaskQueue< - Kokkos::Experimental::HPX, - typename Kokkos::Experimental::HPX::memory_space>; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #if defined( KOKKOS_ENABLE_TASKDAG ) */ -#endif /* #ifndef KOKKOS_HPX_TASK_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_WorkGraphPolicy.hpp b/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_WorkGraphPolicy.hpp deleted file mode 100644 index 5f2eff5774..0000000000 --- a/bundled/kokkos-3.7.00/core/src/HPX/Kokkos_HPX_WorkGraphPolicy.hpp +++ /dev/null @@ -1,122 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HPX_WORKGRAPHPOLICY_HPP -#define KOKKOS_HPX_WORKGRAPHPOLICY_HPP - -#include - -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Experimental::HPX> { - private: - using Policy = Kokkos::WorkGraphPolicy; - using WorkTag = typename Policy::work_tag; - - Policy m_policy; - FunctorType m_functor; - - template - std::enable_if_t::value> execute_functor( - const std::int32_t w) const noexcept { - m_functor(w); - } - - template - std::enable_if_t::value> execute_functor( - const std::int32_t w) const noexcept { - const TagType t{}; - m_functor(t, w); - } - - public: - void execute() const { - dispatch_execute_task(this, m_policy.space()); - m_policy.space().fence( - "Kokkos::Experimental::Impl::HPX::ParallelFor: fence " - "after kernel execution"); - } - - void execute_task() const { - // See [note 1] in Kokkos_HPX.hpp for an explanation. The work graph policy - // does not store an execution space instance, so we only need to reset the - // parallel region count here. - Kokkos::Experimental::HPX::reset_count_on_exit_parallel reset_count_on_exit; - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - for_loop(par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [this](int) { - std::int32_t w = m_policy.pop_work(); - while (w != Policy::COMPLETED_TOKEN) { - if (w != Policy::END_TOKEN) { - execute_functor(w); - m_policy.completed_work(w); - } - - w = m_policy.pop_work(); - } - }); - } - - inline ParallelFor(const FunctorType &arg_functor, const Policy &arg_policy) - : m_policy(arg_policy), m_functor(arg_functor) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif /* #define KOKKOS_HPX_WORKGRAPHPOLICY_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_DeclareBackend.hpp b/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_DeclareBackend.hpp deleted file mode 100644 index 2d53ef0016..0000000000 --- a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_DeclareBackend.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_DECLARE_HPP_ -#define KOKKOS_DECLARE_HPP_ - -#include - - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_FwdBackend.hpp b/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_FwdBackend.hpp deleted file mode 100644 index 5f1b60f7f8..0000000000 --- a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_FwdBackend.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_FWD_HPP_ -#define KOKKOS_FWD_HPP_ - -#include - - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_PostInclude.hpp b/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_PostInclude.hpp deleted file mode 100644 index 6be71a1300..0000000000 --- a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_PostInclude.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_POST_INCLUDE_HPP_ -#define KOKKOS_POST_INCLUDE_HPP_ - - - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_SetupBackend.hpp b/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_SetupBackend.hpp deleted file mode 100644 index 0e21f065a9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/KokkosCore_Config_SetupBackend.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_SETUP_HPP_ -#define KOKKOS_SETUP_HPP_ - - - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/KokkosCore_config.h b/bundled/kokkos-3.7.00/core/src/KokkosCore_config.h deleted file mode 100644 index 744267c8aa..0000000000 --- a/bundled/kokkos-3.7.00/core/src/KokkosCore_config.h +++ /dev/null @@ -1,111 +0,0 @@ - -#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H) -#error \ - "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead." -#else -#define KOKKOS_CORE_CONFIG_H -#endif - -// KOKKOS_VERSION % 100 is the patch level -// KOKKOS_VERSION / 100 % 100 is the minor version -// KOKKOS_VERSION / 10000 is the major version -#define KOKKOS_VERSION 30700 - -/* Execution Spaces */ -#define KOKKOS_ENABLE_SERIAL -/* #undef KOKKOS_ENABLE_OPENMP */ -/* #undef KOKKOS_ENABLE_OPENACC */ -/* #undef KOKKOS_ENABLE_OPENMPTARGET */ -/* #undef KOKKOS_ENABLE_THREADS */ -/* #undef KOKKOS_ENABLE_CUDA */ -/* #undef KOKKOS_ENABLE_HIP */ -/* #undef KOKKOS_ENABLE_HPX */ -/* #undef KOKKOS_ENABLE_MEMKIND */ -/* #undef KOKKOS_ENABLE_LIBRT */ -/* #undef KOKKOS_ENABLE_SYCL */ - -/* General Settings */ -#define KOKKOS_ENABLE_CXX14 -/* #undef KOKKOS_ENABLE_CXX17 */ -/* #undef KOKKOS_ENABLE_CXX20 */ - -/* #undef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE */ -/* #undef KOKKOS_ENABLE_CUDA_UVM */ -/* #undef KOKKOS_ENABLE_CUDA_LAMBDA */ -/* #undef KOKKOS_ENABLE_CUDA_CONSTEXPR */ -/* #undef KOKKOS_ENABLE_CUDA_LDG_INTRINSIC */ -/* #undef KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC */ -/* #undef KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE */ -/* #undef KOKKOS_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS */ -/* #undef KOKKOS_ENABLE_HPX_ASYNC_DISPATCH */ -/* #undef KOKKOS_ENABLE_DEBUG */ -/* #undef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK */ -/* #undef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK */ -/* #undef KOKKOS_ENABLE_PROFILING_LOAD_PRINT */ -/* #undef KOKKOS_ENABLE_TUNING */ -#define KOKKOS_ENABLE_DEPRECATED_CODE_3 -#define KOKKOS_ENABLE_DEPRECATION_WARNINGS -/* #undef KOKKOS_ENABLE_LARGE_MEM_TESTS */ -#define KOKKOS_ENABLE_COMPLEX_ALIGN -#define KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -/* #undef KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION */ -/* #undef KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION */ - -/* TPL Settings */ -/* #undef KOKKOS_ENABLE_HWLOC */ -/* #undef KOKKOS_USE_LIBRT */ -/* #undef KOKKOS_ENABLE_HBWSPACE */ -/* #undef KOKKOS_ENABLE_LIBDL */ -/* #undef KOKKOS_ENABLE_LIBQUADMATH */ -/* #undef KOKKOS_IMPL_CUDA_CLANG_WORKAROUND */ - -/* #undef KOKKOS_COMPILER_CUDA_VERSION */ - -/* #undef KOKKOS_ARCH_SSE42 */ -/* #undef KOKKOS_ARCH_ARMV80 */ -/* #undef KOKKOS_ARCH_ARMV8_THUNDERX */ -/* #undef KOKKOS_ARCH_ARMV81 */ -/* #undef KOKKOS_ARCH_ARMV8_THUNDERX2 */ -/* #undef KOKKOS_ARCH_AMD_AVX2 */ -/* #undef KOKKOS_ARCH_AVX */ -/* #undef KOKKOS_ARCH_AVX2 */ -/* #undef KOKKOS_ARCH_AVX512XEON */ -/* #undef KOKKOS_ARCH_KNC */ -/* #undef KOKKOS_ARCH_AVX512MIC */ -/* #undef KOKKOS_ARCH_POWER7 */ -/* #undef KOKKOS_ARCH_POWER8 */ -/* #undef KOKKOS_ARCH_POWER9 */ -/* #undef KOKKOS_ARCH_INTEL_GEN */ -/* #undef KOKKOS_ARCH_INTEL_DG1 */ -/* #undef KOKKOS_ARCH_INTEL_GEN9 */ -/* #undef KOKKOS_ARCH_INTEL_GEN11 */ -/* #undef KOKKOS_ARCH_INTEL_GEN12LP */ -/* #undef KOKKOS_ARCH_INTEL_XEHP */ -/* #undef KOKKOS_ARCH_INTEL_GPU */ -/* #undef KOKKOS_ARCH_KEPLER */ -/* #undef KOKKOS_ARCH_KEPLER30 */ -/* #undef KOKKOS_ARCH_KEPLER32 */ -/* #undef KOKKOS_ARCH_KEPLER35 */ -/* #undef KOKKOS_ARCH_KEPLER37 */ -/* #undef KOKKOS_ARCH_MAXWELL */ -/* #undef KOKKOS_ARCH_MAXWELL50 */ -/* #undef KOKKOS_ARCH_MAXWELL52 */ -/* #undef KOKKOS_ARCH_MAXWELL53 */ -/* #undef KOKKOS_ARCH_PASCAL */ -/* #undef KOKKOS_ARCH_PASCAL60 */ -/* #undef KOKKOS_ARCH_PASCAL61 */ -/* #undef KOKKOS_ARCH_VOLTA */ -/* #undef KOKKOS_ARCH_VOLTA70 */ -/* #undef KOKKOS_ARCH_VOLTA72 */ -/* #undef KOKKOS_ARCH_TURING75 */ -/* #undef KOKKOS_ARCH_AMPERE */ -/* #undef KOKKOS_ARCH_AMPERE80 */ -/* #undef KOKKOS_ARCH_AMPERE86 */ -/* #undef KOKKOS_ARCH_AMD_ZEN */ -/* #undef KOKKOS_ARCH_AMD_ZEN2 */ -/* #undef KOKKOS_ARCH_AMD_ZEN3 */ -/* #undef KOKKOS_ARCH_VEGA */ -/* #undef KOKKOS_ARCH_VEGA900 */ -/* #undef KOKKOS_ARCH_VEGA906 */ -/* #undef KOKKOS_ARCH_VEGA908 */ -/* #undef KOKKOS_ARCH_VEGA90A */ diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_AcquireUniqueTokenImpl.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_AcquireUniqueTokenImpl.hpp deleted file mode 100644 index 4a22aedd8c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_AcquireUniqueTokenImpl.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_ACQUIRE_UNIQUE_TOKEN_IMPL_HPP -#define KOKKOS_ACQUIRE_UNIQUE_TOKEN_IMPL_HPP - -#include -#include -namespace Kokkos { -namespace Experimental { - -template -KOKKOS_FUNCTION AcquireTeamUniqueToken::AcquireTeamUniqueToken( - AcquireTeamUniqueToken::token_type t, team_member_type team) - : my_token(t), my_team_acquired_val(team.team_scratch(0)), my_team(team) { - Kokkos::single(Kokkos::PerTeam(my_team), - [&]() { my_team_acquired_val() = my_token.acquire(); }); - my_team.team_barrier(); - - my_acquired_val = my_team_acquired_val(); -} - -template -KOKKOS_FUNCTION AcquireTeamUniqueToken::~AcquireTeamUniqueToken() { - my_team.team_barrier(); - Kokkos::single(Kokkos::PerTeam(my_team), - [&]() { my_token.release(my_acquired_val); }); - my_team.team_barrier(); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif // KOKKOS_UNIQUE_TOKEN_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomic.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Atomic.hpp deleted file mode 100644 index 7a2d1c662c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomic.hpp +++ /dev/null @@ -1,425 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/// \file Kokkos_Atomic.hpp -/// \brief Atomic functions -/// -/// This header file defines prototypes for the following atomic functions: -/// - exchange -/// - compare and exchange -/// - add -/// -/// Supported types include: -/// - signed and unsigned 4 and 8 byte integers -/// - float -/// - double -/// -/// They are implemented through GCC compatible intrinsics, OpenMP -/// directives and native CUDA intrinsics. -/// -/// Including this header file requires one of the following -/// compilers: -/// - NVCC (for CUDA device code only) -/// - GCC (for host code only) -/// - Intel (for host code only) -/// - A compiler that supports OpenMP 3.1 (for host code only) - -#ifndef KOKKOS_ATOMIC_HPP -#define KOKKOS_ATOMIC_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC -#endif - -#include - -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#include -#include -#include - -// Helper functions for places where we really should have called SeqCst atomics -// anyway These can go away when we call desul unconditionally Non-Desul -// versions are below -namespace Kokkos { -namespace Impl { -using desul::MemoryOrderSeqCst; -using desul::MemoryScopeDevice; - -template -KOKKOS_INLINE_FUNCTION void desul_atomic_dec(T* dest, MemoryOrderSeqCst, - MemoryScopeDevice) { - return desul::atomic_dec(const_cast(dest), desul::MemoryOrderSeqCst(), - desul::MemoryScopeDevice()); -} - -template -KOKKOS_INLINE_FUNCTION void desul_atomic_inc(T* dest, MemoryOrderSeqCst, - MemoryScopeDevice) { - return desul::atomic_inc(const_cast(dest), desul::MemoryOrderSeqCst(), - desul::MemoryScopeDevice()); -} - -template -KOKKOS_INLINE_FUNCTION T -desul_atomic_exchange(T* dest, const Kokkos::Impl::identity_t val, - MemoryOrderSeqCst, MemoryScopeDevice) { - return desul::atomic_exchange(const_cast(dest), val, - desul::MemoryOrderSeqCst(), - desul::MemoryScopeDevice()); -} - -template -KOKKOS_INLINE_FUNCTION T desul_atomic_compare_exchange( - T* dest, Kokkos::Impl::identity_t compare, - Kokkos::Impl::identity_t val, MemoryOrderSeqCst, - MemoryScopeDevice) { - return desul::atomic_compare_exchange(dest, compare, val, - desul::MemoryOrderSeqCst(), - desul::MemoryScopeDevice()); -} - -} // namespace Impl -} // namespace Kokkos -#else - -#include -#include - -//---------------------------------------------------------------------------- - -// Need to fix this for pure clang on windows -#if defined(_WIN32) -#define KOKKOS_ENABLE_WINDOWS_ATOMICS - -#if defined(KOKKOS_ENABLE_CUDA) -#define KOKKOS_ENABLE_CUDA_ATOMICS -#if defined(KOKKOS_COMPILER_CLANG) -#define KOKKOS_ENABLE_GNU_ATOMICS -#endif -#endif - -#else // _WIN32 -#if defined(KOKKOS_ENABLE_CUDA) - -// Compiling NVIDIA device code, must use Cuda atomics: - -#define KOKKOS_ENABLE_CUDA_ATOMICS - -#elif defined(KOKKOS_ENABLE_HIP) - -#define KOKKOS_ENABLE_HIP_ATOMICS - -#endif - -#if !defined(KOKKOS_ENABLE_GNU_ATOMICS) && \ - !defined(KOKKOS_ENABLE_INTEL_ATOMICS) && \ - !defined(KOKKOS_ENABLE_OPENMP_ATOMICS) && \ - !defined(KOKKOS_ENABLE_STD_ATOMICS) && \ - !defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -// Compiling for non-Cuda atomic implementation has not been pre-selected. -// Choose the best implementation for the detected compiler. -// Preference: GCC, INTEL, OMP31 - -#if defined(KOKKOS_INTERNAL_NOT_PARALLEL) - -#define KOKKOS_ENABLE_SERIAL_ATOMICS - -#elif defined(KOKKOS_COMPILER_GNU) || defined(KOKKOS_COMPILER_CLANG) || \ - (defined(KOKKOS_COMPILER_NVCC) || defined(KOKKOS_COMPILER_IBM)) - -#define KOKKOS_ENABLE_GNU_ATOMICS - -#elif defined(KOKKOS_COMPILER_INTEL) || defined(KOKKOS_COMPILER_CRAYC) - -#define KOKKOS_ENABLE_INTEL_ATOMICS - -#elif defined(_OPENMP) && (201107 <= _OPENMP) - -#define KOKKOS_ENABLE_OPENMP_ATOMICS - -#else - -#error "KOKKOS_ATOMICS_USE : Unsupported compiler" - -#endif - -#endif /* Not pre-selected atomic implementation */ -#endif - -#ifdef KOKKOS_ENABLE_CUDA -#include -#endif - -namespace Kokkos { -template -KOKKOS_INLINE_FUNCTION void atomic_add(volatile T* const dest, const T src); - -// Atomic increment -template -KOKKOS_INLINE_FUNCTION void atomic_increment(volatile T* a); - -template -KOKKOS_INLINE_FUNCTION void atomic_decrement(volatile T* a); -} // namespace Kokkos - -namespace Kokkos { - -inline const char* atomic_query_version() { -#if defined(KOKKOS_ENABLE_CUDA_ATOMICS) - return "KOKKOS_ENABLE_CUDA_ATOMICS"; -#elif defined(KOKKOS_ENABLE_GNU_ATOMICS) - return "KOKKOS_ENABLE_GNU_ATOMICS"; -#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS) - return "KOKKOS_ENABLE_INTEL_ATOMICS"; -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - return "KOKKOS_ENABLE_OPENMP_ATOMICS"; -#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS) - return "KOKKOS_ENABLE_WINDOWS_ATOMICS"; -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - return "KOKKOS_ENABLE_SERIAL_ATOMICS"; -#else -#error "No valid response for atomic_query_version!" -#endif -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -// Atomic Memory Orders -// -// Implements Strongly-typed analogs of C++ standard memory orders -#include "impl/Kokkos_Atomic_Memory_Order.hpp" - -#if defined(KOKKOS_ENABLE_HIP) -#include -#endif - -#if defined(KOKKOS_ENABLE_WINDOWS_ATOMICS) -#include "impl/Kokkos_Atomic_Windows.hpp" -#endif -//---------------------------------------------------------------------------- -// Atomic Assembly -// -// Implements CAS128-bit in assembly - -#include "impl/Kokkos_Atomic_Assembly.hpp" - -//---------------------------------------------------------------------------- -// Memory fence -// -// All loads and stores from this thread will be globally consistent before -// continuing -// -// void memory_fence() {...}; -#include "impl/Kokkos_Memory_Fence.hpp" - -//---------------------------------------------------------------------------- -// Atomic exchange -// -// template< typename T > -// T atomic_exchange( volatile T* const dest , const T val ) -// { T tmp = *dest ; *dest = val ; return tmp ; } - -#include "impl/Kokkos_Atomic_Exchange.hpp" - -//---------------------------------------------------------------------------- -// Atomic compare-and-exchange -// -// template -// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare, -// const T val) { bool equal = compare == *dest ; if ( equal ) { *dest = val ; } -// return equal ; } - -#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp" - -#include "impl/Kokkos_Atomic_Generic.hpp" - -//---------------------------------------------------------------------------- -// Atomic fetch and add -// -// template -// T atomic_fetch_add(volatile T* const dest, const T val) -// { T tmp = *dest ; *dest += val ; return tmp ; } - -#include "impl/Kokkos_Atomic_Fetch_Add.hpp" - -//---------------------------------------------------------------------------- -// Atomic increment -// -// template -// T atomic_increment(volatile T* const dest) -// { dest++; } - -#include "impl/Kokkos_Atomic_Increment.hpp" - -//---------------------------------------------------------------------------- -// Atomic Decrement -// -// template -// T atomic_decrement(volatile T* const dest) -// { dest--; } - -#include "impl/Kokkos_Atomic_Decrement.hpp" - -//---------------------------------------------------------------------------- -// Atomic fetch and sub -// -// template -// T atomic_fetch_sub(volatile T* const dest, const T val) -// { T tmp = *dest ; *dest -= val ; return tmp ; } - -#include "impl/Kokkos_Atomic_Fetch_Sub.hpp" - -//---------------------------------------------------------------------------- -// Atomic fetch and or -// -// template -// T atomic_fetch_or(volatile T* const dest, const T val) -// { T tmp = *dest ; *dest = tmp | val ; return tmp ; } - -#include "impl/Kokkos_Atomic_Fetch_Or.hpp" - -//---------------------------------------------------------------------------- -// Atomic fetch and and -// -// template -// T atomic_fetch_and(volatile T* const dest, const T val) -// { T tmp = *dest ; *dest = tmp & val ; return tmp ; } - -#include "impl/Kokkos_Atomic_Fetch_And.hpp" - -//---------------------------------------------------------------------------- -// Atomic MinMax -// -// template -// T atomic_min(volatile T* const dest, const T val) -// { T tmp = *dest ; *dest = min(*dest, val); return tmp ; } -// template -// T atomic_max(volatile T* const dest, const T val) -// { T tmp = *dest ; *dest = max(*dest, val); return tmp ; } - -#include "impl/Kokkos_Atomic_MinMax.hpp" - -//---------------------------------------------------------------------------- -// Provide volatile_load and safe_load -// -// T volatile_load(T const volatile * const ptr); -// -// T const& safe_load(T const * const ptr); -// XEON PHI -// T safe_load(T const * const ptr - -#include "impl/Kokkos_Volatile_Load.hpp" - -//---------------------------------------------------------------------------- -// Provide atomic loads and stores with memory order semantics - -#include "impl/Kokkos_Atomic_Load.hpp" -#include "impl/Kokkos_Atomic_Store.hpp" - -// Generic functions using the above defined functions -#include "impl/Kokkos_Atomic_Generic_Secondary.hpp" -//---------------------------------------------------------------------------- -// This atomic-style macro should be an inlined function, not a macro - -#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \ - !defined(__CUDA_ARCH__) - -#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0) -#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0) - -#else - -#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0) -#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0) - -#endif - -//---------------------------------------------------------------------------- - -// Helper functions for places where we really should have called SeqCst atomics -// anyway These can go away when we call desul unconditionally -namespace Kokkos { -namespace Impl { -struct MemoryOrderSeqCst {}; -struct MemoryScopeDevice {}; - -template -KOKKOS_INLINE_FUNCTION void desul_atomic_dec(T* dest, MemoryOrderSeqCst, - MemoryScopeDevice) { - return Kokkos::atomic_decrement(dest); -} - -template -KOKKOS_INLINE_FUNCTION void desul_atomic_inc(T* dest, MemoryOrderSeqCst, - MemoryScopeDevice) { - return Kokkos::atomic_increment(dest); -} - -template -KOKKOS_INLINE_FUNCTION T -desul_atomic_exchange(T* dest, Kokkos::Impl::identity_t val, - MemoryOrderSeqCst, MemoryScopeDevice) { - return Kokkos::atomic_exchange(dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T desul_atomic_compare_exchange( - T* dest, Kokkos::Impl::identity_t compare, - Kokkos::Impl::identity_t val, MemoryOrderSeqCst, - MemoryScopeDevice) { - return Kokkos::atomic_compare_exchange(dest, compare, val); -} - -} // namespace Impl -} // namespace Kokkos - -#endif /* !KOKKOS_ENABLE_IMPL_DESUL_ATOMICS */ -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC -#endif -#endif /* KOKKOS_ATOMIC_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Config.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Config.hpp deleted file mode 100644 index ef576d74c3..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Config.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_ATOMICS_DESUL_CONFIG_HPP -#define KOKKOS_ATOMICS_DESUL_CONFIG_HPP - -#include - -#ifdef KOKKOS_ENABLE_OPENMPTARGET -#define DESUL_HAVE_OPENMP_ATOMICS -#endif - -#if defined(KOKKOS_ARCH_KEPLER) || defined(KOKKOS_ARCH_MAXWELL) || \ - defined(KOKKOS_ARCH_PASCAL) -#define DESUL_CUDA_ARCH_IS_PRE_VOLTA -#endif - -#endif // KOKKOS_ATOMICS_DESUL_CONFIG_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp deleted file mode 100644 index b202ab8f83..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp +++ /dev/null @@ -1,208 +0,0 @@ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_DESUL_ATOMICS_VOLATILE_WRAPPER_HPP_ -#define KOKKOS_DESUL_ATOMICS_VOLATILE_WRAPPER_HPP_ -#include -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#include -#include - -#ifdef KOKKOS_INTERNAL_NOT_PARALLEL -#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller() -#else -#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice() -#endif - -// clang-format off -namespace Kokkos { - -template KOKKOS_INLINE_FUNCTION -T atomic_load(volatile T* const dest) { return desul::atomic_load(const_cast(dest), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_store(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_store(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// atomic_fetch_op -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_add (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_add (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -#ifdef DESUL_IMPL_ATOMIC_CUDA_USE_DOUBLE_ATOMICADD -KOKKOS_INLINE_FUNCTION -double atomic_fetch_add(volatile double* const dest, double val) { - #ifdef __CUDA_ARCH__ - return atomicAdd(const_cast(dest),val); - #else - return desul::atomic_fetch_add (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); - #endif -}; - -KOKKOS_INLINE_FUNCTION -double atomic_fetch_sub(volatile double* const dest, double val) { - #ifdef __CUDA_ARCH__ - return atomicAdd(const_cast(dest),-val); - #else - return desul::atomic_fetch_sub (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); - #endif -}; -#endif - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_sub (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_sub (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_max (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_max (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_min (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_min (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mul (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mul (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_div (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_div (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mod (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mod (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_and (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_and (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_or (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_or (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_xor (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_xor (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_nand(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_nand(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_lshift(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_lshift(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_rshift(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_rshift(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_inc(volatile T* const dest) { return desul::atomic_fetch_inc(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_dec(volatile T* const dest) { return desul::atomic_fetch_dec(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op_fetch -template KOKKOS_INLINE_FUNCTION -T atomic_add_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_sub_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_max_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_min_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mul_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_div_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mod_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mod_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_and_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_and_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_or_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_or_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_xor_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_xor_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_nand_fetch(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_nand_fetch(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_lshift_fetch(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_lshift_fetch(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_rshift_fetch(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_rshift_fetch(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_inc_fetch(volatile T* const dest) { return desul::atomic_inc_fetch(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_dec_fetch(volatile T* const dest) { return desul::atomic_dec_fetch(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op -template KOKKOS_INLINE_FUNCTION -void atomic_add(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_sub(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_mul(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_div(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_min(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_max(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_and yet so call fetch_and -template KOKKOS_INLINE_FUNCTION -void atomic_and(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_and (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_or yet so call fetch_or -template KOKKOS_INLINE_FUNCTION -void atomic_or (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_or (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_inc(volatile T* const dest) { return desul::atomic_inc(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_dec(volatile T* const dest) { return desul::atomic_dec(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_increment(volatile T* const dest) { return desul::atomic_inc(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_decrement(volatile T* const dest) { return desul::atomic_dec(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// Exchange - -template KOKKOS_INLINE_FUNCTION -T atomic_exchange(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_exchange(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -bool atomic_compare_exchange_strong(volatile T* const dest, T& expected, const T desired) { - return desul::atomic_compare_exchange_strong(const_cast(dest),expected, desired, - desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -template KOKKOS_INLINE_FUNCTION -T atomic_compare_exchange(volatile T* const dest, const T compare, const T desired) { - return desul::atomic_compare_exchange(const_cast(dest),compare, desired, - desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -} -#undef KOKKOS_DESUL_MEM_SCOPE - -// clang-format on -#endif // KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Wrapper.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Wrapper.hpp deleted file mode 100644 index ed7e8d9ede..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Atomics_Desul_Wrapper.hpp +++ /dev/null @@ -1,291 +0,0 @@ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_ -#define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_ -#include - -#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#include -#include - -#include -#include - -// clang-format off -namespace Kokkos { - -// FIXME: These functions don't have any use/test in unit tests ... -// ========================================================== -inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; } - -#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \ - !defined(__CUDA_ARCH__) - -#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0) -#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0) - -#else - -#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0) -#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0) - -#endif -// ============================================================ - -#ifdef KOKKOS_INTERNAL_NOT_PARALLEL -#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller() -#else -#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice() -#endif - -template KOKKOS_INLINE_FUNCTION -T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { atomic_store(dest,val); } - -KOKKOS_INLINE_FUNCTION -void memory_fence() { - desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE); -} - -KOKKOS_INLINE_FUNCTION -void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); } - -KOKKOS_INLINE_FUNCTION -void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); } - -// atomic_fetch_op -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -#ifdef DESUL_IMPL_ATOMIC_CUDA_USE_DOUBLE_ATOMICADD -KOKKOS_INLINE_FUNCTION -double atomic_fetch_add(double* const dest, double val) { - #ifdef __CUDA_ARCH__ - return atomicAdd(dest,val); - #else - return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); - #endif -}; - -KOKKOS_INLINE_FUNCTION -double atomic_fetch_sub(double* const dest, double val) { - #ifdef __CUDA_ARCH__ - return atomicAdd(dest,-val); - #else - return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); - #endif -}; -#endif - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op_fetch -template KOKKOS_INLINE_FUNCTION -T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op -template KOKKOS_INLINE_FUNCTION -void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_and yet so call fetch_and -template KOKKOS_INLINE_FUNCTION -void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_or yet so call fetch_or -template KOKKOS_INLINE_FUNCTION -void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// Exchange - -template KOKKOS_INLINE_FUNCTION -T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t expected, desul::Impl::dont_deduce_this_parameter_t desired) { - T expected_ref = expected; - return desul::atomic_compare_exchange_strong(dest, expected_ref, desired, - desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -template KOKKOS_INLINE_FUNCTION -T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t compare, desul::Impl::dont_deduce_this_parameter_t desired) { - return desul::atomic_compare_exchange(dest, compare, desired, - desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -namespace Impl { - - template - struct KokkosToDesulMemoryOrder; - - template<> - struct KokkosToDesulMemoryOrder { - using type = desul::MemoryOrderSeqCst; - }; - template<> - struct KokkosToDesulMemoryOrder { - using type = desul::MemoryOrderAcquire; - }; - template<> - struct KokkosToDesulMemoryOrder { - using type = desul::MemoryOrderRelease; - }; - template<> - struct KokkosToDesulMemoryOrder { - using type = desul::MemoryOrderAcqRel; - }; - template<> - struct KokkosToDesulMemoryOrder { - using type = desul::MemoryOrderRelaxed; - }; - template KOKKOS_INLINE_FUNCTION - bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess, MemOrderFailure) { - return desul::atomic_compare_exchange_strong(dest, expected, desired, - typename KokkosToDesulMemoryOrder::type(), - typename KokkosToDesulMemoryOrder::type(), - KOKKOS_DESUL_MEM_SCOPE); - - } - template - KOKKOS_INLINE_FUNCTION - T atomic_load(const T* const src, MemoryOrder) { - return desul::atomic_load(src, typename KokkosToDesulMemoryOrder::type(), KOKKOS_DESUL_MEM_SCOPE); - } - template - KOKKOS_INLINE_FUNCTION - void atomic_store(T* const src, const T val, MemoryOrder) { - return desul::atomic_store(src, val, typename KokkosToDesulMemoryOrder::type(), KOKKOS_DESUL_MEM_SCOPE); - } -} - -} - -#undef KOKKOS_DESUL_MEM_SCOPE - -// clang-format on -#endif // KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Extents.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Extents.hpp deleted file mode 100644 index c51d663ce9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Extents.hpp +++ /dev/null @@ -1,183 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_KOKKOS_EXTENTS_HPP -#define KOKKOS_KOKKOS_EXTENTS_HPP - -#include -#include -#include - -namespace Kokkos { -namespace Experimental { - -constexpr ptrdiff_t dynamic_extent = -1; - -template -struct Extents { - /* TODO @enhancement flesh this out more */ -}; - -template -struct PrependExtent; - -template -struct PrependExtent, NewExtent> { - using type = Extents; -}; - -template -struct AppendExtent; - -template -struct AppendExtent, NewExtent> { - using type = Extents; -}; - -} // end namespace Experimental - -namespace Impl { - -namespace _parse_view_extents_impl { - -template -struct _all_remaining_extents_dynamic : std::true_type {}; - -template -struct _all_remaining_extents_dynamic : _all_remaining_extents_dynamic { -}; - -template -struct _all_remaining_extents_dynamic : std::false_type {}; - -template -struct _parse_impl { - using type = Result; -}; - -// We have to treat the case of int**[x] specially, since it *doesn't* go -// backwards -template -struct _parse_impl, - std::enable_if_t<_all_remaining_extents_dynamic::value>> - : _parse_impl> { -}; - -// int*(*[x])[y] should still work also (meaning int[][x][][y]) -template -struct _parse_impl< - T*, Kokkos::Experimental::Extents, - std::enable_if_t::value>> { - using _next = Kokkos::Experimental::AppendExtent< - typename _parse_impl, - void>::type, - Kokkos::Experimental::dynamic_extent>; - using type = typename _next::type; -}; - -template -struct _parse_impl, void> - : _parse_impl< - T, Kokkos::Experimental::Extents // TODO @pedantic this - // could be a - // narrowing cast - > {}; - -} // end namespace _parse_view_extents_impl - -template -struct ParseViewExtents { - using type = typename _parse_view_extents_impl ::_parse_impl< - DataType, Kokkos::Experimental::Extents<>>::type; -}; - -template -struct ApplyExtent { - using type = ValueType[Ext]; -}; - -template -struct ApplyExtent { - using type = ValueType*; -}; - -template -struct ApplyExtent { - using type = typename ApplyExtent::type[N]; -}; - -template -struct ApplyExtent { - using type = ValueType * [Ext]; -}; - -template -struct ApplyExtent { - using type = - typename ApplyExtent::type*; -}; - -template -struct ApplyExtent { - using type = - typename ApplyExtent::type[N]; -}; - -} // end namespace Impl - -} // end namespace Kokkos - -#endif // KOKKOS_KOKKOS_EXTENTS_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Graph_fwd.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Graph_fwd.hpp deleted file mode 100644 index 6f639658e1..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Graph_fwd.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_KOKKOS_GRAPH_FWD_HPP -#define KOKKOS_KOKKOS_GRAPH_FWD_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_GRAPH_FWD -#endif - -#include - -namespace Kokkos { -namespace Experimental { - -struct TypeErasedTag {}; - -template -struct Graph; - -template -class GraphNodeRef; - -} // end namespace Experimental -} // end namespace Kokkos - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_GRAPH_FWD -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_GRAPH_FWD -#endif -#endif // KOKKOS_KOKKOS_GRAPH_FWD_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_HBWSpace.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_HBWSpace.hpp deleted file mode 100644 index 0c5dbbdc22..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_HBWSpace.hpp +++ /dev/null @@ -1,380 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_HBWSPACE_HPP -#define KOKKOS_HBWSPACE_HPP - -#include -#ifdef KOKKOS_ENABLE_HBWSPACE - -#include - -namespace Kokkos { - -namespace Experimental { - -namespace Impl { - -/// \brief Initialize lock array for arbitrary size atomics. -/// -/// Arbitrary atomics are implemented using a hash table of locks -/// where the hash value is derived from the address of the -/// object for which an atomic operation is performed. -/// This function initializes the locks to zero (unset). -void init_lock_array_hbw_space(); - -/// \brief Acquire a lock for the address -/// -/// This function tries to acquire the lock for the hash value derived -/// from the provided ptr. If the lock is successfully acquired the -/// function returns true. Otherwise it returns false. -bool lock_address_hbw_space(void* ptr); - -/// \brief Release lock for the address -/// -/// This function releases the lock for the hash value derived -/// from the provided ptr. This function should only be called -/// after previously successfully acquiring a lock with -/// lock_address. -void unlock_address_hbw_space(void* ptr); - -} // namespace Impl - -} // namespace Experimental - -} // namespace Kokkos - -namespace Kokkos { - -namespace Experimental { - -/// \class HBWSpace -/// \brief Memory management for host memory. -/// -/// HBWSpace is a memory space that governs host memory. "Host" -/// memory means the usual CPU-accessible memory. -class HBWSpace { - public: - //! Tag this class as a kokkos memory space - using memory_space = HBWSpace; - using size_type = size_t; - - /// \typedef execution_space - /// \brief Default execution space for this memory space. - /// - /// Every memory space has a default execution space. This is - /// useful for things like initializing a View (which happens in - /// parallel using the View's default execution space). - using execution_space = Kokkos::DefaultHostExecutionSpace; - - //! This memory space preferred device_type - using device_type = Kokkos::Device; - - /**\brief Default memory space instance */ - HBWSpace(); - HBWSpace(const HBWSpace& rhs) = default; - HBWSpace& operator=(const HBWSpace&) = default; - ~HBWSpace() = default; - - /**\brief Non-default memory space instance to choose allocation mechansim, - * if available */ - - enum AllocationMechanism { - STD_MALLOC, - POSIX_MEMALIGN, - POSIX_MMAP, - INTEL_MM_ALLOC - }; - - explicit HBWSpace(const AllocationMechanism&); - - /**\brief Allocate untracked memory in the space */ - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class LogicalMemorySpace; - - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - - public: - /**\brief Return Name of the MemorySpace */ - static constexpr const char* name() { return "HBW"; } - - private: - AllocationMechanism m_alloc_mech; - friend class Kokkos::Impl::SharedAllocationRecord< - Kokkos::Experimental::HBWSpace, void>; -}; - -} // namespace Experimental - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template <> -class SharedAllocationRecord - : public SharedAllocationRecord { - private: - friend Kokkos::Experimental::HBWSpace; - - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - - static void deallocate(RecordBase*); - -#ifdef KOKKOS_ENABLE_DEBUG - /**\brief Root record for tracked allocations from this HBWSpace instance */ - static RecordBase s_root_record; -#endif - - const Kokkos::Experimental::HBWSpace m_space; - - protected: - ~SharedAllocationRecord() -#if defined( \ - KOKKOS_IMPL_INTEL_WORKAROUND_NOEXCEPT_SPECIFICATION_VIRTUAL_FUNCTION) - noexcept -#endif - ; - SharedAllocationRecord() = default; - - SharedAllocationRecord( - const Kokkos::Experimental::HBWSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate); - - public: - inline std::string get_label() const { - return std::string(RecordBase::head()->m_label); - } - - KOKKOS_INLINE_FUNCTION static SharedAllocationRecord* allocate( - const Kokkos::Experimental::HBWSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size) { - KOKKOS_IF_ON_HOST((return new SharedAllocationRecord(arg_space, arg_label, - arg_alloc_size);)) - KOKKOS_IF_ON_DEVICE(((void)arg_space; (void)arg_label; (void)arg_alloc_size; - return nullptr;)) - } - - /**\brief Allocate tracked memory in the space */ - static void* allocate_tracked(const Kokkos::Experimental::HBWSpace& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size); - - /**\brief Reallocate tracked memory in the space */ - static void* reallocate_tracked(void* const arg_alloc_ptr, - const size_t arg_alloc_size); - - /**\brief Deallocate tracked memory in the space */ - static void deallocate_tracked(void* const arg_alloc_ptr); - - static SharedAllocationRecord* get_record(void* arg_alloc_ptr); - - static void print_records(std::ostream&, - const Kokkos::Experimental::HBWSpace&, - bool detail = false); -}; - -} // namespace Impl - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -static_assert( - Kokkos::Impl::MemorySpaceAccess::assignable, - ""); - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = true }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -} // namespace Impl - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template <> -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const DefaultHostExecutionSpace& exec, void* dst, const void* src, - size_t n) { - hostspace_parallel_deepcopy(exec, dst, src, n); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const DefaultHostExecutionSpace& exec, void* dst, const void* src, - size_t n) { - hostspace_parallel_deepcopy(exec, dst, src, n); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"); - hostspace_parallel_deepcopy_async(copy_space, dst, src, n); - } -}; - -template <> -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const DefaultHostExecutionSpace& exec, void* dst, const void* src, - size_t n) { - hostspace_parallel_deepcopy(exec, dst, src, n); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"); - hostspace_parallel_deepcopy_async(dst, src, n); - } -}; - -} // namespace Impl - -} // namespace Kokkos - -#endif -#endif // #define KOKKOS_HBWSPACE_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_HIP.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_HIP.hpp deleted file mode 100644 index c387b5945d..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_HIP.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_HIP_HPP -#define KOKKOS_HIP_HPP - -#include - -#if defined(KOKKOS_ENABLE_HIP) - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_HIP_Space.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_HIP_Space.hpp deleted file mode 100644 index 8c195a0f39..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_HIP_Space.hpp +++ /dev/null @@ -1,806 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_HIPSPACE_HPP -#define KOKKOS_HIPSPACE_HPP - -#include - -#if defined(KOKKOS_ENABLE_HIP) - -#include -#include -#include -#include -#include - -#include -#include -#include -#include // HIP_SAFE_CALL - -#include -#include -#include - -#include -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Impl { - -template -struct is_hip_type_space : public std::false_type {}; - -} // namespace Impl - -namespace Experimental { -/** \brief HIP on-device memory management */ - -class HIPSpace { - public: - //! Tag this class as a kokkos memory space - using memory_space = HIPSpace; - using execution_space = Kokkos::Experimental::HIP; - using device_type = Kokkos::Device; - - using size_type = unsigned int; - - /*--------------------------------*/ - - HIPSpace(); - HIPSpace(HIPSpace&& rhs) = default; - HIPSpace(const HIPSpace& rhs) = default; - HIPSpace& operator=(HIPSpace&& rhs) = default; - HIPSpace& operator=(const HIPSpace& rhs) = default; - ~HIPSpace() = default; - - /**\brief Allocate untracked memory in the hip space */ - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the hip space */ - void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class LogicalMemorySpace; - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - - public: - /**\brief Return Name of the MemorySpace */ - static constexpr const char* name() { return "HIP"; } - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - /*--------------------------------*/ - /** \brief Error reporting for HostSpace attempt to access HIPSpace */ - KOKKOS_DEPRECATED static void access_error(); - KOKKOS_DEPRECATED static void access_error(const void* const); -#endif - - private: - int m_device; ///< Which HIP device - - friend class Kokkos::Impl::SharedAllocationRecord< - Kokkos::Experimental::HIPSpace, void>; -}; - -} // namespace Experimental - -template <> -struct Impl::is_hip_type_space : public std::true_type { -}; - -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Experimental { -/** \brief Host memory that is accessible to HIP execution space - * through HIP's host-pinned memory allocation. - */ -class HIPHostPinnedSpace { - public: - //! Tag this class as a kokkos memory space - /** \brief Memory is in HostSpace so use the HostSpace::execution_space */ - using execution_space = HostSpace::execution_space; - using memory_space = HIPHostPinnedSpace; - using device_type = Kokkos::Device; - using size_type = unsigned int; - - /*--------------------------------*/ - - HIPHostPinnedSpace(); - HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) = default; - HIPHostPinnedSpace(const HIPHostPinnedSpace& rhs) = default; - HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) = default; - HIPHostPinnedSpace& operator=(const HIPHostPinnedSpace& rhs) = default; - ~HIPHostPinnedSpace() = default; - - /**\brief Allocate untracked memory in the space */ - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class LogicalMemorySpace; - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - - public: - /**\brief Return Name of the MemorySpace */ - static constexpr const char* name() { return "HIPHostPinned"; } - - /*--------------------------------*/ -}; -} // namespace Experimental - -template <> -struct Impl::is_hip_type_space - : public std::true_type {}; - -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Experimental { -/** \brief Memory that is accessible to HIP execution space - * and host through HIP's memory page migration. - */ -class HIPManagedSpace { - public: - //! Tag this class as a kokkos memory space - /** \brief Memory is unified to both device and host via page migration - * and therefore able to be used by HostSpace::execution_space and - * DeviceSpace::execution_space. - */ - //! tag this class as a kokkos memory space - using memory_space = HIPManagedSpace; - using execution_space = Kokkos::Experimental::HIP; - using device_type = Kokkos::Device; - using size_type = unsigned int; - - /*--------------------------------*/ - - HIPManagedSpace(); - HIPManagedSpace(HIPManagedSpace&& rhs) = default; - HIPManagedSpace(const HIPManagedSpace& rhs) = default; - HIPManagedSpace& operator=(HIPManagedSpace&& rhs) = default; - HIPManagedSpace& operator=(const HIPManagedSpace& rhs) = default; - ~HIPManagedSpace() = default; - - /**\brief Allocate untracked memory in the space */ - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - int m_device; ///< Which HIP device - template - friend class LogicalMemorySpace; - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - - public: - /**\brief Return Name of the MemorySpace */ - static constexpr const char* name() { return "HIPManaged"; } - - /*--------------------------------*/ -}; -} // namespace Experimental - -template <> -struct Impl::is_hip_type_space - : public std::true_type {}; - -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Impl { - -static_assert( - Kokkos::Impl::MemorySpaceAccess::assignable, - ""); - -//---------------------------------------- - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // HostSpace::execution_space == HIPHostPinnedSpace::execution_space - enum : bool { assignable = true }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // HostSpace::execution_space != HIPManagedSpace::execution_space - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -//---------------------------------------- - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // HIPSpace::execution_space != HIPHostPinnedSpace::execution_space - enum : bool { assignable = false }; - enum : bool { accessible = true }; // HIPSpace::execution_space - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // HIPSpace::execution_space == HIPManagedSpace::execution_space - enum : bool { assignable = true }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -//---------------------------------------- -// HIPHostPinnedSpace::execution_space == HostSpace::execution_space -// HIPHostPinnedSpace accessible to both HIP and Host - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // Cannot access from HIP - enum : bool { accessible = true }; // HIPHostPinnedSpace::execution_space - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // Cannot access from Host - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // different exec_space - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -//---------------------------------------- -// HIPManagedSpace::execution_space != HostSpace::execution_space -// HIPManagedSpace accessible to both HIP and Host - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; // HIPHostPinnedSpace::execution_space - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // different exec_space - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -}; // namespace Impl -//---------------------------------------- - -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Impl { - -void DeepCopyHIP(void* dst, const void* src, size_t n); -void DeepCopyAsyncHIP(const Kokkos::Experimental::HIP& instance, void* dst, - const void* src, size_t n); -void DeepCopyAsyncHIP(void* dst, const void* src, size_t n); - -template -struct DeepCopy::value>> { - DeepCopy(void* dst, const void* src, size_t n) { DeepCopyHIP(dst, src, n); } - DeepCopy(const Kokkos::Experimental::HIP& instance, void* dst, - const void* src, size_t n) { - DeepCopyAsyncHIP(instance, dst, src, n); - } -}; - -template -struct DeepCopy::value>> { - DeepCopy(void* dst, const void* src, size_t n) { DeepCopyHIP(dst, src, n); } - DeepCopy(const Kokkos::Experimental::HIP& instance, void* dst, - const void* src, size_t n) { - DeepCopyAsyncHIP(instance, dst, src, n); - } -}; - -template -struct DeepCopy::value && - is_hip_type_space::value>> { - DeepCopy(void* dst, const void* src, size_t n) { DeepCopyHIP(dst, src, n); } - DeepCopy(const Kokkos::Experimental::HIP& instance, void* dst, - const void* src, size_t n) { - DeepCopyAsyncHIP(instance, dst, src, n); - } -}; - -template -struct DeepCopy< - MemSpace1, MemSpace2, ExecutionSpace, - std::enable_if_t< - is_hip_type_space::value && - is_hip_type_space::value && - !std::is_same::value>> { - inline DeepCopy(void* dst, const void* src, size_t n) { - DeepCopyHIP(dst, src, n); - } - - inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, - size_t n) { - exec.fence(fence_string()); - DeepCopyAsyncHIP(dst, src, n); - } - - private: - static const std::string& fence_string() { - static const std::string string = - std::string("Kokkos::Impl::DeepCopy<") + MemSpace1::name() + "Space, " + - MemSpace2::name() + - "Space, ExecutionSpace>::DeepCopy: fence before copy"; - return string; - } -}; - -template -struct DeepCopy< - MemSpace, HostSpace, ExecutionSpace, - std::enable_if_t< - is_hip_type_space::value && - !std::is_same::value>> { - inline DeepCopy(void* dst, const void* src, size_t n) { - DeepCopyHIP(dst, src, n); - } - - inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, - size_t n) { - exec.fence(fence_string()); - DeepCopyAsyncHIP(dst, src, n); - } - - private: - static const std::string& fence_string() { - static const std::string string = - std::string("Kokkos::Impl::DeepCopy<") + MemSpace::name() + - "Space, HostSpace, ExecutionSpace>::DeepCopy: fence before copy"; - return string; - } -}; - -template -struct DeepCopy< - HostSpace, MemSpace, ExecutionSpace, - std::enable_if_t< - is_hip_type_space::value && - !std::is_same::value>> { - inline DeepCopy(void* dst, const void* src, size_t n) { - DeepCopyHIP(dst, src, n); - } - - inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, - size_t n) { - exec.fence(fence_string()); - DeepCopyAsyncHIP(dst, src, n); - } - - private: - static const std::string& fence_string() { - static const std::string string = - std::string("Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"; - return string; - } -}; -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template <> -class SharedAllocationRecord - : public HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::HIPSpace> { - private: - friend class SharedAllocationRecordCommon; - friend class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::HIPSpace>; - using base_t = HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::HIPSpace>; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - -#ifdef KOKKOS_ENABLE_DEBUG - static RecordBase s_root_record; -#endif - - const Kokkos::Experimental::HIPSpace m_space; - - protected: - ~SharedAllocationRecord(); - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec*/, - const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::HIP& exec_space, - const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); - - SharedAllocationRecord( - const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); -}; - -template <> -class SharedAllocationRecord - : public SharedAllocationRecordCommon< - Kokkos::Experimental::HIPHostPinnedSpace> { - private: - friend class SharedAllocationRecordCommon< - Kokkos::Experimental::HIPHostPinnedSpace>; - using base_t = - SharedAllocationRecordCommon; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - -#ifdef KOKKOS_ENABLE_DEBUG - static RecordBase s_root_record; -#endif - - const Kokkos::Experimental::HIPHostPinnedSpace m_space; - - protected: - ~SharedAllocationRecord(); - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::HIPHostPinnedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::HIPHostPinnedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); -}; - -template <> -class SharedAllocationRecord - : public SharedAllocationRecordCommon< - Kokkos::Experimental::HIPManagedSpace> { - private: - friend class SharedAllocationRecordCommon< - Kokkos::Experimental::HIPManagedSpace>; - using base_t = - SharedAllocationRecordCommon; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - -#ifdef KOKKOS_ENABLE_DEBUG - static RecordBase s_root_record; -#endif - - const Kokkos::Experimental::HIPManagedSpace m_space; - - protected: - ~SharedAllocationRecord(); - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::HIPManagedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::HIPManagedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); -}; -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Experimental { -namespace Impl { -class HIPInternal; -} -/// \class HIP -/// \brief Kokkos device for multicore processors in the host memory space. -class HIP { - public: - //------------------------------------ - //! \name Type declarations that all Kokkos devices must provide. - //@{ - - //! Tag this class as a kokkos execution space - using execution_space = HIP; - using memory_space = HIPSpace; - using device_type = Kokkos::Device; - - using array_layout = LayoutLeft; - using size_type = HIPSpace::size_type; - - using scratch_memory_space = ScratchMemorySpace; - - HIP(); - HIP(hipStream_t stream, bool manage_stream = false); - - //@} - //------------------------------------ - //! \name Functions that all Kokkos devices must implement. - //@{ - - KOKKOS_INLINE_FUNCTION static int in_parallel() { -#if defined(__HIP_DEVICE_COMPILE__) - return true; -#else - return false; -#endif - } - - /** \brief Wait until all dispatched functors complete. - * - * The parallel_for or parallel_reduce dispatch of a functor may return - * asynchronously, before the functor completes. This method does not return - * until all dispatched functors on this device have completed. - */ - static void impl_static_fence(const std::string& name); - - void fence(const std::string& name = - "Kokkos::HIP::fence(): Unnamed Instance Fence") const; - - hipStream_t hip_stream() const; - - /// \brief Print configuration information to the given output stream. - void print_configuration(std::ostream& os, bool verbose = false) const; - - /// \brief Free any resources being consumed by the device. - static void impl_finalize(); - - /** \brief Initialize the device. - * - */ - int hip_device() const; - static hipDeviceProp_t const& hip_device_prop(); - - static void impl_initialize(InitializationSettings const&); - - static int impl_is_initialized(); - - // static size_type device_arch(); - - static size_type detect_device_count(); - - static int concurrency(); - static const char* name(); - - inline Impl::HIPInternal* impl_internal_space_instance() const { - return m_space_instance.get(); - } - - uint32_t impl_instance_id() const noexcept; - - private: - Kokkos::Impl::HostSharedPtr m_space_instance; -}; -} // namespace Experimental -namespace Tools { -namespace Experimental { -template <> -struct DeviceTypeTraits { - static constexpr DeviceType id = DeviceType::HIP; - static int device_id(const Kokkos::Experimental::HIP& exec) { - return exec.hip_device(); - } -}; -} // namespace Experimental -} // namespace Tools - -namespace Impl { -template -struct ZeroMemset { - ZeroMemset(const Kokkos::Experimental::HIP& exec_space, - const View& dst, - typename View::const_value_type&) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemsetAsync( - dst.data(), 0, - dst.size() * sizeof(typename View::value_type), - exec_space.hip_stream())); - } - - ZeroMemset(const View& dst, - typename View::const_value_type&) { - KOKKOS_IMPL_HIP_SAFE_CALL( - hipMemset(dst.data(), 0, - dst.size() * sizeof(typename View::value_type))); - } -}; -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = false }; -}; - -} // namespace Impl -} // namespace Kokkos - -#endif /* #if defined( KOKKOS_ENABLE_HIP ) */ -#endif /* #define KOKKOS_HIPSPACE_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_HPX.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_HPX.hpp deleted file mode 100644 index 044e54fb29..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_HPX.hpp +++ /dev/null @@ -1,2395 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_HPX_HPP -#define KOKKOS_HPX_HPP - -#include -#if defined(KOKKOS_ENABLE_HPX) - -#include - -#include -#include -#include - -#ifdef KOKKOS_ENABLE_HBWSPACE -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -// There are currently two different implementations for the parallel dispatch -// functions: -// -// - 0: The HPX way. Unfortunately, this comes with unnecessary -// overheads at the moment, so there is -// - 1: The manual way. This uses for_loop, but only spawns one task per worker -// thread. This is significantly faster in most cases. -// -// In the long run 0 should be the preferred implementation, but until HPX is -// improved 1 will be the default. -#ifndef KOKKOS_HPX_IMPLEMENTATION -#define KOKKOS_HPX_IMPLEMENTATION 1 -#endif - -#if (KOKKOS_HPX_IMPLEMENTATION < 0) || (KOKKOS_HPX_IMPLEMENTATION > 1) -#error "You have chosen an invalid value for KOKKOS_HPX_IMPLEMENTATION" -#endif - -// [note 1] -// -// When using the asynchronous backend and independent instances, we explicitly -// reset the shared data at the end of a parallel task (execute_task). We do -// this to avoid circular references with shared pointers that would otherwise -// never be released. -// -// The HPX instance holds shared data for the instance in a shared_ptr. One of -// the pieces of shared data is the future that we use to sequence parallel -// dispatches. When a parallel task is launched, a copy of the closure -// (ParallelFor, ParallelReduce, etc.) is captured in the task. The closure -// also holds the policy, the policy holds the HPX instance, the instance holds -// the shared data (for use of buffers in the parallel task). When attaching a -// continuation to a future, the continuation is stored in the future (shared -// state). This means that there is a cycle future -> continuation -> closure -// -> policy -> HPX -> shared data -> future. We break this by releasing the -// shared data early, as (the pointer to) the shared data will not be used -// anymore by the closure at the end of execute_task. -// -// We also mark the shared instance data as mutable so that we can reset it -// from the const execute_task member function. - -namespace Kokkos { -namespace Impl { -class thread_buffer { - static constexpr std::size_t m_cache_line_size = 64; - - std::size_t m_num_threads; - std::size_t m_size_per_thread; - std::size_t m_size_total; - char *m_data; - - void pad_to_cache_line(std::size_t &size) { - size = ((size + m_cache_line_size - 1) / m_cache_line_size) * - m_cache_line_size; - } - - public: - thread_buffer() - : m_num_threads(0), - m_size_per_thread(0), - m_size_total(0), - m_data(nullptr) {} - thread_buffer(const std::size_t num_threads, - const std::size_t size_per_thread) { - resize(num_threads, size_per_thread); - } - ~thread_buffer() { delete[] m_data; } - - thread_buffer(const thread_buffer &) = delete; - thread_buffer(thread_buffer &&) = delete; - thread_buffer &operator=(const thread_buffer &) = delete; - thread_buffer &operator=(thread_buffer) = delete; - - void resize(const std::size_t num_threads, - const std::size_t size_per_thread) { - m_num_threads = num_threads; - m_size_per_thread = size_per_thread; - - pad_to_cache_line(m_size_per_thread); - - std::size_t size_total_new = m_num_threads * m_size_per_thread; - - if (m_size_total < size_total_new) { - delete[] m_data; - m_data = new char[size_total_new]; - m_size_total = size_total_new; - } - } - - char *get(std::size_t thread_num) { - assert(thread_num < m_num_threads); - if (m_data == nullptr) { - return nullptr; - } - return &m_data[thread_num * m_size_per_thread]; - } - - std::size_t size_per_thread() const noexcept { return m_size_per_thread; } - std::size_t size_total() const noexcept { return m_size_total; } -}; -} // namespace Impl - -namespace Experimental { -class HPX { - public: - static constexpr uint32_t impl_default_instance_id() { return 1; } - - private: - static bool m_hpx_initialized; - uint32_t m_instance_id = impl_default_instance_id(); - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - static std::atomic m_next_instance_id; - - public: - enum class instance_mode { default_, independent }; - - private: - static uint32_t m_active_parallel_region_count; - static hpx::spinlock m_active_parallel_region_count_mutex; - static hpx::condition_variable_any m_active_parallel_region_count_cond; - - struct instance_data { - instance_data() = default; - instance_data(hpx::shared_future future) : m_future(future) {} - Kokkos::Impl::thread_buffer m_buffer; - hpx::shared_future m_future = hpx::make_ready_future(); - hpx::spinlock m_future_mutex; - }; - - mutable std::shared_ptr m_independent_instance_data; - static instance_data m_default_instance_data; - - std::reference_wrapper m_buffer; - std::reference_wrapper> m_future; - std::reference_wrapper m_future_mutex; -#else - static Kokkos::Impl::thread_buffer m_default_buffer; -#endif - - public: - using execution_space = HPX; - using memory_space = HostSpace; - using device_type = Kokkos::Device; - using array_layout = LayoutRight; - using size_type = memory_space::size_type; - using scratch_memory_space = ScratchMemorySpace; - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - HPX() - noexcept - : m_instance_id(impl_default_instance_id()), - m_buffer(m_default_instance_data.m_buffer), - m_future(m_default_instance_data.m_future), - m_future_mutex(m_default_instance_data.m_future_mutex) {} - - HPX(instance_mode mode) - : m_instance_id(mode == instance_mode::independent - ? m_next_instance_id++ - : impl_default_instance_id()), - m_independent_instance_data(mode == instance_mode::independent - ? (new instance_data()) - : nullptr), - m_buffer(mode == instance_mode::independent - ? m_independent_instance_data->m_buffer - : m_default_instance_data.m_buffer), - m_future(mode == instance_mode::independent - ? m_independent_instance_data->m_future - : m_default_instance_data.m_future), - m_future_mutex(mode == instance_mode::independent - ? m_independent_instance_data->m_future_mutex - : m_default_instance_data.m_future_mutex) {} - - HPX(hpx::shared_future future) - : m_instance_id(m_next_instance_id++), - - m_independent_instance_data(new instance_data(future)), - m_buffer(m_independent_instance_data->m_buffer), - m_future(m_independent_instance_data->m_future), - m_future_mutex(m_independent_instance_data->m_future_mutex) {} - - HPX(HPX &&other) = default; - HPX &operator=(HPX &&other) = default; - HPX(const HPX &other) = default; - HPX &operator=(const HPX &other) = default; -#else - HPX() noexcept {} -#endif - - void print_configuration(std::ostream &os, bool /*verbose*/ = false) const { - os << "HPX backend\n"; - os << "HPX Execution Space:\n"; - os << " KOKKOS_ENABLE_HPX: yes\n"; - os << "\nHPX Runtime Configuration:\n"; - } - uint32_t impl_instance_id() const noexcept { return m_instance_id; } - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - static bool in_parallel(HPX const &instance = HPX()) noexcept { - return !instance.impl_get_future().is_ready(); - } -#else - static bool in_parallel(HPX const & = HPX()) noexcept { return false; } -#endif - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - static void impl_decrement_active_parallel_region_count() { - std::unique_lock l(m_active_parallel_region_count_mutex); - if (--m_active_parallel_region_count == 0) { - l.unlock(); - m_active_parallel_region_count_cond.notify_all(); - }; - } - - static void impl_increment_active_parallel_region_count() { - std::unique_lock l(m_active_parallel_region_count_mutex); - ++m_active_parallel_region_count; - } -#endif - - void fence( - const std::string &name = - "Kokkos::Experimental::HPX::fence: Unnamed Instance Fence") const { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::HPX>( - name, - Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{ - impl_instance_id()}, - [&]() { -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - impl_get_future().wait(); - // Reset the future to free variables that may have been captured in - // parallel regions. - impl_get_future() = hpx::make_ready_future(); -#endif - }); - } - - static void impl_static_fence(const std::string &name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::HPX>( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - [&]() { -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - std::unique_lock l( - m_active_parallel_region_count_mutex); - m_active_parallel_region_count_cond.wait( - l, [&]() { return m_active_parallel_region_count == 0; }); - // Reset the future to free variables that may have been captured in - // parallel regions (however, we don't have access to futures from - // instances other than the default instances, they will only be - // released by fence). - HPX().impl_get_future() = hpx::make_ready_future(); -#endif - }); - } - - static hpx::execution::parallel_executor impl_get_executor() { - return hpx::execution::parallel_executor(); - } - - static bool is_asynchronous(HPX const & = HPX()) noexcept { -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - return true; -#else - return false; -#endif - } - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - static std::vector partition(...) { - Kokkos::abort( - "Kokkos::Experimental::HPX::partition_master: can't partition an HPX " - "instance\n"); - return std::vector(); - } - - template - KOKKOS_DEPRECATED static void partition_master( - F const &, int requested_num_partitions = 0, int = 0) { - if (requested_num_partitions > 1) { - Kokkos::abort( - "Kokkos::Experimental::HPX::partition_master: can't partition an " - "HPX instance\n"); - } - } -#endif - - static int concurrency(); - static void impl_initialize(InitializationSettings const &); - static bool impl_is_initialized() noexcept; - static void impl_finalize(); - - static int impl_thread_pool_size() noexcept { - hpx::runtime *rt = hpx::get_runtime_ptr(); - if (rt == nullptr) { - return 0; - } else { - if (hpx::threads::get_self_ptr() == nullptr) { - return hpx::resource::get_thread_pool(0).get_os_thread_count(); - } else { - return hpx::this_thread::get_pool()->get_os_thread_count(); - } - } - } - - static int impl_thread_pool_rank() noexcept { - hpx::runtime *rt = hpx::get_runtime_ptr(); - if (rt == nullptr) { - return 0; - } else { - if (hpx::threads::get_self_ptr() == nullptr) { - return 0; - } else { - return hpx::this_thread::get_pool()->get_pool_index(); - } - } - } - - static int impl_thread_pool_size(int depth) { - if (depth == 0) { - return impl_thread_pool_size(); - } else { - return 1; - } - } - - static int impl_max_hardware_threads() noexcept { - return hpx::threads::hardware_concurrency(); - } - - static int impl_hardware_thread_id() noexcept { - return hpx::get_worker_thread_num(); - } - - Kokkos::Impl::thread_buffer &impl_get_buffer() const noexcept { -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - return m_buffer.get(); -#else - return m_default_buffer; -#endif - } - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - hpx::shared_future &impl_get_future() const noexcept { - return m_future; - } - - hpx::spinlock &impl_get_future_mutex() const noexcept { - return m_future_mutex; - } -#endif - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) - struct KOKKOS_ATTRIBUTE_NODISCARD reset_on_exit_parallel { - HPX const &m_space; - reset_on_exit_parallel(HPX const &space) : m_space(space) {} - ~reset_on_exit_parallel() { - // See [note 1] for an explanation. m_independent_instance_data is - // marked mutable. - m_space.m_independent_instance_data.reset(); - - HPX::impl_decrement_active_parallel_region_count(); - } - }; - - // This struct is identical to the above except it does not reset the shared - // data. It does, however, still decrement the parallel region count. It is - // meant for use in parallel regions which do not capture the execution space - // instance. - struct KOKKOS_ATTRIBUTE_NODISCARD reset_count_on_exit_parallel { - reset_count_on_exit_parallel() {} - ~reset_count_on_exit_parallel() { - HPX::impl_decrement_active_parallel_region_count(); - } - }; -#else - struct KOKKOS_ATTRIBUTE_NODISCARD reset_on_exit_parallel { - reset_on_exit_parallel(HPX const &) {} - ~reset_on_exit_parallel() {} - }; - - struct KOKKOS_ATTRIBUTE_NODISCARD reset_count_on_exit_parallel { - reset_count_on_exit_parallel() {} - ~reset_count_on_exit_parallel() {} - }; -#endif - - static constexpr const char *name() noexcept { return "HPX"; } -}; -} // namespace Experimental - -namespace Tools { -namespace Experimental { -template <> -struct DeviceTypeTraits { - static constexpr DeviceType id = DeviceType::HPX; - static int device_id(const Kokkos::Experimental::HPX &) { return 0; } -}; -} // namespace Experimental -} // namespace Tools - -namespace Impl { - -#if defined(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH) -template -inline void dispatch_execute_task(Closure *closure, - Kokkos::Experimental::HPX const &instance, - bool force_synchronous = false) { - Kokkos::Experimental::HPX::impl_increment_active_parallel_region_count(); - - Closure closure_copy = *closure; - - { - std::unique_lock l(instance.impl_get_future_mutex()); - hpx::util::ignore_lock(&instance.impl_get_future_mutex()); - hpx::shared_future &fut = instance.impl_get_future(); - - fut = fut.then(hpx::execution::parallel_executor( - hpx::threads::thread_schedule_hint(0)), - [closure_copy](hpx::shared_future &&) { - return closure_copy.execute_task(); - }); - } - - if (force_synchronous) { - instance.fence( - "Kokkos::Experimental::Impl::HPX::dispatch_execute_task: fence due to " - "forced syncronizations"); - } -} -#else -template -inline void dispatch_execute_task(Closure *closure, - Kokkos::Experimental::HPX const &, - bool = false) { - closure->execute_task(); -} -#endif -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = false }; -}; - -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Experimental { -template <> -class UniqueToken { - private: - using buffer_type = Kokkos::View; - int m_count; - buffer_type m_buffer_view; - uint32_t volatile *m_buffer; - - public: - using execution_space = HPX; - using size_type = int; - - /// \brief create object size for concurrency on the given instance - /// - /// This object should not be shared between instances - UniqueToken(execution_space const & = execution_space()) noexcept - : m_count(execution_space::impl_max_hardware_threads()), - m_buffer_view(buffer_type()), - m_buffer(nullptr) {} - - UniqueToken(size_type max_size, execution_space const & = execution_space()) - : m_count(max_size > execution_space::impl_max_hardware_threads() - ? execution_space::impl_max_hardware_threads() - : max_size), - m_buffer_view( - max_size > execution_space::impl_max_hardware_threads() - ? buffer_type() - : buffer_type("UniqueToken::m_buffer_view", - ::Kokkos::Impl::concurrent_bitset::buffer_bound( - m_count))), - m_buffer(m_buffer_view.data()) {} - - /// \brief upper bound for acquired values, i.e. 0 <= value < size() - KOKKOS_INLINE_FUNCTION - int size() const noexcept { return m_count; } - - /// \brief acquire value such that 0 <= value < size() - KOKKOS_INLINE_FUNCTION - int acquire() const noexcept { - KOKKOS_IF_ON_HOST(( - if (m_buffer == nullptr) { - return execution_space::impl_hardware_thread_id(); - } else { - const ::Kokkos::pair result = - ::Kokkos::Impl::concurrent_bitset::acquire_bounded( - m_buffer, m_count, ::Kokkos::Impl::clock_tic() % m_count); - - if (result.first < 0) { - ::Kokkos::abort( - "UniqueToken failure to acquire tokens, no tokens " - "available"); - } - return result.first; - })) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /// \brief release a value acquired by generate - KOKKOS_INLINE_FUNCTION - void release(int i) const noexcept { - KOKKOS_IF_ON_HOST((if (m_buffer != nullptr) { - ::Kokkos::Impl::concurrent_bitset::release(m_buffer, i); - })) - - KOKKOS_IF_ON_DEVICE(((void)i;)) - } -}; - -template <> -class UniqueToken { - public: - using execution_space = HPX; - using size_type = int; - UniqueToken(execution_space const & = execution_space()) noexcept {} - - // NOTE: Currently this assumes that there is no oversubscription. - // hpx::get_num_worker_threads can't be used directly because it may yield - // it's task (problematic if called after hpx::get_worker_thread_num). - int size() const noexcept { return HPX::impl_max_hardware_threads(); } - int acquire() const noexcept { return HPX::impl_hardware_thread_id(); } - void release(int) const noexcept {} -}; -} // namespace Experimental -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -struct HPXTeamMember { - public: - using execution_space = Kokkos::Experimental::HPX; - using scratch_memory_space = - Kokkos::ScratchMemorySpace; - - private: - scratch_memory_space m_team_shared; - - int m_league_size; - int m_league_rank; - int m_team_size; - int m_team_rank; - - public: - KOKKOS_INLINE_FUNCTION - const scratch_memory_space &team_shmem() const { - return m_team_shared.set_team_thread_mode(0, 1, 0); - } - - KOKKOS_INLINE_FUNCTION - const execution_space::scratch_memory_space &team_scratch(const int) const { - return m_team_shared.set_team_thread_mode(0, 1, 0); - } - - KOKKOS_INLINE_FUNCTION - const execution_space::scratch_memory_space &thread_scratch(const int) const { - return m_team_shared.set_team_thread_mode(0, team_size(), team_rank()); - } - - KOKKOS_INLINE_FUNCTION int league_rank() const noexcept { - return m_league_rank; - } - - KOKKOS_INLINE_FUNCTION int league_size() const noexcept { - return m_league_size; - } - - KOKKOS_INLINE_FUNCTION int team_rank() const noexcept { return m_team_rank; } - KOKKOS_INLINE_FUNCTION int team_size() const noexcept { return m_team_size; } - - template - constexpr KOKKOS_INLINE_FUNCTION HPXTeamMember( - const TeamPolicyInternal - &policy, - const int team_rank, const int league_rank, void *scratch, - size_t scratch_size) noexcept - : m_team_shared(scratch, scratch_size, scratch, scratch_size), - m_league_size(policy.league_size()), - m_league_rank(league_rank), - m_team_size(policy.team_size()), - m_team_rank(team_rank) {} - - KOKKOS_INLINE_FUNCTION - void team_barrier() const {} - - template - KOKKOS_INLINE_FUNCTION void team_broadcast(ValueType &, const int &) const {} - - template - KOKKOS_INLINE_FUNCTION void team_broadcast(const Closure &closure, - ValueType &value, - const int &) const { - closure(value); - } - - template - KOKKOS_INLINE_FUNCTION ValueType team_reduce(const ValueType &value, - const JoinOp &) const { - return value; - } - - template - KOKKOS_INLINE_FUNCTION std::enable_if_t::value> - team_reduce(const ReducerType &) const {} - - template - KOKKOS_INLINE_FUNCTION Type - team_scan(const Type &value, Type *const global_accum = nullptr) const { - if (global_accum) { - Kokkos::atomic_fetch_add(global_accum, value); - } - - return 0; - } -}; - -template -class TeamPolicyInternal - : public PolicyTraits { - int m_league_size; - int m_team_size; - std::size_t m_team_scratch_size[2]; - std::size_t m_thread_scratch_size[2]; - int m_chunk_size; - - public: - using traits = PolicyTraits; - - //! Tag this class as a kokkos execution policy - using execution_policy = TeamPolicyInternal; - - using member_type = HPXTeamMember; - - //! Execution space of this execution policy: - using execution_space = Kokkos::Experimental::HPX; - - // NOTE: Max size is 1 for simplicity. In most cases more than 1 is not - // necessary on CPU. Implement later if there is a need. - template - inline static int team_size_max(const FunctorType &) { - return 1; - } - - template - inline static int team_size_recommended(const FunctorType &) { - return 1; - } - - template - inline static int team_size_recommended(const FunctorType &, const int &) { - return 1; - } - - template - int team_size_max(const FunctorType &, const ParallelForTag &) const { - return 1; - } - - template - int team_size_max(const FunctorType &, const ParallelReduceTag &) const { - return 1; - } - - template - int team_size_max(const FunctorType &, const ReducerType &, - const ParallelReduceTag &) const { - return 1; - } - - template - int team_size_recommended(const FunctorType &, const ParallelForTag &) const { - return 1; - } - - template - int team_size_recommended(const FunctorType &, - const ParallelReduceTag &) const { - return 1; - } - - template - int team_size_recommended(const FunctorType &, const ReducerType &, - const ParallelReduceTag &) const { - return 1; - } - - static int vector_length_max() { return 1; } - - inline int impl_vector_length() noexcept { return 1; } - inline bool impl_auto_team_size() noexcept { return false; } - inline bool impl_auto_vector_length() noexcept { return false; } - inline void impl_set_vector_length(int) noexcept {} - inline void impl_set_team_size(int) noexcept {} - - private: - inline void init(const int league_size_request, const int team_size_request) { - m_league_size = league_size_request; - const int max_team_size = 1; // TODO: Can't use team_size_max(...) because - // it requires a functor as argument. - m_team_size = - team_size_request > max_team_size ? max_team_size : team_size_request; - - if (m_chunk_size > 0) { - if (!Impl::is_integral_power_of_two(m_chunk_size)) - Kokkos::abort("TeamPolicy blocking granularity must be power of two"); - } else { - int new_chunk_size = 1; - while (new_chunk_size * 4 * Kokkos::Experimental::HPX::concurrency() < - m_league_size) { - new_chunk_size *= 2; - } - - if (new_chunk_size < 128) { - new_chunk_size = 1; - while ((new_chunk_size * Kokkos::Experimental::HPX::concurrency() < - m_league_size) && - (new_chunk_size < 128)) - new_chunk_size *= 2; - } - - m_chunk_size = new_chunk_size; - } - } - - public: - inline int team_size() const { return m_team_size; } - inline int league_size() const { return m_league_size; } - - size_t scratch_size(const int &level, int team_size_ = -1) const { - if (team_size_ < 0) { - team_size_ = m_team_size; - } - return m_team_scratch_size[level] + - team_size_ * m_thread_scratch_size[level]; - } - - inline static int scratch_size_max(int level) { - return (level == 0 ? 1024 * 32 : // Roughly L1 size - 20 * 1024 * 1024); // Limit to keep compatibility with CUDA - } - - public: - template - friend class TeamPolicyInternal; - - const typename traits::execution_space &space() const { - static typename traits::execution_space m_space; - return m_space; - } - - template - TeamPolicyInternal(const TeamPolicyInternal &p) { - m_league_size = p.m_league_size; - m_team_size = p.m_team_size; - m_team_scratch_size[0] = p.m_team_scratch_size[0]; - m_thread_scratch_size[0] = p.m_thread_scratch_size[0]; - m_team_scratch_size[1] = p.m_team_scratch_size[1]; - m_thread_scratch_size[1] = p.m_thread_scratch_size[1]; - m_chunk_size = p.m_chunk_size; - } - - TeamPolicyInternal(const typename traits::execution_space &, - int league_size_request, int team_size_request, - int /* vector_length_request */ = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, team_size_request); - } - - TeamPolicyInternal(const typename traits::execution_space &, - int league_size_request, const Kokkos::AUTO_t &, - int /* vector_length_request */ = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, 1); - } - - TeamPolicyInternal(const typename traits::execution_space &, - int league_size_request, - const Kokkos::AUTO_t &, /* team_size_request */ - const Kokkos::AUTO_t & /* vector_length_request */) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, 1); - } - - TeamPolicyInternal(const typename traits::execution_space &, - int league_size_request, int team_size_request, - const Kokkos::AUTO_t & /* vector_length_request */ - ) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, team_size_request); - } - - TeamPolicyInternal(int league_size_request, - const Kokkos::AUTO_t &, /* team_size_request */ - const Kokkos::AUTO_t & /* vector_length_request */) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, 1); - } - - TeamPolicyInternal(int league_size_request, int team_size_request, - const Kokkos::AUTO_t & /* vector_length_request */ - ) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, team_size_request); - } - - TeamPolicyInternal(int league_size_request, int team_size_request, - int /* vector_length_request */ = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, team_size_request); - } - - TeamPolicyInternal(int league_size_request, const Kokkos::AUTO_t &, - int /* vector_length_request */ = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(0) { - init(league_size_request, 1); - } - - inline int chunk_size() const { return m_chunk_size; } - - inline TeamPolicyInternal &set_chunk_size( - typename traits::index_type chunk_size_) { - m_chunk_size = chunk_size_; - return *this; - } - - inline TeamPolicyInternal &set_scratch_size(const int &level, - const PerTeamValue &per_team) { - m_team_scratch_size[level] = per_team.value; - return *this; - } - - inline TeamPolicyInternal &set_scratch_size( - const int &level, const PerThreadValue &per_thread) { - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - inline TeamPolicyInternal &set_scratch_size( - const int &level, const PerTeamValue &per_team, - const PerThreadValue &per_thread) { - m_team_scratch_size[level] = per_team.value; - m_thread_scratch_size[level] = per_thread.value; - return *this; - } -}; -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -template -typename Policy::member_type get_hpx_adjusted_chunk_size(Policy const &policy) { - const int concurrency = Kokkos::Experimental::HPX::concurrency(); - const typename Policy::member_type n = policy.end() - policy.begin(); - typename Policy::member_type new_chunk_size = policy.chunk_size(); - - while (n >= 4 * concurrency * new_chunk_size) { - new_chunk_size *= 2; - } - - return new_chunk_size; -} - -template -class ParallelFor, - Kokkos::Experimental::HPX> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - const FunctorType m_functor; - const Policy m_policy; - - template - static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Member i) { - functor(i); - } - - template - static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Member i) { - const TagType t{}; - functor(t, i); - } - - template - static std::enable_if_t::value> execute_functor_range( - const FunctorType &functor, const Member i_begin, const Member i_end) { - for (Member i = i_begin; i < i_end; ++i) { - functor(i); - } - } - - template - static std::enable_if_t::value> execute_functor_range( - const FunctorType &functor, const Member i_begin, const Member i_end) { - const TagType t{}; - for (Member i = i_begin; i < i_end; ++i) { - functor(t, i); - } - } - - public: - void execute() const { - Kokkos::Impl::dispatch_execute_task(this, m_policy.space()); - } - - void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_policy.space()); - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - using hpx::execution::par; - using hpx::execution::static_chunk_size; - -#if KOKKOS_HPX_IMPLEMENTATION == 0 - using hpx::for_loop; - - for_loop(par.on(exec).with(static_chunk_size(m_policy.chunk_size())), - m_policy.begin(), m_policy.end(), [this](const Member i) { - execute_functor(m_functor, i); - }); - -#elif KOKKOS_HPX_IMPLEMENTATION == 1 - using hpx::for_loop_strided; - - const Member chunk_size = get_hpx_adjusted_chunk_size(m_policy); - - for_loop_strided( - par.on(exec), m_policy.begin(), m_policy.end(), chunk_size, - [this, chunk_size](const Member i_begin) { - const Member i_end = (std::min)(i_begin + chunk_size, m_policy.end()); - execute_functor_range(m_functor, i_begin, i_end); - }); -#endif - } - - inline ParallelFor(const FunctorType &arg_functor, Policy arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} -}; - -template -class ParallelFor, - Kokkos::Experimental::HPX> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - using WorkTag = typename MDRangePolicy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - using iterate_type = - typename Kokkos::Impl::HostIterateTile; - - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; - - public: - void execute() const { dispatch_execute_task(this, m_mdr_policy.space()); } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_mdr_policy.space()); - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - using hpx::execution::par; - using hpx::execution::static_chunk_size; - -#if KOKKOS_HPX_IMPLEMENTATION == 0 - using hpx::for_loop; - - for_loop(par.on(exec).with( - static_chunk_size(get_hpx_adjusted_chunk_size(m_policy))), - m_policy.begin(), m_policy.end(), [this](const Member i) { - iterate_type(m_mdr_policy, m_functor)(i); - }); - -#elif KOKKOS_HPX_IMPLEMENTATION == 1 - using hpx::for_loop_strided; - - const Member chunk_size = get_hpx_adjusted_chunk_size(m_policy); - - for_loop_strided(par.on(exec), m_policy.begin(), m_policy.end(), chunk_size, - [this, chunk_size](const Member i_begin) { - const Member i_end = - (std::min)(i_begin + chunk_size, m_policy.end()); - for (Member i = i_begin; i < i_end; ++i) { - iterate_type(m_mdr_policy, m_functor)(i); - } - }); -#endif - } - - inline ParallelFor(const FunctorType &arg_functor, MDRangePolicy arg_policy) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, arg_policy.m_num_tiles).set_chunk_size(1)) {} - template - static int max_tile_size_product(const Policy &, const Functor &) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } -}; -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::HPX> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = - FunctorAnalysis; - using value_type = typename Analysis::value_type; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - bool m_force_synchronous; - - template - inline static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Member i, reference_type update) { - functor(i, update); - } - - template - inline static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Member i, reference_type update) { - const TagType t{}; - functor(t, i, update); - } - - template - inline std::enable_if_t::value> execute_functor_range( - reference_type update, const Member i_begin, const Member i_end) const { - for (Member i = i_begin; i < i_end; ++i) { - m_functor(i, update); - } - } - - template - inline std::enable_if_t::value> execute_functor_range( - reference_type update, const Member i_begin, const Member i_end) const { - const TagType t{}; - - for (Member i = i_begin; i < i_end; ++i) { - m_functor(t, i, update); - } - } - - class value_type_wrapper { - private: - std::size_t m_value_size; - char *m_value_buffer; - - public: - value_type_wrapper() : m_value_size(0), m_value_buffer(nullptr) {} - - value_type_wrapper(const std::size_t value_size) - : m_value_size(value_size), m_value_buffer(new char[m_value_size]) {} - - value_type_wrapper(const value_type_wrapper &other) - : m_value_size(0), m_value_buffer(nullptr) { - if (this != &other) { - m_value_buffer = new char[other.m_value_size]; - m_value_size = other.m_value_size; - - std::copy(other.m_value_buffer, other.m_value_buffer + m_value_size, - m_value_buffer); - } - } - - ~value_type_wrapper() { delete[] m_value_buffer; } - - value_type_wrapper(value_type_wrapper &&other) - : m_value_size(0), m_value_buffer(nullptr) { - if (this != &other) { - m_value_buffer = other.m_value_buffer; - m_value_size = other.m_value_size; - - other.m_value_buffer = nullptr; - other.m_value_size = 0; - } - } - - value_type_wrapper &operator=(const value_type_wrapper &other) { - if (this != &other) { - delete[] m_value_buffer; - m_value_buffer = new char[other.m_value_size]; - m_value_size = other.m_value_size; - - std::copy(other.m_value_buffer, other.m_value_buffer + m_value_size, - m_value_buffer); - } - - return *this; - } - - value_type_wrapper &operator=(value_type_wrapper &&other) { - if (this != &other) { - delete[] m_value_buffer; - m_value_buffer = other.m_value_buffer; - m_value_size = other.m_value_size; - - other.m_value_buffer = nullptr; - other.m_value_size = 0; - } - - return *this; - } - - pointer_type pointer() const { - return reinterpret_cast(m_value_buffer); - } - - reference_type reference() const { - return Analysis::Reducer::reference( - reinterpret_cast(m_value_buffer)); - } - }; - - public: - void execute() const { - if (m_policy.end() <= m_policy.begin()) { - if (m_result_ptr) { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - final_reducer.init(m_result_ptr); - final_reducer.final(m_result_ptr); - } - return; - } - dispatch_execute_task(this, m_policy.space(), m_force_synchronous); - } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_policy.space()); - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - const std::size_t value_size = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - -#if KOKKOS_HPX_IMPLEMENTATION == 0 - // NOTE: This version makes the most use of HPX functionality, but - // requires the struct value_type_wrapper to handle different - // reference_types. It is also significantly slower than the version - // below due to not reusing the buffer used by other functions. - using hpx::parallel::reduction; - - value_type_wrapper final_value(value_size); - value_type_wrapper identity(value_size); - - final_reducer.init(final_value.pointer()); - final_reducer.init(identity.pointer()); - - for_loop(par.on(exec).with( - static_chunk_size(get_hpx_adjusted_chunk_size(m_policy))), - m_policy.begin(), m_policy.end(), - reduction(final_value, identity, - [final_reducer]( - value_type_wrapper &a, - value_type_wrapper &b) -> value_type_wrapper & { - final_reducer.join(a.pointer(), b.pointer()); - return a; - }), - [this](Member i, value_type_wrapper &update) { - execute_functor(m_functor, i, update.reference()); - }); - - pointer_type final_value_ptr = final_value.pointer(); - -#elif KOKKOS_HPX_IMPLEMENTATION == 1 - using hpx::for_loop_strided; - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - - thread_buffer &buffer = m_policy.space().impl_get_buffer(); - buffer.resize(num_worker_threads, value_size); - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [&buffer, final_reducer ](const int t) noexcept { - final_reducer.init(reinterpret_cast(buffer.get(t))); - }); - - const Member chunk_size = get_hpx_adjusted_chunk_size(m_policy); - - for_loop_strided( - par.on(exec), m_policy.begin(), m_policy.end(), chunk_size, - [this, &buffer, chunk_size](const Member i_begin) { - reference_type update = Analysis::Reducer::reference( - reinterpret_cast(buffer.get( - Kokkos::Experimental::HPX::impl_hardware_thread_id()))); - const Member i_end = (std::min)(i_begin + chunk_size, m_policy.end()); - execute_functor_range(update, i_begin, i_end); - }); - - for (int i = 1; i < num_worker_threads; ++i) { - final_reducer.join(reinterpret_cast(buffer.get(0)), - reinterpret_cast(buffer.get(i))); - } - - pointer_type final_value_ptr = - reinterpret_cast(buffer.get(0)); -#endif - - final_reducer.final(final_value_ptr); - - if (m_result_ptr != nullptr) { - const int n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - - for (int j = 0; j < n; ++j) { - m_result_ptr[j] = final_value_ptr[j]; - } - } - } - - template - inline ParallelReduce( - const FunctorType &arg_functor, Policy arg_policy, - const ViewType &arg_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void *> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_view.data()), - m_force_synchronous(!arg_view.impl_track().has_record()) {} - - inline ParallelReduce(const FunctorType &arg_functor, Policy arg_policy, - const ReducerType &reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_force_synchronous(!reducer.view().impl_track().has_record()) {} -}; - -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::HPX> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - using WorkTag = typename MDRangePolicy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - using iterate_type = - typename Kokkos::Impl::HostIterateTile; - - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - bool m_force_synchronous; - - public: - void execute() const { - dispatch_execute_task(this, m_mdr_policy.space(), m_force_synchronous); - } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_mdr_policy.space()); - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - const std::size_t value_size = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - - thread_buffer &buffer = m_mdr_policy.space().impl_get_buffer(); - buffer.resize(num_worker_threads, value_size); - - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - -#if KOKKOS_HPX_IMPLEMENTATION == 0 - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [&buffer, final_reducer](std::size_t t) { - final_reducer.init(reinterpret_cast(buffer.get(t))); - }); - - for_loop(par.on(exec).with( - static_chunk_size(get_hpx_adjusted_chunk_size(m_policy))), - m_policy.begin(), m_policy.end(), [this, &buffer](const Member i) { - reference_type update = Analysis::Reducer::reference( - reinterpret_cast(buffer.get( - Kokkos::Experimental::HPX::impl_hardware_thread_id()))); - iterate_type(m_mdr_policy, m_functor, update)(i); - }); - -#elif KOKKOS_HPX_IMPLEMENTATION == 1 - using hpx::for_loop_strided; - - for_loop( - par.on(exec).with(static_chunk_size(1)), std::size_t(0), - num_worker_threads, [&buffer, final_reducer](const std::size_t t) { - final_reducer.init(reinterpret_cast(buffer.get(t))); - }); - - const Member chunk_size = get_hpx_adjusted_chunk_size(m_policy); - - for_loop_strided( - par.on(exec), m_policy.begin(), m_policy.end(), chunk_size, - [this, &buffer, chunk_size](const Member i_begin) { - reference_type update = Analysis::Reducer::reference( - reinterpret_cast(buffer.get( - Kokkos::Experimental::HPX::impl_hardware_thread_id()))); - const Member i_end = (std::min)(i_begin + chunk_size, m_policy.end()); - - for (Member i = i_begin; i < i_end; ++i) { - iterate_type(m_mdr_policy, m_functor, update)(i); - } - }); -#endif - - for (int i = 1; i < num_worker_threads; ++i) { - final_reducer.join(reinterpret_cast(buffer.get(0)), - reinterpret_cast(buffer.get(i))); - } - - final_reducer.final(reinterpret_cast(buffer.get(0))); - - if (m_result_ptr != nullptr) { - const int n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - - for (int j = 0; j < n; ++j) { - m_result_ptr[j] = reinterpret_cast(buffer.get(0))[j]; - } - } - } - - template - inline ParallelReduce( - const FunctorType &arg_functor, MDRangePolicy arg_policy, - const ViewType &arg_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void *> = nullptr) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, arg_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(InvalidType()), - m_result_ptr(arg_view.data()), - m_force_synchronous(!arg_view.impl_track().has_record()) {} - - inline ParallelReduce(const FunctorType &arg_functor, - MDRangePolicy arg_policy, const ReducerType &reducer) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_force_synchronous(!reducer.view().impl_track().has_record()) {} - template - static int max_tile_size_product(const Policy &, const Functor &) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } -}; -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -template -class ParallelScan, - Kokkos::Experimental::HPX> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - using Analysis = - FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - using value_type = typename Analysis::value_type; - - const FunctorType m_functor; - const Policy m_policy; - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Member i_begin, - const Member i_end, reference_type update, - const bool final) { - for (Member i = i_begin; i < i_end; ++i) { - functor(i, update, final); - } - } - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Member i_begin, - const Member i_end, reference_type update, - const bool final) { - const TagType t{}; - for (Member i = i_begin; i < i_end; ++i) { - functor(t, i, update, final); - } - } - - public: - void execute() const { dispatch_execute_task(this, m_policy.space()); } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_policy.space()); - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - const int value_count = Analysis::value_count(m_functor); - const std::size_t value_size = Analysis::value_size(m_functor); - - thread_buffer &buffer = m_policy.space().impl_get_buffer(); - buffer.resize(num_worker_threads, 2 * value_size); - - using hpx::barrier; - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - - barrier<> bar(num_worker_threads); - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - typename Analysis::Reducer final_reducer(&m_functor); - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [this, &bar, &buffer, num_worker_threads, value_count, value_size, - final_reducer](int t) { - reference_type update_sum = - final_reducer.init(reinterpret_cast(buffer.get(t))); - - const WorkRange range(m_policy, t, num_worker_threads); - execute_functor_range(m_functor, range.begin(), range.end(), - update_sum, false); - - bar.arrive_and_wait(); - - if (t == 0) { - final_reducer.init( - reinterpret_cast(buffer.get(0) + value_size)); - - for (int i = 1; i < num_worker_threads; ++i) { - pointer_type ptr_1_prev = - reinterpret_cast(buffer.get(i - 1)); - pointer_type ptr_2_prev = reinterpret_cast( - buffer.get(i - 1) + value_size); - pointer_type ptr_2 = - reinterpret_cast(buffer.get(i) + value_size); - - for (int j = 0; j < value_count; ++j) { - ptr_2[j] = ptr_2_prev[j]; - } - - final_reducer.join(ptr_2, ptr_1_prev); - } - } - - bar.arrive_and_wait(); - - reference_type update_base = Analysis::Reducer::reference( - reinterpret_cast(buffer.get(t) + value_size)); - - execute_functor_range(m_functor, range.begin(), range.end(), - update_base, true); - }); - } - - inline ParallelScan(const FunctorType &arg_functor, const Policy &arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} -}; - -template -class ParallelScanWithTotal, - ReturnType, Kokkos::Experimental::HPX> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - using Analysis = - FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - using value_type = typename Analysis::value_type; - - const FunctorType m_functor; - const Policy m_policy; - ReturnType &m_returnvalue; - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Member i_begin, - const Member i_end, reference_type update, - const bool final) { - for (Member i = i_begin; i < i_end; ++i) { - functor(i, update, final); - } - } - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Member i_begin, - const Member i_end, reference_type update, - const bool final) { - const TagType t{}; - for (Member i = i_begin; i < i_end; ++i) { - functor(t, i, update, final); - } - } - - public: - void execute() const { dispatch_execute_task(this, m_policy.space()); } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_policy.space()); - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - const int value_count = Analysis::value_count(m_functor); - const std::size_t value_size = Analysis::value_size(m_functor); - - thread_buffer &buffer = m_policy.space().impl_get_buffer(); - buffer.resize(num_worker_threads, 2 * value_size); - - using hpx::barrier; - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - - barrier<> bar(num_worker_threads); - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - typename Analysis::Reducer final_reducer(&m_functor); - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [this, &bar, &buffer, num_worker_threads, value_count, value_size, - final_reducer](int t) { - reference_type update_sum = - final_reducer.init(reinterpret_cast(buffer.get(t))); - - const WorkRange range(m_policy, t, num_worker_threads); - execute_functor_range(m_functor, range.begin(), range.end(), - update_sum, false); - - bar.arrive_and_wait(); - - if (t == 0) { - final_reducer.init( - reinterpret_cast(buffer.get(0) + value_size)); - - for (int i = 1; i < num_worker_threads; ++i) { - pointer_type ptr_1_prev = - reinterpret_cast(buffer.get(i - 1)); - pointer_type ptr_2_prev = reinterpret_cast( - buffer.get(i - 1) + value_size); - pointer_type ptr_2 = - reinterpret_cast(buffer.get(i) + value_size); - - for (int j = 0; j < value_count; ++j) { - ptr_2[j] = ptr_2_prev[j]; - } - - final_reducer.join(ptr_2, ptr_1_prev); - } - } - - bar.arrive_and_wait(); - - reference_type update_base = Analysis::Reducer::reference( - reinterpret_cast(buffer.get(t) + value_size)); - - execute_functor_range(m_functor, range.begin(), range.end(), - update_base, true); - - if (t == num_worker_threads - 1) { - m_returnvalue = update_base; - } - }); - } - - inline ParallelScanWithTotal(const FunctorType &arg_functor, - const Policy &arg_policy, - ReturnType &arg_returnvalue) - : m_functor(arg_functor), - m_policy(arg_policy), - m_returnvalue(arg_returnvalue) {} -}; -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { -template -class ParallelFor, - Kokkos::Experimental::HPX> { - private: - using Policy = TeamPolicyInternal; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using memory_space = Kokkos::HostSpace; - - const FunctorType m_functor; - const Policy m_policy; - const int m_league; - const std::size_t m_shared; - - template - inline static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Policy &policy, const int league_rank, - char *local_buffer, const std::size_t local_buffer_size) { - functor(Member(policy, 0, league_rank, local_buffer, local_buffer_size)); - } - - template - inline static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Policy &policy, const int league_rank, - char *local_buffer, const std::size_t local_buffer_size) { - const TagType t{}; - functor(t, Member(policy, 0, league_rank, local_buffer, local_buffer_size)); - } - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Policy &policy, - const int league_rank_begin, const int league_rank_end, - char *local_buffer, - const std::size_t local_buffer_size) { - for (int league_rank = league_rank_begin; league_rank < league_rank_end; - ++league_rank) { - functor(Member(policy, 0, league_rank, local_buffer, local_buffer_size)); - } - } - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Policy &policy, - const int league_rank_begin, const int league_rank_end, - char *local_buffer, - const std::size_t local_buffer_size) { - const TagType t{}; - for (int league_rank = league_rank_begin; league_rank < league_rank_end; - ++league_rank) { - functor(t, - Member(policy, 0, league_rank, local_buffer, local_buffer_size)); - } - } - - public: - void execute() const { dispatch_execute_task(this, m_policy.space()); } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_policy.space()); - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - - thread_buffer &buffer = m_policy.space().impl_get_buffer(); - buffer.resize(num_worker_threads, m_shared); - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - using hpx::execution::par; - using hpx::execution::static_chunk_size; - -#if KOKKOS_HPX_IMPLEMENTATION == 0 - using hpx::for_loop; - - for_loop( - par.on(exec).with(static_chunk_size(m_policy.chunk_size())), 0, - m_policy.league_size(), [this, &buffer](const int league_rank) { - execute_functor( - m_functor, m_policy, league_rank, - buffer.get(Kokkos::Experimental::HPX::impl_hardware_thread_id()), - m_shared); - }); - -#elif KOKKOS_HPX_IMPLEMENTATION == 1 - using hpx::for_loop_strided; - - for_loop_strided( - par.on(exec), 0, m_policy.league_size(), m_policy.chunk_size(), - [this, &buffer](const int league_rank_begin) { - const int league_rank_end = - (std::min)(league_rank_begin + m_policy.chunk_size(), - m_policy.league_size()); - execute_functor_range( - m_functor, m_policy, league_rank_begin, league_rank_end, - buffer.get(Kokkos::Experimental::HPX::impl_hardware_thread_id()), - m_shared); - }); -#endif - } - - ParallelFor(const FunctorType &arg_functor, const Policy &arg_policy) - : m_functor(arg_functor), - m_policy(arg_policy), - m_league(arg_policy.league_size()), - m_shared(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) {} -}; - -template -class ParallelReduce, - ReducerType, Kokkos::Experimental::HPX> { - private: - using Policy = TeamPolicyInternal; - using Member = typename Policy::member_type; - using WorkTag = typename Policy::work_tag; - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = - FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - using value_type = typename Analysis::value_type; - - const FunctorType m_functor; - const int m_league; - const Policy m_policy; - const ReducerType m_reducer; - pointer_type m_result_ptr; - const std::size_t m_shared; - - bool m_force_synchronous; - - template - inline static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Policy &policy, const int league_rank, - char *local_buffer, const std::size_t local_buffer_size, - reference_type update) { - functor(Member(policy, 0, league_rank, local_buffer, local_buffer_size), - update); - } - - template - inline static std::enable_if_t::value> execute_functor( - const FunctorType &functor, const Policy &policy, const int league_rank, - char *local_buffer, const std::size_t local_buffer_size, - reference_type update) { - const TagType t{}; - functor(t, Member(policy, 0, league_rank, local_buffer, local_buffer_size), - update); - } - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Policy &policy, - const int league_rank_begin, const int league_rank_end, - char *local_buffer, const std::size_t local_buffer_size, - reference_type update) { - for (int league_rank = league_rank_begin; league_rank < league_rank_end; - ++league_rank) { - functor(Member(policy, 0, league_rank, local_buffer, local_buffer_size), - update); - } - } - - template - inline static std::enable_if_t::value> - execute_functor_range(const FunctorType &functor, const Policy &policy, - const int league_rank_begin, const int league_rank_end, - char *local_buffer, const std::size_t local_buffer_size, - reference_type update) { - const TagType t{}; - for (int league_rank = league_rank_begin; league_rank < league_rank_end; - ++league_rank) { - functor(t, - Member(policy, 0, league_rank, local_buffer, local_buffer_size), - update); - } - } - - public: - void execute() const { - if (m_policy.league_size() * m_policy.team_size() == 0) { - if (m_result_ptr) { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - final_reducer.init(m_result_ptr); - final_reducer.final(m_result_ptr); - } - return; - } - dispatch_execute_task(this, m_policy.space()); - } - - inline void execute_task() const { - // See [note 1] for an explanation. - Kokkos::Experimental::HPX::reset_on_exit_parallel reset_on_exit( - m_policy.space()); - - const int num_worker_threads = Kokkos::Experimental::HPX::concurrency(); - const std::size_t value_size = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - - thread_buffer &buffer = m_policy.space().impl_get_buffer(); - buffer.resize(num_worker_threads, value_size + m_shared); - - auto exec = Kokkos::Experimental::HPX::impl_get_executor(); - - using hpx::for_loop; - using hpx::execution::par; - using hpx::execution::static_chunk_size; - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - -#if KOKKOS_HPX_IMPLEMENTATION == 0 - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [&buffer, final_reducer](const std::size_t t) { - final_reducer.init(reinterpret_cast(buffer.get(t))); - }); - - for_loop(par.on(exec).with(static_chunk_size(m_policy.chunk_size())), 0, - m_policy.league_size(), - [this, &buffer, value_size](const int league_rank) { - std::size_t t = - Kokkos::Experimental::HPX::impl_hardware_thread_id(); - reference_type update = Analysis::Reducer::reference( - reinterpret_cast(buffer.get(t))); - - execute_functor(m_functor, m_policy, league_rank, - buffer.get(t) + value_size, m_shared, - update); - }); - -#elif KOKKOS_HPX_IMPLEMENTATION == 1 - using hpx::for_loop_strided; - - for_loop( - par.on(exec).with(static_chunk_size(1)), 0, num_worker_threads, - [&buffer, final_reducer](std::size_t const t) { - final_reducer.init(reinterpret_cast(buffer.get(t))); - }); - - for_loop_strided( - par.on(exec), 0, m_policy.league_size(), m_policy.chunk_size(), - [this, &buffer, value_size](int const league_rank_begin) { - std::size_t t = Kokkos::Experimental::HPX::impl_hardware_thread_id(); - reference_type update = Analysis::Reducer::reference( - reinterpret_cast(buffer.get(t))); - const int league_rank_end = - (std::min)(league_rank_begin + m_policy.chunk_size(), - m_policy.league_size()); - execute_functor_range( - m_functor, m_policy, league_rank_begin, league_rank_end, - buffer.get(t) + value_size, m_shared, update); - }); -#endif - - const pointer_type ptr = reinterpret_cast(buffer.get(0)); - for (int t = 1; t < num_worker_threads; ++t) { - final_reducer.join(ptr, reinterpret_cast(buffer.get(t))); - } - - final_reducer.final(ptr); - - if (m_result_ptr) { - const int n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - - for (int j = 0; j < n; ++j) { - m_result_ptr[j] = ptr[j]; - } - } - } - - template - ParallelReduce(const FunctorType &arg_functor, const Policy &arg_policy, - const ViewType &arg_result, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void *> = nullptr) - : m_functor(arg_functor), - m_league(arg_policy.league_size()), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_shared(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - m_functor, arg_policy.team_size())), - m_force_synchronous(!arg_result.impl_track().has_record()) {} - - inline ParallelReduce(const FunctorType &arg_functor, Policy arg_policy, - const ReducerType &reducer) - : m_functor(arg_functor), - m_league(arg_policy.league_size()), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_shared(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())), - m_force_synchronous(!reducer.view().impl_track().has_record()) {} -}; -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { - -template -KOKKOS_INLINE_FUNCTION - Impl::TeamThreadRangeBoundariesStruct - TeamThreadRange(const Impl::HPXTeamMember &thread, const iType &count) { - return Impl::TeamThreadRangeBoundariesStruct( - thread, count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - std::common_type_t, Impl::HPXTeamMember> -TeamThreadRange(const Impl::HPXTeamMember &thread, const iType1 &i_begin, - const iType2 &i_end) { - using iType = std::common_type_t; - return Impl::TeamThreadRangeBoundariesStruct( - thread, iType(i_begin), iType(i_end)); -} - -template -KOKKOS_INLINE_FUNCTION - Impl::TeamThreadRangeBoundariesStruct - TeamVectorRange(const Impl::HPXTeamMember &thread, const iType &count) { - return Impl::TeamThreadRangeBoundariesStruct( - thread, count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - std::common_type_t, Impl::HPXTeamMember> -TeamVectorRange(const Impl::HPXTeamMember &thread, const iType1 &i_begin, - const iType2 &i_end) { - using iType = std::common_type_t; - return Impl::TeamThreadRangeBoundariesStruct( - thread, iType(i_begin), iType(i_end)); -} - -template -KOKKOS_INLINE_FUNCTION - Impl::ThreadVectorRangeBoundariesStruct - ThreadVectorRange(const Impl::HPXTeamMember &thread, const iType &count) { - return Impl::ThreadVectorRangeBoundariesStruct( - thread, count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::ThreadVectorRangeBoundariesStruct< - std::common_type_t, Impl::HPXTeamMember> -ThreadVectorRange(const Impl::HPXTeamMember &thread, const iType1 &i_begin, - const iType2 &i_end) { - using iType = std::common_type_t; - return Impl::ThreadVectorRangeBoundariesStruct( - thread, iType(i_begin), iType(i_end)); -} - -KOKKOS_INLINE_FUNCTION -Impl::ThreadSingleStruct PerTeam( - const Impl::HPXTeamMember &thread) { - return Impl::ThreadSingleStruct(thread); -} - -KOKKOS_INLINE_FUNCTION -Impl::VectorSingleStruct PerThread( - const Impl::HPXTeamMember &thread) { - return Impl::VectorSingleStruct(thread); -} - -/** \brief Inter-thread parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all threads of the the calling thread team. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::TeamThreadRangeBoundariesStruct - &loop_boundaries, - const Lambda &lambda) { - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) - lambda(i); -} - -/** \brief Inter-thread vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all threads of the the calling thread team - * and a summation of val is performed and put into result. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct - &loop_boundaries, - const Lambda &lambda, ValueType &result) { - result = ValueType(); - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, result); - } -} - -/** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling thread. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::ThreadVectorRangeBoundariesStruct - &loop_boundaries, - const Lambda &lambda) { -#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP -#pragma ivdep -#endif - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i); - } -} - -/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling thread - * and a summation of val is performed and put into result. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct - &loop_boundaries, - const Lambda &lambda, ValueType &result) { - result = ValueType(); -#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP -#pragma ivdep -#endif - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, result); - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct - &loop_boundaries, - const Lambda &lambda, const ReducerType &reducer) { - reducer.init(reducer.reference()); - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, reducer.reference()); - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct - &loop_boundaries, - const Lambda &lambda, const ReducerType &reducer) { - reducer.init(reducer.reference()); -#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP -#pragma ivdep -#endif - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, reducer.reference()); - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - Impl::TeamThreadRangeBoundariesStruct const - &loop_boundaries, - const FunctorType &lambda) { - using value_type = typename Kokkos::Impl::FunctorAnalysis< - Kokkos::Impl::FunctorPatternInterface::SCAN, void, - FunctorType>::value_type; - - value_type scan_val = value_type(); - - // Intra-member scan - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, scan_val, false); - } - - // 'scan_val' output is the exclusive prefix sum - scan_val = loop_boundaries.thread.team_scan(scan_val); - - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, scan_val, true); - } -} - -/** \brief Intra-thread vector parallel exclusive prefix sum. Executes - * lambda(iType i, ValueType & val, bool final) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes in the thread and a scan - * operation is performed. Depending on the target execution space the operator - * might be called twice: once with final=false and once with final=true. When - * final==true val contains the prefix sum value. The contribution of this "i" - * needs to be added to val no matter whether final==true or not. In a serial - * execution (i.e. team_size==1) the operator is only called once with - * final==true. Scan_val will be set to the final sum value over all vector - */ -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct - &loop_boundaries, - const FunctorType &lambda) { - using value_type = - typename Impl::FunctorAnalysis, - FunctorType>::value_type; - - value_type scan_val = value_type(); - -#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP -#pragma ivdep -#endif - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, scan_val, true); - } -} - -/** \brief Intra-thread vector parallel scan with reducer - * - */ -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct - &loop_boundaries, - const FunctorType &lambda, const ReducerType &reducer) { - typename ReducerType::value_type scan_val; - reducer.init(scan_val); - -#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP -#pragma ivdep -#endif - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, scan_val, true); - } -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::VectorSingleStruct &, - const FunctorType &lambda) { - lambda(); -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::ThreadSingleStruct &, - const FunctorType &lambda) { - lambda(); -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::VectorSingleStruct &, - const FunctorType &lambda, ValueType &val) { - lambda(val); -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::ThreadSingleStruct &, - const FunctorType &lambda, ValueType &val) { - lambda(val); -} - -} // namespace Kokkos - -#include - -#endif /* #if defined( KOKKOS_ENABLE_HPX ) */ -#endif /* #ifndef KOKKOS_HPX_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_HostSpace.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_HostSpace.hpp deleted file mode 100644 index 1d67e2f9cd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_HostSpace.hpp +++ /dev/null @@ -1,363 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_HOSTSPACE_HPP -#define KOKKOS_HOSTSPACE_HPP - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include "impl/Kokkos_HostSpace_deepcopy.hpp" -#include - -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { - -namespace Impl { - -/// \brief Initialize lock array for arbitrary size atomics. -/// -/// Arbitrary atomics are implemented using a hash table of locks -/// where the hash value is derived from the address of the -/// object for which an atomic operation is performed. -/// This function initializes the locks to zero (unset). -void init_lock_array_host_space(); - -/// \brief Acquire a lock for the address -/// -/// This function tries to acquire the lock for the hash value derived -/// from the provided ptr. If the lock is successfully acquired the -/// function returns true. Otherwise it returns false. -bool lock_address_host_space(void* ptr); - -/// \brief Release lock for the address -/// -/// This function releases the lock for the hash value derived -/// from the provided ptr. This function should only be called -/// after previously successfully acquiring a lock with -/// lock_address. -void unlock_address_host_space(void* ptr); - -} // namespace Impl - -} // namespace Kokkos - -namespace Kokkos { -/// \class HostSpace -/// \brief Memory management for host memory. -/// -/// HostSpace is a memory space that governs host memory. "Host" -/// memory means the usual CPU-accessible memory. -class HostSpace { - public: - //! Tag this class as a kokkos memory space - using memory_space = HostSpace; - using size_type = size_t; - - /// \typedef execution_space - /// \brief Default execution space for this memory space. - /// - /// Every memory space has a default execution space. This is - /// useful for things like initializing a View (which happens in - /// parallel using the View's default execution space). - using execution_space = DefaultHostExecutionSpace; - - //! This memory space preferred device_type - using device_type = Kokkos::Device; - - /**\brief Default memory space instance */ - HostSpace(); - HostSpace(HostSpace&& rhs) = default; - HostSpace(const HostSpace& rhs) = default; - HostSpace& operator=(HostSpace&&) = default; - HostSpace& operator=(const HostSpace&) = default; - ~HostSpace() = default; - - /**\brief Non-default memory space instance to choose allocation mechansim, - * if available */ - - enum AllocationMechanism { - STD_MALLOC, - POSIX_MEMALIGN, - POSIX_MMAP, - INTEL_MM_ALLOC - }; - - explicit HostSpace(const AllocationMechanism&); - - /**\brief Allocate untracked memory in the space */ - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class Kokkos::Experimental::LogicalMemorySpace; - - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - - public: - /**\brief Return Name of the MemorySpace */ - static constexpr const char* name() { return m_name; } - - private: - AllocationMechanism m_alloc_mech; - static constexpr const char* m_name = "Host"; - friend class Kokkos::Impl::SharedAllocationRecord; -}; - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -static_assert(Kokkos::Impl::MemorySpaceAccess::assignable, - ""); - -template -struct HostMirror { - private: - // If input execution space can access HostSpace then keep it. - // Example: Kokkos::OpenMP can access, Kokkos::Cuda cannot - enum { - keep_exe = Kokkos::Impl::MemorySpaceAccess< - typename S::execution_space::memory_space, - Kokkos::HostSpace>::accessible - }; - - // If HostSpace can access memory space then keep it. - // Example: Cannot access Kokkos::CudaSpace, can access Kokkos::CudaUVMSpace - enum { - keep_mem = - Kokkos::Impl::MemorySpaceAccess::accessible - }; - - public: - using Space = std::conditional_t< - keep_exe && keep_mem, S, - std::conditional_t, - Kokkos::HostSpace>>; -}; - -} // namespace Impl - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template <> -class SharedAllocationRecord - : public SharedAllocationRecordCommon { - private: - friend Kokkos::HostSpace; - friend class SharedAllocationRecordCommon; - - using base_t = SharedAllocationRecordCommon; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - -#ifdef KOKKOS_ENABLE_DEBUG - /**\brief Root record for tracked allocations from this HostSpace instance */ - static RecordBase s_root_record; -#endif - - const Kokkos::HostSpace m_space; - - protected: - ~SharedAllocationRecord() -#if defined( \ - KOKKOS_IMPL_INTEL_WORKAROUND_NOEXCEPT_SPECIFICATION_VIRTUAL_FUNCTION) - noexcept -#endif - ; - SharedAllocationRecord() = default; - - // This constructor does not forward to the one without exec_space arg - // in order to work around https://github.com/kokkos/kokkos/issues/5258 - // This constructor is templated so I can't just put it into the cpp file - // like the other constructor. - template - SharedAllocationRecord( - const ExecutionSpace& /* exec_space*/, const Kokkos::HostSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate) - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - this->base_t::_fill_host_accessible_header_info(*RecordBase::m_alloc_ptr, - arg_label); - } - - SharedAllocationRecord( - const Kokkos::HostSpace& arg_space, const std::string& arg_label, - const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate); - - public: - KOKKOS_INLINE_FUNCTION static SharedAllocationRecord* allocate( - const Kokkos::HostSpace& arg_space, const std::string& arg_label, - const size_t arg_alloc_size) { - KOKKOS_IF_ON_HOST((return new SharedAllocationRecord(arg_space, arg_label, - arg_alloc_size);)) - KOKKOS_IF_ON_DEVICE(((void)arg_space; (void)arg_label; (void)arg_alloc_size; - return nullptr;)) - } -}; - -} // namespace Impl - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template -struct ZeroMemset { - ZeroMemset(const typename HostSpace::execution_space& exec, - const View& dst, - typename View::const_value_type&) { - // Host spaces, except for HPX, are synchronous and we need to fence for HPX - // since we can't properly enqueue a std::memset otherwise. - // We can't use exec.fence() directly since we don't have a full definition - // of HostSpace here. - hostspace_fence(exec); - using ValueType = typename View::value_type; - std::memset(dst.data(), 0, sizeof(ValueType) * dst.size()); - } - - ZeroMemset(const View& dst, - typename View::const_value_type&) { - using ValueType = typename View::value_type; - std::memset(dst.data(), 0, sizeof(ValueType) * dst.size()); - } -}; - -template <> -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const DefaultHostExecutionSpace& exec, void* dst, const void* src, - size_t n) { - hostspace_parallel_deepcopy_async(exec, dst, src, n); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"); - hostspace_parallel_deepcopy_async(dst, src, n); - } -}; - -} // namespace Impl - -} // namespace Kokkos - -#endif // #define KOKKOS_HOSTSPACE_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_LogicalSpaces.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_LogicalSpaces.hpp deleted file mode 100644 index d3ce354c2a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_LogicalSpaces.hpp +++ /dev/null @@ -1,445 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_LOGICALSPACES_HPP -#define KOKKOS_LOGICALSPACES_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -namespace Kokkos { -namespace Experimental { -struct DefaultMemorySpaceNamer { - static constexpr const char* get_name() { - return "DefaultLogicalMemorySpaceName"; - } -}; - -struct LogicalSpaceSharesAccess { - struct shared_access {}; - struct no_shared_access {}; -}; - -/// \class LogicalMemorySpace -/// \brief -/// -/// LogicalMemorySpace is a space that is identical to another space, -/// but differentiable by name and template argument -template -class LogicalMemorySpace { -#ifdef KOKKOS_ENABLE_OPENMPTARGET - // [DZP] For some reason I don't yet know, using LogicalMemorySpaces - // inside an OpenMPTarget build causes errors in the - // SharedAllocationRecords of other types. This is my way of erroring - // a build if we instantiate a LogicalMemSpace in an OMPTarget build - static_assert(!std::is_same::value, - "Can't use LogicalMemorySpaces in an OpenMPTarget build, we're " - "debugging memory issues"); -#endif - public: - //! Tag this class as a kokkos memory space - using memory_space = LogicalMemorySpace; - using size_type = typename BaseSpace::size_type; - - /// \typedef execution_space - /// \brief Default execution space for this memory space. - /// - /// Every memory space has a default execution space. This is - /// useful for things like initializing a View (which happens in - /// parallel using the View's default execution space). - - using execution_space = - std::conditional_t::value, - typename BaseSpace::execution_space, - DefaultBaseExecutionSpace>; - - using device_type = Kokkos::Device; - - LogicalMemorySpace() = default; - - template - LogicalMemorySpace(Args&&... args) : underlying_space((Args &&) args...) {} - - /**\brief Allocate untracked memory in the space */ - void* allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); - } - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); - } - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); - } - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); - } - - /**\brief Return Name of the MemorySpace */ - constexpr static const char* name() { return Namer::get_name(); } - - private: - BaseSpace underlying_space; - template - friend class LogicalMemorySpace; - friend class Kokkos::Impl::SharedAllocationRecord; - - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - Kokkos::Tools::SpaceHandle arg_handle = - Kokkos::Tools::make_space_handle(name())) const { - return underlying_space.impl_allocate(arg_label, arg_alloc_size, - arg_logical_size, arg_handle); - } - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle arg_handle = - Kokkos::Tools::make_space_handle(name())) const { - underlying_space.impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, - arg_logical_size, arg_handle); - } -}; -} // namespace Experimental -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template -struct MemorySpaceAccess< - Kokkos::Experimental::LogicalMemorySpace< - BaseSpace, DefaultBaseExecutionSpace, Namer, - Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>, - OtherSpace> { - enum { assignable = MemorySpaceAccess::assignable }; - enum { accessible = MemorySpaceAccess::accessible }; - enum { deepcopy = MemorySpaceAccess::deepcopy }; -}; - -template -struct MemorySpaceAccess< - OtherSpace, - Kokkos::Experimental::LogicalMemorySpace< - BaseSpace, DefaultBaseExecutionSpace, Namer, - Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>> { - enum { assignable = MemorySpaceAccess::assignable }; - enum { accessible = MemorySpaceAccess::accessible }; - enum { deepcopy = MemorySpaceAccess::deepcopy }; -}; - -template -struct MemorySpaceAccess< - Kokkos::Experimental::LogicalMemorySpace< - BaseSpace, DefaultBaseExecutionSpace, Namer, - Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>, - Kokkos::Experimental::LogicalMemorySpace< - BaseSpace, DefaultBaseExecutionSpace, Namer, - Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>> { - enum { assignable = true }; - enum { accessible = true }; - enum { deepcopy = true }; -}; - -} // namespace Impl - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { -template -class SharedAllocationRecord, - void> : public SharedAllocationRecord { - private: - using SpaceType = - Kokkos::Experimental::LogicalMemorySpace; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - - static void deallocate(RecordBase* arg_rec) { - delete static_cast(arg_rec); - } - -#ifdef KOKKOS_ENABLE_DEBUG - /**\brief Root record for tracked allocations from this - * LogicalMemorySpace instance */ - static RecordBase s_root_record; -#endif - - const SpaceType m_space; - - protected: - ~SharedAllocationRecord() { - m_space.deallocate(RecordBase::m_alloc_ptr->m_label, - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size, - (SharedAllocationRecord::m_alloc_size - - sizeof(SharedAllocationHeader))); - } - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, const SpaceType& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const SpaceType& arg_space, const std::string& arg_label, - const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate) - : SharedAllocationRecord( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - // Fill in the Header information - RecordBase::m_alloc_ptr->m_record = - static_cast*>(this); - - strncpy(RecordBase::m_alloc_ptr->m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length - 1); - // Set last element zero, in case c_str is too long - RecordBase::m_alloc_ptr - ->m_label[SharedAllocationHeader::maximum_label_length - 1] = '\0'; - } - - public: - inline std::string get_label() const { - return std::string(RecordBase::head()->m_label); - } - KOKKOS_INLINE_FUNCTION static SharedAllocationRecord* allocate( - const SpaceType& arg_space, const std::string& arg_label, - const size_t arg_alloc_size) { - KOKKOS_IF_ON_HOST((return new SharedAllocationRecord(arg_space, arg_label, - arg_alloc_size);)) - KOKKOS_IF_ON_DEVICE(((void)arg_space; (void)arg_label; (void)arg_alloc_size; - return nullptr;)) - } - - /**\brief Allocate tracked memory in the space */ - static void* allocate_tracked(const SpaceType& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size) { - if (!arg_alloc_size) return (void*)nullptr; - - SharedAllocationRecord* const r = - allocate(arg_space, arg_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); - } - - /**\brief Reallocate tracked memory in the space */ - static void* reallocate_tracked(void* const arg_alloc_ptr, - const size_t arg_alloc_size) { - SharedAllocationRecord* const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord* const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - Kokkos::fence( - "SharedAllocationRecord::reallocate_tracked: fence after copying data"); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); - } - /**\brief Deallocate tracked memory in the space */ - static void deallocate_tracked(void* const arg_alloc_ptr) { - if (arg_alloc_ptr != nullptr) { - SharedAllocationRecord* const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } - } - - static SharedAllocationRecord* get_record(void* alloc_ptr) { - using Header = SharedAllocationHeader; - using RecordHost = SharedAllocationRecord; - - SharedAllocationHeader const* const head = - alloc_ptr ? Header::get_header(alloc_ptr) - : (SharedAllocationHeader*)nullptr; - RecordHost* const record = - head ? static_cast(head->m_record) : (RecordHost*)nullptr; - - if (!alloc_ptr || record->m_alloc_ptr != head) { - Kokkos::Impl::throw_runtime_exception(std::string( - "Kokkos::Impl::SharedAllocationRecord< LogicalMemorySpace<> , " - "void >::get_record ERROR")); - } - - return record; - } -#ifdef KOKKOS_ENABLE_DEBUG - static void print_records(std::ostream& s, const SpaceType&, - bool detail = false) { - SharedAllocationRecord::print_host_accessible_records( - s, "HostSpace", &s_root_record, detail); - } -#else - static void print_records(std::ostream&, const SpaceType&, - bool detail = false) { - (void)detail; - throw_runtime_exception( - "SharedAllocationRecord::print_records only works " - "with KOKKOS_ENABLE_DEBUG enabled"); - } -#endif -}; -#ifdef KOKKOS_ENABLE_DEBUG -/**\brief Root record for tracked allocations from this LogicalSpace - * instance */ -template -SharedAllocationRecord - SharedAllocationRecord, - void>::s_root_record; -#endif - -} // namespace Impl - -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template -struct DeepCopy, - Kokkos::Experimental::LogicalMemorySpace< - BaseSpace, DefaultBaseExecutionSpace, Namer, SharesAccess>, - ExecutionSpace> { - DeepCopy(void* dst, void* src, size_t n) { - DeepCopy(dst, src, n); - } - DeepCopy(const ExecutionSpace& exec, void* dst, void* src, size_t n) { - DeepCopy(exec, dst, src, n); - } -}; - -template -struct DeepCopy, - ExecutionSpace> { - DeepCopy(void* dst, void* src, size_t n) { - DeepCopy(dst, src, n); - } - DeepCopy(const ExecutionSpace& exec, void* dst, void* src, size_t n) { - DeepCopy(exec, dst, src, n); - } -}; - -template -struct DeepCopy, - DestinationSpace, ExecutionSpace> { - DeepCopy(void* dst, void* src, size_t n) { - DeepCopy(dst, src, n); - } - DeepCopy(const ExecutionSpace& exec, void* dst, void* src, size_t n) { - DeepCopy(exec, dst, src, n); - } -}; -} // namespace Impl - -} // namespace Kokkos -#endif // KOKKOS_LOGICALSPACES_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_MasterLock.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_MasterLock.hpp deleted file mode 100644 index 5e48595bf0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_MasterLock.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_MASTER_LOCK_HPP -#define KOKKOS_MASTER_LOCK_HPP - -#include - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - -namespace Kokkos { -namespace Experimental { - -// my be used to coordinate work between master instances -// SHOULD NOT be used within a parallel algorithm -// -// This lock should be used with with a scoped lock guard -// i.e. std::unique_lock, std::lock_guard -// -// cannot be copied or moved -// has the following functions available -// -// Lock() -// ~Lock() -// -// void lock() -// void unlock() -// bool try_lock() -// -template -class MasterLock; - -} // namespace Experimental -} // namespace Kokkos - -#endif - -#endif // KOKKOS_MASTER_LOCK_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_MathematicalConstants.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_MathematicalConstants.hpp deleted file mode 100644 index 8f7b559e78..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_MathematicalConstants.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_MATHEMATICAL_CONSTANTS_HPP -#define KOKKOS_MATHEMATICAL_CONSTANTS_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_MATHCONSTANTS -#endif - -#include -#include - -namespace Kokkos { -namespace Experimental { - -#if defined(KOKKOS_ENABLE_CXX17) -#define KOKKOS_IMPL_MATH_CONSTANT(TRAIT, VALUE) \ - template \ - inline constexpr auto TRAIT##_v = \ - std::enable_if_t, T>(VALUE) -#else -#define KOKKOS_IMPL_MATH_CONSTANT(TRAIT, VALUE) \ - template \ - constexpr auto TRAIT##_v = \ - std::enable_if_t::value, T>(VALUE) -#endif - -// clang-format off -KOKKOS_IMPL_MATH_CONSTANT(e, 2.718281828459045235360287471352662498L); -KOKKOS_IMPL_MATH_CONSTANT(log2e, 1.442695040888963407359924681001892137L); -KOKKOS_IMPL_MATH_CONSTANT(log10e, 0.434294481903251827651128918916605082L); -KOKKOS_IMPL_MATH_CONSTANT(pi, 3.141592653589793238462643383279502884L); -KOKKOS_IMPL_MATH_CONSTANT(inv_pi, 0.318309886183790671537767526745028724L); -KOKKOS_IMPL_MATH_CONSTANT(inv_sqrtpi, 0.564189583547756286948079451560772586L); -KOKKOS_IMPL_MATH_CONSTANT(ln2, 0.693147180559945309417232121458176568L); -KOKKOS_IMPL_MATH_CONSTANT(ln10, 2.302585092994045684017991454684364208L); -KOKKOS_IMPL_MATH_CONSTANT(sqrt2, 1.414213562373095048801688724209698079L); -KOKKOS_IMPL_MATH_CONSTANT(sqrt3, 1.732050807568877293527446341505872367L); -KOKKOS_IMPL_MATH_CONSTANT(inv_sqrt3, 0.577350269189625764509148780501957456L); -KOKKOS_IMPL_MATH_CONSTANT(egamma, 0.577215664901532860606512090082402431L); -KOKKOS_IMPL_MATH_CONSTANT(phi, 1.618033988749894848204586834365638118L); -// clang-format on - -#undef KOKKOS_IMPL_MATH_CONSTANT - -} // namespace Experimental -} // namespace Kokkos -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_MATHCONSTANTS -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_MATHCONSTANTS -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_NumericTraits.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_NumericTraits.hpp deleted file mode 100644 index e529aba853..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_NumericTraits.hpp +++ /dev/null @@ -1,661 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_NUMERIC_TRAITS_HPP -#define KOKKOS_NUMERIC_TRAITS_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_NUMERICTRAITS -#endif - -#include -#include -#include -#include -#include -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { -// clang-format off -template struct infinity_helper {}; -template <> struct infinity_helper { static constexpr float value = HUGE_VALF; }; -template <> struct infinity_helper { static constexpr double value = HUGE_VAL; }; -template <> struct infinity_helper { static constexpr long double value = HUGE_VALL; }; -template struct finite_min_helper {}; -template <> struct finite_min_helper { static constexpr bool value = false; }; -template <> struct finite_min_helper { static constexpr char value = CHAR_MIN; }; -template <> struct finite_min_helper { static constexpr signed char value = SCHAR_MIN; }; -template <> struct finite_min_helper { static constexpr unsigned char value = 0; }; -template <> struct finite_min_helper { static constexpr short value = SHRT_MIN; }; -template <> struct finite_min_helper { static constexpr unsigned short value = 0; }; -template <> struct finite_min_helper { static constexpr int value = INT_MIN; }; -template <> struct finite_min_helper { static constexpr unsigned int value = 0; }; -template <> struct finite_min_helper { static constexpr long int value = LONG_MIN; }; -template <> struct finite_min_helper { static constexpr unsigned long int value = 0; }; -template <> struct finite_min_helper { static constexpr long long int value = LLONG_MIN; }; -template <> struct finite_min_helper { static constexpr unsigned long long int value = 0; }; -template <> struct finite_min_helper { static constexpr float value = -FLT_MAX; }; -template <> struct finite_min_helper { static constexpr double value = -DBL_MAX; }; -template <> struct finite_min_helper { static constexpr long double value = -LDBL_MAX; }; -template struct finite_max_helper {}; -template <> struct finite_max_helper { static constexpr bool value = true; }; -template <> struct finite_max_helper { static constexpr char value = CHAR_MAX; }; -template <> struct finite_max_helper { static constexpr signed char value = SCHAR_MAX; }; -template <> struct finite_max_helper { static constexpr unsigned char value = UCHAR_MAX; }; -template <> struct finite_max_helper { static constexpr short value = SHRT_MAX; }; -template <> struct finite_max_helper { static constexpr unsigned short value = USHRT_MAX; }; -template <> struct finite_max_helper { static constexpr int value = INT_MAX; }; -template <> struct finite_max_helper { static constexpr unsigned int value = UINT_MAX; }; -template <> struct finite_max_helper { static constexpr long int value = LONG_MAX; }; -template <> struct finite_max_helper { static constexpr unsigned long int value = ULONG_MAX; }; -template <> struct finite_max_helper { static constexpr long long int value = LLONG_MAX; }; -template <> struct finite_max_helper { static constexpr unsigned long long int value = ULLONG_MAX; }; -template <> struct finite_max_helper { static constexpr float value = FLT_MAX; }; -template <> struct finite_max_helper { static constexpr double value = DBL_MAX; }; -template <> struct finite_max_helper { static constexpr long double value = LDBL_MAX; }; -template struct epsilon_helper {}; -namespace{ - // FIXME workaround for LDL_EPSILON with XL - template - constexpr T machineeps() { - T epsilon = 1, prev = 1, expression = 1; - do { - prev = epsilon; - epsilon /= 2; - expression = 1 + epsilon; - } while (expression > 1); - return prev; - } -} -template <> struct epsilon_helper { static constexpr float value = FLT_EPSILON; }; -template <> struct epsilon_helper { static constexpr double value = DBL_EPSILON; }; -template <> struct epsilon_helper { -#ifdef KOKKOS_COMPILER_IBM - static constexpr long double value = machineeps(); -#else - static constexpr long double value = LDBL_EPSILON; -#endif -}; -template struct round_error_helper {}; -template <> struct round_error_helper { static constexpr float value = 0.5F; }; -template <> struct round_error_helper { static constexpr double value = 0.5; }; -template <> struct round_error_helper { static constexpr long double value = 0.5L; }; -template struct norm_min_helper {}; -template <> struct norm_min_helper { static constexpr float value = FLT_MIN; }; -template <> struct norm_min_helper { static constexpr double value = DBL_MIN; }; -template <> struct norm_min_helper { static constexpr long double value = LDBL_MIN; }; -template struct denorm_min_helper {}; -// Workaround for GCC <9.2, Clang <9, Intel -// vvvvvvvvvvvvvvvvvvvvvvvvv -#if defined(KOKKOS_ENABLE_CXX17) && defined (FLT_TRUE_MIN) || defined(_MSC_VER) -template <> struct denorm_min_helper { static constexpr float value = FLT_TRUE_MIN; }; -template <> struct denorm_min_helper { static constexpr double value = DBL_TRUE_MIN; }; -template <> struct denorm_min_helper { static constexpr long double value = LDBL_TRUE_MIN; }; -#else -template <> struct denorm_min_helper { static constexpr float value = __FLT_DENORM_MIN__; }; -template <> struct denorm_min_helper { static constexpr double value = __DBL_DENORM_MIN__; }; -template <> struct denorm_min_helper { static constexpr long double value = __LDBL_DENORM_MIN__; }; -#endif -// GCC <10.3 is not able to evaluate T(1) / finite_max_v at compile time when passing -frounding-math -// https://godbolt.org/z/zj9svb1T7 -// Similar issue was reported on IBM Power without the compiler option -#define KOKKOS_IMPL_WORKAROUND_CONSTANT_EXPRESSION_COMPILER_BUG -#ifndef KOKKOS_IMPL_WORKAROUND_CONSTANT_EXPRESSION_COMPILER_BUG -// NOTE see ?lamch routine from LAPACK that determines machine parameters for floating-point arithmetic -template -constexpr T safe_minimum(T /*ignored*/) { - constexpr auto one = static_cast(1); - constexpr auto eps = epsilon_helper::value; - constexpr auto tiny = norm_min_helper::value; - constexpr auto huge = finite_max_helper::value; - constexpr auto small = one / huge; // error: is not a constant expression - return small >= tiny ? small * (one + eps) : tiny; -} -template struct reciprocal_overflow_threshold_helper {}; -template <> struct reciprocal_overflow_threshold_helper { static constexpr float value = safe_minimum(0.f); }; -template <> struct reciprocal_overflow_threshold_helper { static constexpr double value = safe_minimum(0.); }; -template <> struct reciprocal_overflow_threshold_helper { static constexpr long double value = safe_minimum(0.l); }; -#else -template struct reciprocal_overflow_threshold_helper {}; -template <> struct reciprocal_overflow_threshold_helper { static constexpr float value = norm_min_helper::value; }; // OK for IEEE-754 floating-point numbers -template <> struct reciprocal_overflow_threshold_helper { static constexpr double value = norm_min_helper::value; }; -template <> struct reciprocal_overflow_threshold_helper { static constexpr long double value = norm_min_helper::value; }; -#endif -#undef KOKKOS_IMPL_WORKAROUND_CONSTANT_EXPRESSION_COMPILER_BUG -template struct quiet_NaN_helper {}; -template <> struct quiet_NaN_helper { static constexpr float value = __builtin_nanf(""); }; -template <> struct quiet_NaN_helper { static constexpr double value = __builtin_nan(""); }; -#if defined(_MSC_VER) -template <> struct quiet_NaN_helper { static constexpr long double value = __builtin_nan(""); }; -#else -template <> struct quiet_NaN_helper { static constexpr long double value = __builtin_nanl(""); }; -#endif -template struct signaling_NaN_helper {}; -template <> struct signaling_NaN_helper { static constexpr float value = __builtin_nansf(""); }; -template <> struct signaling_NaN_helper { static constexpr double value = __builtin_nans(""); }; -#if defined(_MSC_VER) -template <> struct signaling_NaN_helper { static constexpr long double value = __builtin_nans(""); }; -#else -template <> struct signaling_NaN_helper { static constexpr long double value = __builtin_nansl(""); }; -#endif -template struct digits_helper {}; -template <> struct digits_helper { static constexpr int value = 1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT - std::is_signed::value; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT - 1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(short)-1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(short); }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(int)-1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(int); }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(long int)-1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(long int); }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(long long int)-1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(long long int); }; -template <> struct digits_helper { static constexpr int value = FLT_MANT_DIG; }; -template <> struct digits_helper { static constexpr int value = DBL_MANT_DIG; }; -template <> struct digits_helper { static constexpr int value = LDBL_MANT_DIG; }; -template struct digits10_helper {}; -template <> struct digits10_helper { static constexpr int value = 0; }; -// The fraction 643/2136 approximates log10(2) to 7 significant digits. -// Workaround GCC compiler bug with -frounding-math that prevented the -// floating-point expression to be evaluated at compile time. -#define DIGITS10_HELPER_INTEGRAL(TYPE) \ -template <> struct digits10_helper { static constexpr int value = digits_helper::value * 643L / 2136; }; -DIGITS10_HELPER_INTEGRAL(char) -DIGITS10_HELPER_INTEGRAL(signed char) -DIGITS10_HELPER_INTEGRAL(unsigned char) -DIGITS10_HELPER_INTEGRAL(short) -DIGITS10_HELPER_INTEGRAL(unsigned short) -DIGITS10_HELPER_INTEGRAL(int) -DIGITS10_HELPER_INTEGRAL(unsigned int) -DIGITS10_HELPER_INTEGRAL(long int) -DIGITS10_HELPER_INTEGRAL(unsigned long int) -DIGITS10_HELPER_INTEGRAL(long long int) -DIGITS10_HELPER_INTEGRAL(unsigned long long int) -#undef DIGITS10_HELPER_INTEGRAL -template <> struct digits10_helper { static constexpr int value = FLT_DIG; }; -template <> struct digits10_helper { static constexpr int value = DBL_DIG; }; -template <> struct digits10_helper { static constexpr int value = LDBL_DIG; }; -template struct max_digits10_helper {}; -// Approximate ceil(digits::value * log10(2) + 1) -#define MAX_DIGITS10_HELPER(TYPE) \ -template <> struct max_digits10_helper { static constexpr int value = (digits_helper::value * 643L + 2135) / 2136 + 1; }; -#ifdef FLT_DECIMAL_DIG -template <> struct max_digits10_helper { static constexpr int value = FLT_DECIMAL_DIG; }; -#else -MAX_DIGITS10_HELPER(float) -#endif -#ifdef DBL_DECIMAL_DIG -template <> struct max_digits10_helper { static constexpr int value = DBL_DECIMAL_DIG; }; -#else -MAX_DIGITS10_HELPER(double) -#endif -#ifdef DECIMAL_DIG -template <> struct max_digits10_helper { static constexpr int value = DECIMAL_DIG; }; -#elif LDBL_DECIMAL_DIG -template <> struct max_digits10_helper { static constexpr int value = LDBL_DECIMAL_DIG; }; -#else -MAX_DIGITS10_HELPER(long double) -#endif -#undef MAX_DIGITS10_HELPER -template struct radix_helper {}; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = 2; }; -template <> struct radix_helper { static constexpr int value = FLT_RADIX; }; -template <> struct radix_helper { static constexpr int value = FLT_RADIX; }; -template <> struct radix_helper { static constexpr int value = FLT_RADIX; }; -template struct min_exponent_helper {}; -template <> struct min_exponent_helper { static constexpr int value = FLT_MIN_EXP; }; -template <> struct min_exponent_helper { static constexpr int value = DBL_MIN_EXP; }; -template <> struct min_exponent_helper { static constexpr int value = LDBL_MIN_EXP; }; -template struct min_exponent10_helper {}; -template <> struct min_exponent10_helper { static constexpr int value = FLT_MIN_10_EXP; }; -template <> struct min_exponent10_helper { static constexpr int value = DBL_MIN_10_EXP; }; -template <> struct min_exponent10_helper { static constexpr int value = LDBL_MIN_10_EXP; }; -template struct max_exponent_helper {}; -template <> struct max_exponent_helper { static constexpr int value = FLT_MAX_EXP; }; -template <> struct max_exponent_helper { static constexpr int value = DBL_MAX_EXP; }; -template <> struct max_exponent_helper { static constexpr int value = LDBL_MAX_EXP; }; -template struct max_exponent10_helper{}; -template <> struct max_exponent10_helper { static constexpr int value = FLT_MAX_10_EXP; }; -template <> struct max_exponent10_helper { static constexpr int value = DBL_MAX_10_EXP; }; -template <> struct max_exponent10_helper { static constexpr int value = LDBL_MAX_10_EXP; }; -// clang-format on -} // namespace Impl - -#if defined(KOKKOS_ENABLE_CXX17) -#define KOKKOS_IMPL_DEFINE_TRAIT(TRAIT) \ - template \ - struct TRAIT : Impl::TRAIT##_helper> {}; \ - template \ - inline constexpr auto TRAIT##_v = TRAIT::value; -#else -#define KOKKOS_IMPL_DEFINE_TRAIT(TRAIT) \ - template \ - struct TRAIT : Impl::TRAIT##_helper> {}; -#endif - -// Numeric distinguished value traits -KOKKOS_IMPL_DEFINE_TRAIT(infinity) -KOKKOS_IMPL_DEFINE_TRAIT(finite_min) -KOKKOS_IMPL_DEFINE_TRAIT(finite_max) -KOKKOS_IMPL_DEFINE_TRAIT(epsilon) -KOKKOS_IMPL_DEFINE_TRAIT(round_error) -KOKKOS_IMPL_DEFINE_TRAIT(norm_min) -KOKKOS_IMPL_DEFINE_TRAIT(denorm_min) -KOKKOS_IMPL_DEFINE_TRAIT(reciprocal_overflow_threshold) -KOKKOS_IMPL_DEFINE_TRAIT(quiet_NaN) -KOKKOS_IMPL_DEFINE_TRAIT(signaling_NaN) - -// Numeric characteristics traits -KOKKOS_IMPL_DEFINE_TRAIT(digits) -KOKKOS_IMPL_DEFINE_TRAIT(digits10) -KOKKOS_IMPL_DEFINE_TRAIT(max_digits10) -KOKKOS_IMPL_DEFINE_TRAIT(radix) -KOKKOS_IMPL_DEFINE_TRAIT(min_exponent) -KOKKOS_IMPL_DEFINE_TRAIT(min_exponent10) -KOKKOS_IMPL_DEFINE_TRAIT(max_exponent) -KOKKOS_IMPL_DEFINE_TRAIT(max_exponent10) - -#undef KOKKOS_IMPL_DEFINE_TRAIT - -} // namespace Experimental - -template -struct reduction_identity; /*{ - KOKKOS_FORCEINLINE_FUNCTION constexpr static T sum() { return T(); } // 0 - KOKKOS_FORCEINLINE_FUNCTION constexpr static T prod() // 1 - { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom prod reduction type"); return T(); } - KOKKOS_FORCEINLINE_FUNCTION constexpr static T max() // minimum value - { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom max reduction type"); return T(); } - KOKKOS_FORCEINLINE_FUNCTION constexpr static T min() // maximum value - { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom min reduction type"); return T(); } - KOKKOS_FORCEINLINE_FUNCTION constexpr static T bor() // 0, only for integer -type { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom bor reduction type"); return T(); } - KOKKOS_FORCEINLINE_FUNCTION constexpr static T band() // !0, only for integer -type { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom band reduction type"); return T(); } - KOKKOS_FORCEINLINE_FUNCTION constexpr static T lor() // 0, only for integer -type { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom lor reduction type"); return T(); } - KOKKOS_FORCEINLINE_FUNCTION constexpr static T land() // !0, only for integer -type { static_assert( false, "Missing specialization of -Kokkos::reduction_identity for custom land reduction type"); return T(); } -};*/ - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char max() { - return SCHAR_MIN; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char min() { - return SCHAR_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static signed char land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static bool lor() { - return static_cast(false); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static bool land() { - return static_cast(true); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static short sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short max() { return SHRT_MIN; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short min() { return SHRT_MAX; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static short land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static int sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int max() { return INT_MIN; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int min() { return INT_MAX; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static int land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static long sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long max() { return LONG_MIN; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long min() { return LONG_MAX; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long max() { - return LLONG_MIN; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long min() { - return LLONG_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static long long land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char max() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char min() { - return UCHAR_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned char land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short max() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short min() { - return USHRT_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned short land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int max() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int min() { - return UINT_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned int land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long max() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long min() { - return ULONG_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long sum() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long prod() { - return static_cast(1); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long max() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long min() { - return ULLONG_MAX; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long bor() { - return static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long band() { - return ~static_cast(0x0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long lor() { - return static_cast(0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static unsigned long long land() { - return static_cast(1); - } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static float sum() { - return static_cast(0.0f); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static float prod() { - return static_cast(1.0f); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static float max() { return -FLT_MAX; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static float min() { return FLT_MAX; } -}; - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static double sum() { - return static_cast(0.0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static double prod() { - return static_cast(1.0); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static double max() { return -DBL_MAX; } - KOKKOS_FORCEINLINE_FUNCTION constexpr static double min() { return DBL_MAX; } -}; - -// No __host__ __device__ annotation because long double treated as double in -// device code. May be revisited later if that is not true any more. -template <> -struct reduction_identity { - constexpr static long double sum() { return static_cast(0.0); } - constexpr static long double prod() { return static_cast(1.0); } - constexpr static long double max() { return -LDBL_MAX; } - constexpr static long double min() { return LDBL_MAX; } -}; - -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_NUMERICTRAITS -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_NUMERICTRAITS -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_OpenMPTargetSpace.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_OpenMPTargetSpace.hpp deleted file mode 100644 index b4897449cb..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_OpenMPTargetSpace.hpp +++ /dev/null @@ -1,331 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_OPENMPTARGETSPACE_HPP -#define KOKKOS_OPENMPTARGETSPACE_HPP - -#include -#include -#include -#include - -#include - -#ifdef KOKKOS_ENABLE_OPENMPTARGET - -#include -#include -#include - -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Impl { - -/// \brief Initialize lock array for arbitrary size atomics. -/// -/// Arbitrary atomics are implemented using a hash table of locks -/// where the hash value is derived from the address of the -/// object for which an atomic operation is performed. -/// This function initializes the locks to zero (unset). -// void init_lock_array_host_space(); - -/// \brief Acquire a lock for the address -/// -/// This function tries to acquire the lock for the hash value derived -/// from the provided ptr. If the lock is successfully acquired the -/// function returns true. Otherwise it returns false. -// bool lock_address_host_space(void* ptr); - -/// \brief Release lock for the address -/// -/// This function releases the lock for the hash value derived -/// from the provided ptr. This function should only be called -/// after previously successfully acquiring a lock with -/// lock_address. -// void unlock_address_host_space(void* ptr); - -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -//---------------------------------------- - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -//---------------------------------------- - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -//---------------------------------------- -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Experimental { - -/// \class OpenMPTargetSpace -/// \brief Memory management for host memory. -/// -/// OpenMPTargetSpace is a memory space that governs host memory. "Host" -/// memory means the usual CPU-accessible memory. -class OpenMPTargetSpace { - public: - //! Tag this class as a kokkos memory space - using memory_space = OpenMPTargetSpace; - using size_type = unsigned; - - /// \typedef execution_space - /// \brief Default execution space for this memory space. - /// - /// Every memory space has a default execution space. This is - /// useful for things like initializing a View (which happens in - /// parallel using the View's default execution space). - using execution_space = Kokkos::Experimental::OpenMPTarget; - - //! This memory space preferred device_type - using device_type = Kokkos::Device; - - /*--------------------------------*/ - - /**\brief Default memory space instance */ - OpenMPTargetSpace(); - OpenMPTargetSpace(OpenMPTargetSpace&& rhs) = default; - OpenMPTargetSpace(const OpenMPTargetSpace& rhs) = default; - OpenMPTargetSpace& operator=(OpenMPTargetSpace&&) = default; - OpenMPTargetSpace& operator=(const OpenMPTargetSpace&) = default; - ~OpenMPTargetSpace() = default; - - /**\brief Allocate untracked memory in the space */ - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, - const std::size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - static constexpr const char* name() { return "OpenMPTargetSpace"; } - - private: - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - - friend class Kokkos::Impl::SharedAllocationRecord< - Kokkos::Experimental::OpenMPTargetSpace, void>; -}; -} // namespace Experimental -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template <> -class SharedAllocationRecord - : public HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenMPTargetSpace> { - private: - friend class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenMPTargetSpace>; - friend class SharedAllocationRecordCommon< - Kokkos::Experimental::OpenMPTargetSpace>; - friend Kokkos::Experimental::OpenMPTargetSpace; - - using base_t = HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenMPTargetSpace>; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - - /**\brief Root record for tracked allocations from this OpenMPTargetSpace - * instance */ - static RecordBase s_root_record; - - const Kokkos::Experimental::OpenMPTargetSpace m_space; - - protected: - ~SharedAllocationRecord(); - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::OpenMPTargetSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::OpenMPTargetSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate); - - public: - KOKKOS_INLINE_FUNCTION static SharedAllocationRecord* allocate( - const Kokkos::Experimental::OpenMPTargetSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc) { - KOKKOS_IF_ON_HOST( - (return new SharedAllocationRecord(arg_space, arg_label, arg_alloc);)) - KOKKOS_IF_ON_DEVICE( - ((void)arg_space; (void)arg_label; (void)arg_alloc; return nullptr;)) - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -// TODO: implement all possible deep_copies -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - // In the Release and RelWithDebInfo builds, the size of the memcpy should - // be greater than zero to avoid error. omp_target_memcpy returns zero on - // success. - if (n > 0) - OMPT_SAFE_CALL(omp_target_memcpy(dst, const_cast(src), n, 0, 0, - omp_get_default_device(), - omp_get_default_device())); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy: fence " - "before " - "copy"); - if (n > 0) - OMPT_SAFE_CALL(omp_target_memcpy(dst, const_cast(src), n, 0, 0, - omp_get_default_device(), - omp_get_default_device())); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - if (n > 0) - OMPT_SAFE_CALL(omp_target_memcpy(dst, const_cast(src), n, 0, 0, - omp_get_default_device(), - omp_get_initial_device())); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy: fence before " - "copy"); - if (n > 0) - OMPT_SAFE_CALL(omp_target_memcpy(dst, const_cast(src), n, 0, 0, - omp_get_default_device(), - omp_get_initial_device())); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - if (n > 0) - OMPT_SAFE_CALL(omp_target_memcpy(dst, const_cast(src), n, 0, 0, - omp_get_initial_device(), - omp_get_default_device())); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy: fence before " - "copy"); - if (n > 0) - OMPT_SAFE_CALL(omp_target_memcpy(dst, const_cast(src), n, 0, 0, - omp_get_initial_device(), - omp_get_default_device())); - } -}; - -} // namespace Impl -} // namespace Kokkos - -#endif -#endif /* #define KOKKOS_OPENMPTARGETSPACE_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_PointerOwnership.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_PointerOwnership.hpp deleted file mode 100644 index 41b18a8d14..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_PointerOwnership.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -// Experimental unified task-data parallel manycore LDRD - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_IMPL_POINTEROWNERSHIP_HPP -#define KOKKOS_IMPL_POINTEROWNERSHIP_HPP - -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -/// Trivial wrapper for raw pointers that express ownership. -template -using OwningRawPtr = T*; - -/// Trivial wrapper for raw pointers that do not express ownership. -template -using ObservingRawPtr = T*; - -} // end namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #ifndef KOKKOS_IMPL_POINTEROWNERSHIP_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Profiling_ProfileSection.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Profiling_ProfileSection.hpp deleted file mode 100644 index 266605c0fe..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Profiling_ProfileSection.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - //@HEADER - // ************************************************************************ - // - // Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). - // - // Under the terms of Contract DE-NA0003525 with NTESS, - // the U.S. Government retains certain rights in this software. - // - // Redistribution and use in source and binary forms, with or without - // modification, are permitted provided that the following conditions are - // met: - // - // 1. Redistributions of source code must retain the above copyright - // notice, this list of conditions and the following disclaimer. - // - // 2. Redistributions in binary form must reproduce the above copyright - // notice, this list of conditions and the following disclaimer in the - // documentation and/or other materials provided with the distribution. - // - // 3. Neither the name of the Corporation nor the names of the - // contributors may be used to endorse or promote products derived from - // this software without specific prior written permission. - // - // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY - // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE - // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - // - // Questions? Contact Christian R. Trott (crtrott@sandia.gov) - // - // ************************************************************************ - //@HEADER - */ - -#ifndef KOKKOSP_PROFILE_SECTION_HPP -#define KOKKOSP_PROFILE_SECTION_HPP -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE_PROFILING_PROFILESECTION -#endif - -#include -#include -#include - -#include - -namespace Kokkos { -namespace Profiling { - -class ProfilingSection { - public: - ProfilingSection(ProfilingSection const&) = delete; - ProfilingSection& operator=(ProfilingSection const&) = delete; - - ProfilingSection(const std::string& sectionName) -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - : secName(sectionName) -#endif - { - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::createProfileSection(sectionName, &secID); - } - } - - void start() { - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::startSection(secID); - } - } - - void stop() { - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::stopSection(secID); - } - } - - ~ProfilingSection() { - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::destroyProfileSection(secID); - } - } - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - KOKKOS_DEPRECATED std::string getName() { return secName; } - - KOKKOS_DEPRECATED uint32_t getSectionID() { return secID; } -#endif - - protected: -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - const std::string secName; -#endif - uint32_t secID; -}; - -} // namespace Profiling -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE -#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_PROFILING_PROFILESECTION -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Rank.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Rank.hpp deleted file mode 100644 index 025cf511fe..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Rank.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_KOKKOS_RANK_HPP -#define KOKKOS_KOKKOS_RANK_HPP - -#include -#include // Iterate - -namespace Kokkos { - -// Iteration Pattern -template -struct Rank { - static_assert(N != 0u, "Kokkos Error: rank 0 undefined"); - static_assert(N != 1u, - "Kokkos Error: rank 1 is not a multi-dimensional range"); - static_assert(N < 7u, "Kokkos Error: Unsupported rank..."); - - using iteration_pattern = Rank; - - static constexpr int rank = N; - static constexpr Iterate outer_direction = OuterDir; - static constexpr Iterate inner_direction = InnerDir; -}; - -} // end namespace Kokkos - -#endif // KOKKOS_KOKKOS_RANK_HPP diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_SYCL_Space.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_SYCL_Space.hpp deleted file mode 100644 index e147d04dc8..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_SYCL_Space.hpp +++ /dev/null @@ -1,492 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_SYCLSPACE_HPP -#define KOKKOS_SYCLSPACE_HPP - -#include - -#ifdef KOKKOS_ENABLE_SYCL -#include -#include -#include -#include -#include -#include - -namespace Kokkos { - -namespace Impl { -template -struct is_sycl_type_space : public std::false_type {}; -} // namespace Impl - -namespace Experimental { - -class SYCLDeviceUSMSpace { - public: - using execution_space = SYCL; - using memory_space = SYCLDeviceUSMSpace; - using device_type = Kokkos::Device; - using size_type = Impl::SYCLInternal::size_type; - - SYCLDeviceUSMSpace(); - explicit SYCLDeviceUSMSpace(sycl::queue queue); - - void* allocate(const SYCL& exec_space, - const std::size_t arg_alloc_size) const; - void* allocate(const SYCL& exec_space, const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - void* allocate(const std::size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - void deallocate(void* const arg_alloc_ptr, - const std::size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class LogicalMemorySpace; - - public: - static constexpr const char* name() { return "SYCLDeviceUSM"; }; - - private: - sycl::queue m_queue; -}; - -class SYCLSharedUSMSpace { - public: - using execution_space = SYCL; - using memory_space = SYCLSharedUSMSpace; - using device_type = Kokkos::Device; - using size_type = Impl::SYCLInternal::size_type; - - SYCLSharedUSMSpace(); - explicit SYCLSharedUSMSpace(sycl::queue queue); - - void* allocate(const SYCL& exec_space, - const std::size_t arg_alloc_size) const; - void* allocate(const SYCL& exec_space, const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - void* allocate(const std::size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - void deallocate(void* const arg_alloc_ptr, - const std::size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class LogicalMemorySpace; - - public: - static constexpr const char* name() { return "SYCLSharedUSM"; }; - - private: - sycl::queue m_queue; -}; - -class SYCLHostUSMSpace { - public: - using execution_space = HostSpace::execution_space; - using memory_space = SYCLHostUSMSpace; - using device_type = Kokkos::Device; - using size_type = Impl::SYCLInternal::size_type; - - SYCLHostUSMSpace(); - explicit SYCLHostUSMSpace(sycl::queue queue); - - void* allocate(const SYCL& exec_space, - const std::size_t arg_alloc_size) const; - void* allocate(const SYCL& exec_space, const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - void* allocate(const std::size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - void deallocate(void* const arg_alloc_ptr, - const std::size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - private: - template - friend class LogicalMemorySpace; - - public: - static constexpr const char* name() { return "SYCLHostUSM"; }; - - private: - sycl::queue m_queue; -}; - -} // namespace Experimental - -namespace Impl { - -template <> -struct is_sycl_type_space - : public std::true_type {}; - -template <> -struct is_sycl_type_space - : public std::true_type {}; - -template <> -struct is_sycl_type_space - : public std::true_type {}; - -static_assert(Kokkos::Impl::MemorySpaceAccess< - Kokkos::Experimental::SYCLDeviceUSMSpace, - Kokkos::Experimental::SYCLDeviceUSMSpace>::assignable, - ""); - -static_assert(Kokkos::Impl::MemorySpaceAccess< - Kokkos::Experimental::SYCLSharedUSMSpace, - Kokkos::Experimental::SYCLSharedUSMSpace>::assignable, - ""); - -static_assert(Kokkos::Impl::MemorySpaceAccess< - Kokkos::Experimental::SYCLDeviceUSMSpace, - Kokkos::Experimental::SYCLDeviceUSMSpace>::assignable, - ""); - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // HostSpace::execution_space != SYCLSharedUSMSpace::execution_space - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // HostSpace::execution_space == - // Experimental::SYCLHostUSMSpace::execution_space - enum : bool { assignable = true }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // SYCLDeviceUSMSpace::execution_space == SYCLSharedUSMSpace::execution_space - enum : bool { assignable = true }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // Experimental::SYCLDeviceUSMSpace::execution_space != - // Experimental::SYCLHostUSMSpace::execution_space - enum : bool { assignable = false }; - enum : bool { - accessible = true - }; // Experimental::SYCLDeviceUSMSpace::execution_space - enum : bool { deepcopy = true }; -}; - -//---------------------------------------- -// SYCLSharedUSMSpace::execution_space == SYCL -// SYCLSharedUSMSpace accessible to both SYCL and Host - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; // SYCL cannot access HostSpace - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // SYCLSharedUSMSpace::execution_space == SYCLDeviceUSMSpace::execution_space - // Can access SYCLSharedUSMSpace from Host but cannot access - // SYCLDeviceUSMSpace from Host - enum : bool { assignable = false }; - - // SYCLSharedUSMSpace::execution_space can access SYCLDeviceUSMSpace - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - // Experimental::SYCLSharedUSMSpace::execution_space != - // Experimental::SYCLHostUSMSpace::execution_space - enum : bool { assignable = false }; - enum : bool { - accessible = true - }; // Experimental::SYCLSharedUSMSpace::execution_space - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // Cannot access from SYCL - enum : bool { - accessible = true - }; // Experimental::SYCLHostUSMSpace::execution_space - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // Cannot access from Host - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess { - enum : bool { assignable = false }; // different execution_space - enum : bool { accessible = true }; // same accessibility - enum : bool { deepcopy = true }; -}; - -template <> -struct MemorySpaceAccess< - Kokkos::Experimental::SYCLDeviceUSMSpace, - Kokkos::ScratchMemorySpace> { - enum : bool { assignable = false }; - enum : bool { accessible = true }; - enum : bool { deepcopy = false }; -}; - -} // namespace Impl - -namespace Impl { - -template <> -class SharedAllocationRecord - : public HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::SYCLDeviceUSMSpace> { - private: - friend class SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLDeviceUSMSpace>; - friend class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::SYCLDeviceUSMSpace>; - using base_t = HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::SYCLDeviceUSMSpace>; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord(SharedAllocationRecord&&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(SharedAllocationRecord&&) = delete; - -#ifdef KOKKOS_ENABLE_DEBUG - static RecordBase s_root_record; -#endif - - const Kokkos::Experimental::SYCLDeviceUSMSpace m_space; - - protected: - ~SharedAllocationRecord(); - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::SYCLDeviceUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::SYCL& exec_space, - const Kokkos::Experimental::SYCLDeviceUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); - - SharedAllocationRecord( - const Kokkos::Experimental::SYCLDeviceUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); -}; - -template <> -class SharedAllocationRecord - : public SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLSharedUSMSpace> { - private: - friend class SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLSharedUSMSpace>; - using base_t = - SharedAllocationRecordCommon; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord(SharedAllocationRecord&&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(SharedAllocationRecord&&) = delete; - - static RecordBase s_root_record; - - const Kokkos::Experimental::SYCLSharedUSMSpace m_space; - - protected: - ~SharedAllocationRecord(); - - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::SYCLSharedUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::SYCL& exec_space, - const Kokkos::Experimental::SYCLSharedUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); - - SharedAllocationRecord( - const Kokkos::Experimental::SYCLSharedUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); -}; - -template <> -class SharedAllocationRecord - : public SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLHostUSMSpace> { - private: - friend class SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLHostUSMSpace>; - using base_t = - SharedAllocationRecordCommon; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord(SharedAllocationRecord&&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(SharedAllocationRecord&&) = delete; - - static RecordBase s_root_record; - - const Kokkos::Experimental::SYCLHostUSMSpace m_space; - - protected: - ~SharedAllocationRecord(); - - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::SYCLHostUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::SYCL& exec_space, - const Kokkos::Experimental::SYCLHostUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); - - SharedAllocationRecord( - const Kokkos::Experimental::SYCLHostUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &base_t::deallocate); -}; - -} // namespace Impl - -} // namespace Kokkos - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_TaskPolicy.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_TaskPolicy.hpp deleted file mode 100644 index 9751fab460..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_TaskPolicy.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -// For backward compatibility: -#include - -KOKKOS_IMPL_WARNING( - "This file is deprecated. Use instead.") - -#include diff --git a/bundled/kokkos-3.7.00/core/src/Kokkos_Vectorization.hpp b/bundled/kokkos-3.7.00/core/src/Kokkos_Vectorization.hpp deleted file mode 100644 index 4314ea4417..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Kokkos_Vectorization.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/// \file Kokkos_Vectorization.hpp -/// \brief Declaration and definition of Kokkos::Vectorization interface. -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif -#ifndef KOKKOS_VECTORIZATION_HPP -#define KOKKOS_VECTORIZATION_HPP - -#include - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#elif defined(KOKKOS_ENABLE_HIP) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC.cpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC.cpp deleted file mode 100644 index f3216095be..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#define KOKKOS_IMPL_PUBLIC_INCLUDE - -#include -#include -#include -#include - -#include - -Kokkos::Experimental::OpenACC::OpenACC() - : m_space_instance(Impl::OpenACCInternal::singleton()) {} - -void Kokkos::Experimental::OpenACC::impl_initialize( - InitializationSettings const& settings) { - Impl::OpenACCInternal::singleton()->initialize(settings); -} - -void Kokkos::Experimental::OpenACC::impl_finalize() { - Impl::OpenACCInternal::singleton()->finalize(); -} - -bool Kokkos::Experimental::OpenACC::impl_is_initialized() { - return Impl::OpenACCInternal::singleton()->is_initialized(); -} - -void Kokkos::Experimental::OpenACC::print_configuration(std::ostream& os, - bool verbose) const { - os << "macro KOKKOS_ENABLE_OPENACC is defined\n"; // FIXME_OPENACC - m_space_instance->print_configuration(os, verbose); -} - -void Kokkos::Experimental::OpenACC::fence(std::string const& name) const { - Impl::OpenACCInternal::singleton()->fence(name); -} - -void Kokkos::Experimental::OpenACC::impl_static_fence(std::string const& name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::OpenACC>( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - [&]() { acc_wait_all(); }); -} - -uint32_t Kokkos::Experimental::OpenACC::impl_instance_id() const noexcept { - return m_space_instance->instance_id(); -} - -namespace Kokkos { -namespace Impl { -int g_openacc_space_factory_initialized = - initialize_space_factory("170_OpenACC"); -} // namespace Impl -} // Namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC.hpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC.hpp deleted file mode 100644 index 3ad59057b5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC.hpp +++ /dev/null @@ -1,126 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif - -#ifndef KOKKOS_OPENACC_HPP -#define KOKKOS_OPENACC_HPP - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace Kokkos::Experimental::Impl { -class OpenACCInternal; -} - -namespace Kokkos::Experimental { - -class OpenACC { - Impl::OpenACCInternal* m_space_instance = nullptr; - - public: - using execution_space = OpenACC; - using memory_space = OpenACCSpace; - using device_type = Kokkos::Device; - - using array_layout = LayoutLeft; - using size_type = memory_space::size_type; - - using scratch_memory_space = ScratchMemorySpace; - - OpenACC(); - - static void impl_initialize(InitializationSettings const& settings); - static void impl_finalize(); - static bool impl_is_initialized(); - - void print_configuration(std::ostream& os, bool verbose = false) const; - - void fence(std::string const& name = - "Kokkos::OpenACC::fence(): Unnamed Instance Fence") const; - static void impl_static_fence(std::string const& name); - - static char const* name() { return "OpenACC"; } - static int concurrency() { return 256000; } // FIXME_OPENACC - static bool in_parallel() { return acc_on_device(acc_device_not_host); } - uint32_t impl_instance_id() const noexcept; -}; - -} // namespace Kokkos::Experimental - -template <> -struct Kokkos::Tools::Experimental::DeviceTypeTraits< - ::Kokkos::Experimental::OpenACC> { - static constexpr DeviceType id = - ::Kokkos::Profiling::Experimental::DeviceType::OpenACC; - // FIXME_OPENACC: Need to return the device id from the execution space - // instance. In fact, acc_get_device_num() will return the same value as the - // device id from the execution space instance except for the host fallback - // case, where the device id may need to be updated with the value of - // acc_get_device_num(). - static int device_id(const Kokkos::Experimental::OpenACC&) { - using Kokkos::Experimental::Impl::OpenACC_Traits; - return acc_get_device_num(OpenACC_Traits::dev_type); - } -}; - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACCSpace.cpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACCSpace.cpp deleted file mode 100644 index bc2ba18156..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACCSpace.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#define KOKKOS_IMPL_PUBLIC_INCLUDE - -#include -#include -#include -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -void *Kokkos::Experimental::OpenACCSpace::allocate( - const Kokkos::Experimental::OpenACC &exec_space, - const size_t arg_alloc_size) const { - return allocate(exec_space, "[unlabeled]", arg_alloc_size); -} - -void *Kokkos::Experimental::OpenACCSpace::allocate( - const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} - -void *Kokkos::Experimental::OpenACCSpace::allocate( - const Kokkos::Experimental::OpenACC &exec_space, const char *arg_label, - const size_t arg_alloc_size, const size_t arg_logical_size) const { - return impl_allocate(exec_space, arg_label, arg_alloc_size, arg_logical_size); -} - -void *Kokkos::Experimental::OpenACCSpace::allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} - -void *Kokkos::Experimental::OpenACCSpace::impl_allocate( - const Kokkos::Experimental::OpenACC &exec_space, const char *arg_label, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - static_assert(sizeof(void *) == sizeof(uintptr_t), - "Error sizeof(void*) != sizeof(uintptr_t)"); - - void *ptr = nullptr; - - // FIXME_OPENACC multiple device instances are not yet supported, and thus - // exec_space is ignored for now. - (void)exec_space; - - ptr = acc_malloc(arg_alloc_size); - - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void *Kokkos::Experimental::OpenACCSpace::impl_allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - static_assert(sizeof(void *) == sizeof(uintptr_t), - "Error sizeof(void*) != sizeof(uintptr_t)"); - - void *ptr = nullptr; - - //[DEBUG] Disabled due to the synchronous behavior of the current - // implementation. - /* - OpenACC::impl_static_fence( - "Kokkos::OpenACCSpace::impl_allocate: Pre OpenACC Allocation"); - */ - - ptr = acc_malloc(arg_alloc_size); - - //[DEBUG] Disabled due to the synchronous behavior of the current - // implementation. - /* - OpenACC::impl_static_fence( - "Kokkos::OpenACCSpace::impl_allocate: Post OpenACC Allocation"); - */ - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void Kokkos::Experimental::OpenACCSpace::deallocate( - void *const arg_alloc_ptr, const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void Kokkos::Experimental::OpenACCSpace::deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} - -void Kokkos::Experimental::OpenACCSpace::impl_deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - - if (arg_alloc_ptr) { - acc_free(arg_alloc_ptr); - } -} - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#ifdef KOKKOS_ENABLE_DEBUG -Kokkos::Impl::SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::OpenACCSpace, void>::s_root_record; -#endif - -Kokkos::Impl::SharedAllocationRecord::~SharedAllocationRecord() { - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - (SharedAllocationRecord::m_alloc_size - - sizeof(SharedAllocationHeader))); -} - -Kokkos::Impl::SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::OpenACCSpace &arg_space, - const std::string &arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, arg_label); - - Kokkos::Impl::DeepCopy( - RecordBase::m_alloc_ptr, &header, sizeof(SharedAllocationHeader)); - Kokkos::fence( - "SharedAllocationRecord::SharedAllocationRecord(): fence after copying header from " - "HostSpace"); -} - -//============================================================================== -// {{{1 - -#include - -// To avoid additional compilation cost for something that's (mostly?) not -// performance sensitive, we explicitly instantiate these CRTP base classes -// here, where we have access to the associated *_timpl.hpp header files. -template class Kokkos::Impl::HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenACCSpace>; -template class Kokkos::Impl::SharedAllocationRecordCommon< - Kokkos::Experimental::OpenACCSpace>; - -// end Explicit instantiations of CRTP Base classes }}}1 -//============================================================================== diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACCSpace.hpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACCSpace.hpp deleted file mode 100644 index a7347e8f91..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACCSpace.hpp +++ /dev/null @@ -1,249 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3 -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#else -KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.") -#endif -#endif - -#ifndef KOKKOS_OPENACC_SPACE_HPP -#define KOKKOS_OPENACC_SPACE_HPP - -#include - -#include -#include - -#include -#include - -namespace Kokkos::Experimental { - -class OpenACC; - -class OpenACCSpace { - public: - using memory_space = OpenACCSpace; - using execution_space = OpenACC; - using device_type = Kokkos::Device; - - using size_type = size_t; - - OpenACCSpace() = default; - - /**\brief Allocate untracked memory in the space */ - void* allocate(const Kokkos::Experimental::OpenACC& exec_space, - const size_t arg_alloc_size) const; - void* allocate(const Kokkos::Experimental::OpenACC& exec_space, - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - void* allocate(const size_t arg_alloc_size) const; - void* allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - /**\brief Deallocate untracked memory in the space */ - void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const; - void deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const; - - static constexpr char const* name() { return "OpenACCSpace"; } - - private: - void* impl_allocate(const Kokkos::Experimental::OpenACC& exec_space, - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; - void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; -}; - -} // namespace Kokkos::Experimental - -/*--------------------------------------------------------------------------*/ - -template <> -struct Kokkos::Impl::MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct Kokkos::Impl::MemorySpaceAccess { - enum : bool { assignable = false }; - enum : bool { accessible = false }; - enum : bool { deepcopy = true }; -}; - -template <> -struct Kokkos::Impl::MemorySpaceAccess { - enum : bool { assignable = true }; - enum : bool { accessible = true }; - enum : bool { deepcopy = true }; -}; -/*--------------------------------------------------------------------------*/ - -template <> -class Kokkos::Impl::SharedAllocationRecord - : public HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenACCSpace> { - private: - friend class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenACCSpace>; - friend class SharedAllocationRecordCommon; - friend Kokkos::Experimental::OpenACCSpace; - - using base_t = HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenACCSpace>; - using RecordBase = SharedAllocationRecord; - - SharedAllocationRecord(const SharedAllocationRecord&) = delete; - SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - - /**\brief Root record for tracked allocations from this OpenACCSpace - * instance */ - static RecordBase s_root_record; - - const Kokkos::Experimental::OpenACCSpace m_space; - - protected: - ~SharedAllocationRecord(); - SharedAllocationRecord() = default; - - template - SharedAllocationRecord( - const ExecutionSpace& /*exec_space*/, - const Kokkos::Experimental::OpenACCSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate) - : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size, - arg_dealloc) {} - - SharedAllocationRecord( - const Kokkos::Experimental::OpenACCSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate); - - public: - KOKKOS_INLINE_FUNCTION static SharedAllocationRecord* allocate( - const Kokkos::Experimental::OpenACCSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size) { - if (acc_on_device(acc_device_host)) { - return new SharedAllocationRecord(arg_space, arg_label, arg_alloc_size); - } else { - return nullptr; - } - } -}; - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -// FIXME_OPENACC: Need to update the DeepCopy implementations below to support -// multiple execution space instances. -// The current OpenACC backend implementation assumes that there is only one -// device execution space instance, and all the device operations (e.g., memory -// transfers, kernel launches, etc.) are implemented to be synchronous, which -// does not violate the Kokkos execution semantics with the single execution -// space instance. -template -struct Kokkos::Impl::DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - // The behavior of acc_memcpy_device when bytes argument is zero is - // clarified only in the latest OpenACC specification (V3.2), and thus the - // value checking is added as a safeguard. (The current NVHPC (V22.5) - // supports OpenACC V2.7.) - if (n > 0) acc_memcpy_device(dst, const_cast(src), n); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence(); - if (n > 0) acc_memcpy_device(dst, const_cast(src), n); - } -}; - -template -struct Kokkos::Impl::DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - if (n > 0) acc_memcpy_to_device(dst, const_cast(src), n); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence(); - if (n > 0) acc_memcpy_to_device(dst, const_cast(src), n); - } -}; - -template -struct Kokkos::Impl::DeepCopy< - Kokkos::HostSpace, Kokkos::Experimental::OpenACCSpace, ExecutionSpace> { - DeepCopy(void* dst, const void* src, size_t n) { - if (n > 0) acc_memcpy_from_device(dst, const_cast(src), n); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence(); - if (n > 0) acc_memcpy_from_device(dst, const_cast(src), n); - } -}; - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp deleted file mode 100644 index 15d38803f9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#define KOKKOS_IMPL_PUBLIC_INCLUDE - -#include -#include -#include -#include -#include - -#include - -#include - -namespace Kokkos { -bool show_warnings() noexcept; -} - -Kokkos::Experimental::Impl::OpenACCInternal* -Kokkos::Experimental::Impl::OpenACCInternal::singleton() { - static OpenACCInternal self; - return &self; -} - -void Kokkos::Experimental::Impl::OpenACCInternal::initialize( - InitializationSettings const& settings) { - if (OpenACC_Traits::may_fallback_to_host && - acc_get_num_devices(OpenACC_Traits::dev_type) == 0 && - !settings.has_device_id()) { - if (show_warnings()) { - std::cerr << "Warning: No GPU available for execution, falling back to" - " using the host!" - << std::endl; - } - acc_set_device_type(acc_device_host); - // FIXME_OPENACC if multiple execution space instances are supported, - // device id variable should be explicitly set to the value returned by - // acc_get_device_num(acc_device_host). - } else { - using Kokkos::Impl::get_gpu; - int const dev_num = get_gpu(settings); - acc_set_device_num(dev_num, OpenACC_Traits::dev_type); - } - m_is_initialized = true; -} - -void Kokkos::Experimental::Impl::OpenACCInternal::finalize() { - m_is_initialized = false; -} - -bool Kokkos::Experimental::Impl::OpenACCInternal::is_initialized() const { - return m_is_initialized; -} - -void Kokkos::Experimental::Impl::OpenACCInternal::print_configuration( - std::ostream& os, bool /*verbose*/) const { - os << "Using OpenACC\n"; // FIXME_OPENACC -} - -void Kokkos::Experimental::Impl::OpenACCInternal::fence( - std::string const& name) const { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::OpenACC>( - name, - Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{instance_id()}, - [&]() { - //[DEBUG] disabled due to synchronous behaviors of the current - // parallel construct implementations. acc_wait_all(); - }); -} - -uint32_t Kokkos::Experimental::Impl::OpenACCInternal::instance_id() const - noexcept { - return Kokkos::Tools::Experimental::Impl::idForInstance< - Kokkos::Experimental::OpenACC>(reinterpret_cast(this)); -} diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp deleted file mode 100644 index cb69b4ae7a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENACC_INSTANCE_HPP -#define KOKKOS_OPENACC_INSTANCE_HPP - -#include - -#include -#include -#include - -namespace Kokkos::Experimental::Impl { - -class OpenACCInternal { - bool m_is_initialized = false; - - OpenACCInternal() = default; - OpenACCInternal(const OpenACCInternal&) = default; - OpenACCInternal& operator=(const OpenACCInternal&) = default; - - public: - static OpenACCInternal* singleton(); - - void initialize(InitializationSettings const& settings); - void finalize(); - bool is_initialized() const; - - void print_configuration(std::ostream& os, bool verbose = false) const; - - void fence(std::string const& name) const; - - uint32_t instance_id() const noexcept; -}; - -} // namespace Kokkos::Experimental::Impl - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp b/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp deleted file mode 100644 index f9451ecfe6..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENACC_TRAITS_HPP -#define KOKKOS_OPENACC_TRAITS_HPP - -#include - -namespace Kokkos::Experimental::Impl { - -struct OpenACC_Traits { -#if defined(KOKKOS_ARCH_PASCAL) || defined(KOKKOS_ARCH_VOLTA) || \ - defined(KOKKOS_ARCH_AMPERE) - static constexpr acc_device_t dev_type = acc_device_nvidia; - static constexpr bool may_fallback_to_host = false; -#else - static constexpr acc_device_t dev_type = acc_device_not_host; - static constexpr bool may_fallback_to_host = true; -#endif -}; - -} // namespace Kokkos::Experimental::Impl - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Instance.cpp b/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Instance.cpp deleted file mode 100644 index 2397aa4785..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Instance.cpp +++ /dev/null @@ -1,492 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -int g_openmp_hardware_max_threads = 1; - -thread_local int t_openmp_hardware_id = 0; -// FIXME_OPENMP we can remove this after we remove partition_master -thread_local OpenMPInternal *t_openmp_instance = nullptr; - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -void OpenMPInternal::validate_partition_impl(const int nthreads, - int &num_partitions, - int &partition_size) { - if (nthreads == 1) { - num_partitions = 1; - partition_size = 1; - } else if (num_partitions < 1 && partition_size < 1) { - int idle = nthreads; - for (int np = 2; np <= nthreads; ++np) { - for (int ps = 1; ps <= nthreads / np; ++ps) { - if (nthreads - np * ps < idle) { - idle = nthreads - np * ps; - num_partitions = np; - partition_size = ps; - } - if (idle == 0) { - break; - } - } - } - } else if (num_partitions < 1 && partition_size > 0) { - if (partition_size <= nthreads) { - num_partitions = nthreads / partition_size; - } else { - num_partitions = 1; - partition_size = nthreads; - } - } else if (num_partitions > 0 && partition_size < 1) { - if (num_partitions <= nthreads) { - partition_size = nthreads / num_partitions; - } else { - num_partitions = nthreads; - partition_size = 1; - } - } else if (num_partitions * partition_size > nthreads) { - int idle = nthreads; - const int NP = num_partitions; - const int PS = partition_size; - for (int np = NP; np > 0; --np) { - for (int ps = PS; ps > 0; --ps) { - if ((np * ps <= nthreads) && (nthreads - np * ps < idle)) { - idle = nthreads - np * ps; - num_partitions = np; - partition_size = ps; - } - if (idle == 0) { - break; - } - } - } - } -} -#endif - -void OpenMPInternal::clear_thread_data() { - const size_t member_bytes = - sizeof(int64_t) * - HostThreadTeamData::align_to_int64(sizeof(HostThreadTeamData)); - - const int old_alloc_bytes = - m_pool[0] ? (member_bytes + m_pool[0]->scratch_bytes()) : 0; - - OpenMP::memory_space space; - -#pragma omp parallel num_threads(m_pool_size) - { - const int rank = omp_get_thread_num(); - - if (nullptr != m_pool[rank]) { - m_pool[rank]->disband_pool(); - - space.deallocate(m_pool[rank], old_alloc_bytes); - - m_pool[rank] = nullptr; - } - } - /* END #pragma omp parallel */ -} - -void OpenMPInternal::resize_thread_data(size_t pool_reduce_bytes, - size_t team_reduce_bytes, - size_t team_shared_bytes, - size_t thread_local_bytes) { - const size_t member_bytes = - sizeof(int64_t) * - HostThreadTeamData::align_to_int64(sizeof(HostThreadTeamData)); - - HostThreadTeamData *root = m_pool[0]; - - const size_t old_pool_reduce = root ? root->pool_reduce_bytes() : 0; - const size_t old_team_reduce = root ? root->team_reduce_bytes() : 0; - const size_t old_team_shared = root ? root->team_shared_bytes() : 0; - const size_t old_thread_local = root ? root->thread_local_bytes() : 0; - const size_t old_alloc_bytes = - root ? (member_bytes + root->scratch_bytes()) : 0; - - // Allocate if any of the old allocation is tool small: - - const bool allocate = (old_pool_reduce < pool_reduce_bytes) || - (old_team_reduce < team_reduce_bytes) || - (old_team_shared < team_shared_bytes) || - (old_thread_local < thread_local_bytes); - - if (allocate) { - if (pool_reduce_bytes < old_pool_reduce) { - pool_reduce_bytes = old_pool_reduce; - } - if (team_reduce_bytes < old_team_reduce) { - team_reduce_bytes = old_team_reduce; - } - if (team_shared_bytes < old_team_shared) { - team_shared_bytes = old_team_shared; - } - if (thread_local_bytes < old_thread_local) { - thread_local_bytes = old_thread_local; - } - - const size_t alloc_bytes = - member_bytes + - HostThreadTeamData::scratch_size(pool_reduce_bytes, team_reduce_bytes, - team_shared_bytes, thread_local_bytes); - - OpenMP::memory_space space; - - memory_fence(); - -#pragma omp parallel num_threads(m_pool_size) - { - const int rank = omp_get_thread_num(); - - if (nullptr != m_pool[rank]) { - m_pool[rank]->disband_pool(); - - space.deallocate(m_pool[rank], old_alloc_bytes); - } - - void *ptr = nullptr; - try { - ptr = space.allocate(alloc_bytes); - } catch ( - Kokkos::Experimental::RawMemoryAllocationFailure const &failure) { - // For now, just rethrow the error message the existing way - Kokkos::Impl::throw_runtime_exception(failure.get_error_message()); - } - - m_pool[rank] = new (ptr) HostThreadTeamData(); - - m_pool[rank]->scratch_assign(((char *)ptr) + member_bytes, alloc_bytes, - pool_reduce_bytes, team_reduce_bytes, - team_shared_bytes, thread_local_bytes); - - memory_fence(); - } - /* END #pragma omp parallel */ - - HostThreadTeamData::organize_pool(m_pool, m_pool_size); - } -} - -OpenMPInternal &OpenMPInternal::singleton() { - static OpenMPInternal *self = nullptr; - if (self == nullptr) { - self = new OpenMPInternal(get_current_max_threads()); - } - - return *self; -} - -int OpenMPInternal::get_current_max_threads() noexcept { - // Using omp_get_max_threads(); is problematic in conjunction with - // Hwloc on Intel (essentially an initial call to the OpenMP runtime - // without a parallel region before will set a process mask for a single core - // The runtime will than bind threads for a parallel region to other cores on - // the entering the first parallel region and make the process mask the - // aggregate of the thread masks. The intend seems to be to make serial code - // run fast, if you compile with OpenMP enabled but don't actually use - // parallel regions or so static int omp_max_threads = omp_get_max_threads(); - - int count = 0; -#pragma omp parallel - { -#pragma omp atomic - ++count; - } - return count; -} - -void OpenMPInternal::initialize(int thread_count) { - if (m_initialized) { - Kokkos::abort( - "Calling OpenMP::initialize after OpenMP::finalize is illegal\n"); - } - - if (omp_in_parallel()) { - std::string msg("Kokkos::OpenMP::initialize ERROR : in parallel"); - Kokkos::Impl::throw_runtime_exception(msg); - } - - { - if (Kokkos::show_warnings() && nullptr == std::getenv("OMP_PROC_BIND")) { - printf( - "Kokkos::OpenMP::initialize WARNING: OMP_PROC_BIND environment " - "variable not set\n"); - printf( - " In general, for best performance with OpenMP 4.0 or better set " - "OMP_PROC_BIND=spread and OMP_PLACES=threads\n"); - printf(" For best performance with OpenMP 3.1 set OMP_PROC_BIND=true\n"); - printf(" For unit testing set OMP_PROC_BIND=false\n"); - } - - OpenMP::memory_space space; - - // Before any other call to OMP query the maximum number of threads - // and save the value for re-initialization unit testing. - - Impl::g_openmp_hardware_max_threads = get_current_max_threads(); - - int process_num_threads = Impl::g_openmp_hardware_max_threads; - - if (Kokkos::hwloc::available()) { - process_num_threads = Kokkos::hwloc::get_available_numa_count() * - Kokkos::hwloc::get_available_cores_per_numa() * - Kokkos::hwloc::get_available_threads_per_core(); - } - - // if thread_count < 0, use g_openmp_hardware_max_threads; - // if thread_count == 0, set g_openmp_hardware_max_threads to - // process_num_threads if thread_count > 0, set - // g_openmp_hardware_max_threads to thread_count - if (thread_count < 0) { - thread_count = Impl::g_openmp_hardware_max_threads; - } else if (thread_count == 0) { - if (Impl::g_openmp_hardware_max_threads != process_num_threads) { - Impl::g_openmp_hardware_max_threads = process_num_threads; - omp_set_num_threads(Impl::g_openmp_hardware_max_threads); - } - } else { - if (Kokkos::show_warnings() && thread_count > process_num_threads) { - printf( - "Kokkos::OpenMP::initialize WARNING: You are likely " - "oversubscribing your CPU cores.\n"); - printf(" process threads available : %3d, requested thread : %3d\n", - process_num_threads, thread_count); - } - Impl::g_openmp_hardware_max_threads = thread_count; - omp_set_num_threads(Impl::g_openmp_hardware_max_threads); - } - -// setup thread local -#pragma omp parallel num_threads(Impl::g_openmp_hardware_max_threads) - { - Impl::t_openmp_hardware_id = omp_get_thread_num(); - Impl::SharedAllocationRecord::tracking_enable(); - } - - auto &instance = OpenMPInternal::singleton(); - instance.m_pool_size = Impl::g_openmp_hardware_max_threads; - - // New, unified host thread team data: - { - size_t pool_reduce_bytes = 32 * thread_count; - size_t team_reduce_bytes = 32 * thread_count; - size_t team_shared_bytes = 1024 * thread_count; - size_t thread_local_bytes = 1024; - - instance.resize_thread_data(pool_reduce_bytes, team_reduce_bytes, - team_shared_bytes, thread_local_bytes); - } - } - - // Check for over-subscription - if (Kokkos::show_warnings() && - (Impl::mpi_ranks_per_node() * long(thread_count) > - Impl::processors_per_node())) { - std::cerr << "Kokkos::OpenMP::initialize WARNING: You are likely " - "oversubscribing your CPU cores." - << std::endl; - std::cerr << " Detected: " - << Impl::processors_per_node() << " cores per node." << std::endl; - std::cerr << " Detected: " - << Impl::mpi_ranks_per_node() << " MPI_ranks per node." - << std::endl; - std::cerr << " Requested: " - << thread_count << " threads per process." << std::endl; - } - // Init the array for used for arbitrarily sized atomics - init_lock_array_host_space(); - - m_initialized = true; -} - -void OpenMPInternal::finalize() { - if (omp_in_parallel()) { - std::string msg("Kokkos::OpenMP::finalize ERROR "); - if (this != &singleton()) msg.append(": not initialized"); - if (omp_in_parallel()) msg.append(": in parallel"); - Kokkos::Impl::throw_runtime_exception(msg); - } - - if (this == &singleton()) { - auto const &instance = singleton(); - // Silence Cuda Warning - const int nthreads = - instance.m_pool_size <= Impl::g_openmp_hardware_max_threads - ? Impl::g_openmp_hardware_max_threads - : instance.m_pool_size; - (void)nthreads; - -#pragma omp parallel num_threads(nthreads) - { - Impl::t_openmp_hardware_id = 0; - Impl::SharedAllocationRecord::tracking_disable(); - } - - // allow main thread to track - Impl::SharedAllocationRecord::tracking_enable(); - - Impl::g_openmp_hardware_max_threads = 1; - } - - m_initialized = false; - - Kokkos::Profiling::finalize(); -} - -void OpenMPInternal::print_configuration(std::ostream &s) const { - s << "Kokkos::OpenMP"; - - if (m_initialized) { - const int numa_count = 1; - const int core_per_numa = Impl::g_openmp_hardware_max_threads; - const int thread_per_core = 1; - - s << " thread_pool_topology[ " << numa_count << " x " << core_per_numa - << " x " << thread_per_core << " ]" << std::endl; - } else { - s << " not initialized" << std::endl; - } -} - -bool OpenMPInternal::verify_is_initialized(const char *const label) const { - if (!m_initialized) { - std::cerr << "Kokkos::OpenMP " << label - << " : ERROR OpenMP is not initialized" << std::endl; - } - return m_initialized; -} -} // namespace Impl - -//---------------------------------------------------------------------------- - -OpenMP::OpenMP() -#ifdef KOKKOS_IMPL_WORKAROUND_ICE_IN_TRILINOS_WITH_OLD_INTEL_COMPILERS - : m_space_instance(&Impl::OpenMPInternal::singleton()) { -} -#else - : m_space_instance(&Impl::OpenMPInternal::singleton(), - [](Impl::OpenMPInternal *) {}) { - Impl::OpenMPInternal::singleton().verify_is_initialized( - "OpenMP instance constructor"); -} -#endif - -int OpenMP::impl_get_current_max_threads() noexcept { - return Impl::OpenMPInternal::get_current_max_threads(); -} - -void OpenMP::impl_initialize(InitializationSettings const &settings) { - Impl::OpenMPInternal::singleton().initialize( - settings.has_num_threads() ? settings.get_num_threads() : -1); -} - -void OpenMP::impl_finalize() { Impl::OpenMPInternal::singleton().finalize(); } - -void OpenMP::print_configuration(std::ostream &os, bool /*verbose*/) const { - os << "Host Parallel Execution Space:\n"; - os << " KOKKOS_ENABLE_OPENMP: yes\n"; - - os << "OpenMP Atomics:\n"; - os << " KOKKOS_ENABLE_OPENMP_ATOMICS: "; -#ifdef KOKKOS_ENABLE_OPENMP_ATOMICS - os << "yes\n"; -#else - os << "no\n"; -#endif - - os << "\nOpenMP Runtime Configuration:\n"; - - m_space_instance->print_configuration(os); -} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -std::vector OpenMP::partition(...) { return std::vector(1); } - -OpenMP OpenMP::create_instance(...) { return OpenMP(); } -#endif - -int OpenMP::concurrency() { return Impl::g_openmp_hardware_max_threads; } - -void OpenMP::fence(const std::string &name) const { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{1}, []() {}); -} - -namespace Impl { - -int g_openmp_space_factory_initialized = - initialize_space_factory("050_OpenMP"); - -} // namespace Impl - -#ifdef KOKKOS_ENABLE_CXX14 -namespace Tools { -namespace Experimental { -constexpr DeviceType DeviceTypeTraits::id; -} -} // namespace Tools -#endif - -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Instance.hpp b/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Instance.hpp deleted file mode 100644 index 1a2ee95a79..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Instance.hpp +++ /dev/null @@ -1,366 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMP_INSTANCE_HPP -#define KOKKOS_OPENMP_INSTANCE_HPP - -#include -#if !defined(_OPENMP) && !defined(__CUDA_ARCH__) && \ - !defined(__HIP_DEVICE_COMPILE__) && !defined(__SYCL_DEVICE_ONLY__) -#error \ - "You enabled Kokkos OpenMP support without enabling OpenMP in the compiler!" -#endif - -#include - -#include -#include - -#include - -#include -#include - -#include - -namespace Kokkos { -namespace Impl { - -class OpenMPInternal; - -extern int g_openmp_hardware_max_threads; - -extern thread_local int t_openmp_hardware_id; -// FIXME_OPENMP we can remove this after we remove partition_master -extern thread_local OpenMPInternal* t_openmp_instance; - -struct OpenMPTraits { - static int constexpr MAX_THREAD_COUNT = 512; -}; - -class OpenMPInternal { - private: - OpenMPInternal(int arg_pool_size) - : m_pool_size{arg_pool_size}, m_level{omp_get_level()}, m_pool() {} - - ~OpenMPInternal() { clear_thread_data(); } - - static int get_current_max_threads() noexcept; - - bool m_initialized = false; - - int m_pool_size; - int m_level; - - HostThreadTeamData* m_pool[OpenMPTraits::MAX_THREAD_COUNT]; - - public: - friend class Kokkos::OpenMP; - - static OpenMPInternal& singleton(); - - void initialize(int thread_cound); - - void finalize(); - - void clear_thread_data(); - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - KOKKOS_DEPRECATED static void validate_partition(const int nthreads, - int& num_partitions, - int& partition_size) { - validate_partition_impl(nthreads, num_partitions, partition_size); - } - static void validate_partition_impl(const int nthreads, int& num_partitions, - int& partition_size); -#endif - - void resize_thread_data(size_t pool_reduce_bytes, size_t team_reduce_bytes, - size_t team_shared_bytes, size_t thread_local_bytes); - - HostThreadTeamData* get_thread_data() const noexcept { - return m_pool[m_level == omp_get_level() ? 0 : omp_get_thread_num()]; - } - - HostThreadTeamData* get_thread_data(int i) const noexcept { - return m_pool[i]; - } - - bool is_initialized() const { return m_initialized; } - - bool verify_is_initialized(const char* const label) const; - - void print_configuration(std::ostream& s) const; -}; - -} // namespace Impl -inline bool OpenMP::impl_is_initialized() noexcept { - return Impl::OpenMPInternal::singleton().is_initialized(); -} - -inline bool OpenMP::in_parallel(OpenMP const&) noexcept { - // FIXME_OPENMP We are forced to use t_openmp_instance because the function is - // static and does not use the OpenMP object - return ((Impl::OpenMPInternal::singleton().m_level < omp_get_level()) && - (!Impl::t_openmp_instance || - Impl::t_openmp_instance->m_level < omp_get_level())); -} - -inline int OpenMP::impl_thread_pool_size() noexcept { - // FIXME_OPENMP We are forced to use t_openmp_instance because the function is - // static - return OpenMP::in_parallel() - ? omp_get_num_threads() - : (Impl::t_openmp_instance - ? Impl::t_openmp_instance->m_pool_size - : Impl::OpenMPInternal::singleton().m_pool_size); -} - -KOKKOS_INLINE_FUNCTION -int OpenMP::impl_thread_pool_rank() noexcept { - // FIXME_OPENMP We are forced to use t_openmp_instance because the function is - // static - KOKKOS_IF_ON_HOST( - (return Impl::t_openmp_instance ? 0 : omp_get_thread_num();)) - - KOKKOS_IF_ON_DEVICE((return -1;)) -} - -inline void OpenMP::impl_static_fence(std::string const& name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - []() {}); -} - -inline bool OpenMP::is_asynchronous(OpenMP const& /*instance*/) noexcept { - return false; -} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -template -KOKKOS_DEPRECATED void OpenMP::partition_master(F const& f, int num_partitions, - int partition_size) { -#if _OPENMP >= 201511 - if (omp_get_max_active_levels() > 1) { -#else - if (omp_get_nested()) { -#endif - using Exec = Impl::OpenMPInternal; - - Exec* prev_instance = &Impl::OpenMPInternal::singleton(); - - Exec::validate_partition_impl(prev_instance->m_pool_size, num_partitions, - partition_size); - - OpenMP::memory_space space; - -#pragma omp parallel num_threads(num_partitions) - { - Exec thread_local_instance(partition_size); - Impl::t_openmp_instance = &thread_local_instance; - - size_t pool_reduce_bytes = 32 * partition_size; - size_t team_reduce_bytes = 32 * partition_size; - size_t team_shared_bytes = 1024 * partition_size; - size_t thread_local_bytes = 1024; - - thread_local_instance.resize_thread_data( - pool_reduce_bytes, team_reduce_bytes, team_shared_bytes, - thread_local_bytes); - - omp_set_num_threads(partition_size); - f(omp_get_thread_num(), omp_get_num_threads()); - Impl::t_openmp_instance = nullptr; - } - } else { - // nested openmp not enabled - f(0, 1); - } -} -#endif - -namespace Experimental { - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -template <> -class MasterLock { - public: - void lock() { omp_set_lock(&m_lock); } - void unlock() { omp_unset_lock(&m_lock); } - bool try_lock() { return static_cast(omp_test_lock(&m_lock)); } - - KOKKOS_DEPRECATED MasterLock() { omp_init_lock(&m_lock); } - ~MasterLock() { omp_destroy_lock(&m_lock); } - - MasterLock(MasterLock const&) = delete; - MasterLock(MasterLock&&) = delete; - MasterLock& operator=(MasterLock const&) = delete; - MasterLock& operator=(MasterLock&&) = delete; - - private: - omp_lock_t m_lock; -}; -#endif - -template <> -class UniqueToken { - private: - using buffer_type = Kokkos::View; - int m_count; - buffer_type m_buffer_view; - uint32_t volatile* m_buffer; - - public: - using execution_space = OpenMP; - using size_type = int; - - /// \brief create object size for concurrency on the given instance - /// - /// This object should not be shared between instances - UniqueToken(execution_space const& = execution_space()) noexcept - : m_count(::Kokkos::OpenMP::impl_thread_pool_size()), - m_buffer_view(buffer_type()), - m_buffer(nullptr) {} - - UniqueToken(size_type max_size, execution_space const& = execution_space()) - : m_count(max_size), - m_buffer_view("UniqueToken::m_buffer_view", - ::Kokkos::Impl::concurrent_bitset::buffer_bound(m_count)), - m_buffer(m_buffer_view.data()) {} - - /// \brief upper bound for acquired values, i.e. 0 <= value < size() - KOKKOS_INLINE_FUNCTION - int size() const noexcept { - KOKKOS_IF_ON_HOST((return m_count;)) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /// \brief acquire value such that 0 <= value < size() - KOKKOS_INLINE_FUNCTION - int acquire() const noexcept { - KOKKOS_IF_ON_HOST( - (if (m_count >= ::Kokkos::OpenMP::impl_thread_pool_size()) return :: - Kokkos::OpenMP::impl_thread_pool_rank(); - const ::Kokkos::pair result = - ::Kokkos::Impl::concurrent_bitset::acquire_bounded( - m_buffer, m_count, ::Kokkos::Impl::clock_tic() % m_count); - - if (result.first < 0) { - ::Kokkos::abort( - "UniqueToken failure to acquire tokens, no tokens " - "available"); - } - - return result.first;)) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /// \brief release a value acquired by generate - KOKKOS_INLINE_FUNCTION - void release(int i) const noexcept { - KOKKOS_IF_ON_HOST( - (if (m_count < ::Kokkos::OpenMP::impl_thread_pool_size()) { - ::Kokkos::Impl::concurrent_bitset::release(m_buffer, i); - })) - - KOKKOS_IF_ON_DEVICE(((void)i;)) - } -}; - -template <> -class UniqueToken { - public: - using execution_space = OpenMP; - using size_type = int; - - /// \brief create object size for concurrency on the given instance - /// - /// This object should not be shared between instances - UniqueToken(execution_space const& = execution_space()) noexcept {} - - /// \brief upper bound for acquired values, i.e. 0 <= value < size() - KOKKOS_INLINE_FUNCTION - int size() const noexcept { - KOKKOS_IF_ON_HOST((return Kokkos::Impl::g_openmp_hardware_max_threads;)) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /// \brief acquire value such that 0 <= value < size() - KOKKOS_INLINE_FUNCTION - int acquire() const noexcept { - KOKKOS_IF_ON_HOST((return Kokkos::Impl::t_openmp_hardware_id;)) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /// \brief release a value acquired by generate - KOKKOS_INLINE_FUNCTION - void release(int) const noexcept {} -}; - -} // namespace Experimental - -inline int OpenMP::impl_thread_pool_size(int depth) { - return depth < 2 ? impl_thread_pool_size() : 1; -} - -KOKKOS_INLINE_FUNCTION -int OpenMP::impl_hardware_thread_id() noexcept { - KOKKOS_IF_ON_HOST((return Impl::t_openmp_hardware_id;)) - - KOKKOS_IF_ON_DEVICE((return -1;)) -} - -inline int OpenMP::impl_max_hardware_threads() noexcept { - return Impl::g_openmp_hardware_max_threads; -} - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Parallel.hpp b/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Parallel.hpp deleted file mode 100644 index 94c465dc2e..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Parallel.hpp +++ /dev/null @@ -1,1262 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMP_PARALLEL_HPP -#define KOKKOS_OPENMP_PARALLEL_HPP - -#include -#if defined(KOKKOS_ENABLE_OPENMP) - -#include -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#define KOKKOS_PRAGMA_IVDEP_IF_ENABLED -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#undef KOKKOS_PRAGMA_IVDEP_IF_ENABLED -#define KOKKOS_PRAGMA_IVDEP_IF_ENABLED _Pragma("ivdep") -#endif - -#ifndef KOKKOS_COMPILER_NVHPC -#define KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE , m_policy.chunk_size() -#else -#define KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE -#endif - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, Kokkos::OpenMP> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const Policy m_policy; - - inline static void exec_range(const FunctorType& functor, const Member ibeg, - const Member iend) { - KOKKOS_PRAGMA_IVDEP_IF_ENABLED - for (auto iwork = ibeg; iwork < iend; ++iwork) { - exec_work(functor, iwork); - } - } - - template - inline static std::enable_if_t::value && - std::is_same::value> - exec_work(const FunctorType& functor, const Member iwork) { - functor(iwork); - } - - template - inline static std::enable_if_t::value && - std::is_same::value> - exec_work(const FunctorType& functor, const Member iwork) { - functor(WorkTag{}, iwork); - } - - template - std::enable_if_t::value> - execute_parallel() const { - // prevent bug in NVHPC 21.9/CUDA 11.4 (entering zero iterations loop) - if (m_policy.begin() >= m_policy.end()) return; -#pragma omp parallel for schedule(dynamic KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE) \ - num_threads(OpenMP::impl_thread_pool_size()) - KOKKOS_PRAGMA_IVDEP_IF_ENABLED - for (auto iwork = m_policy.begin(); iwork < m_policy.end(); ++iwork) { - exec_work(m_functor, iwork); - } - } - - template - std::enable_if_t::value> - execute_parallel() const { -#pragma omp parallel for schedule(static KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE) \ - num_threads(OpenMP::impl_thread_pool_size()) - KOKKOS_PRAGMA_IVDEP_IF_ENABLED - for (auto iwork = m_policy.begin(); iwork < m_policy.end(); ++iwork) { - exec_work(m_functor, iwork); - } - } - - public: - inline void execute() const { - if (OpenMP::in_parallel()) { - exec_range(m_functor, m_policy.begin(), m_policy.end()); - return; - } - -#ifndef KOKKOS_INTERNAL_DISABLE_NATIVE_OPENMP - execute_parallel(); -#else - constexpr bool is_dynamic = - std::is_same::value; -#pragma omp parallel num_threads(OpenMP::impl_thread_pool_size()) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - - data.set_work_partition(m_policy.end() - m_policy.begin(), - m_policy.chunk_size()); - - if (is_dynamic) { - // Make sure work partition is set before stealing - if (data.pool_rendezvous()) data.pool_rendezvous_release(); - } - - std::pair range(0, 0); - - do { - range = is_dynamic ? data.get_work_stealing_chunk() - : data.get_work_partition(); - - exec_range(m_functor, range.first + m_policy.begin(), - range.second + m_policy.begin()); - - } while (is_dynamic && 0 <= range.first); - } -#endif - } - - inline ParallelFor(const FunctorType& arg_functor, Policy arg_policy) - : m_instance(nullptr), m_functor(arg_functor), m_policy(arg_policy) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - } -}; - -// MDRangePolicy impl -template -class ParallelFor, - Kokkos::OpenMP> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - using WorkTag = typename MDRangePolicy::work_tag; - - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using iterate_type = typename Kokkos::Impl::HostIterateTile< - MDRangePolicy, FunctorType, typename MDRangePolicy::work_tag, void>; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; // construct as RangePolicy( 0, num_tiles - // ).set_chunk_size(1) in ctor - - inline static void exec_range(const MDRangePolicy& mdr_policy, - const FunctorType& functor, const Member ibeg, - const Member iend) { - KOKKOS_PRAGMA_IVDEP_IF_ENABLED - for (Member iwork = ibeg; iwork < iend; ++iwork) { - iterate_type(mdr_policy, functor)(iwork); - } - } - - template - typename std::enable_if_t::value> - execute_parallel() const { -#pragma omp parallel for schedule(dynamic KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE) \ - num_threads(OpenMP::impl_thread_pool_size()) - KOKKOS_PRAGMA_IVDEP_IF_ENABLED - for (auto iwork = m_policy.begin(); iwork < m_policy.end(); ++iwork) { - iterate_type(m_mdr_policy, m_functor)(iwork); - } - } - - template - typename std::enable_if::value>::type - execute_parallel() const { -#pragma omp parallel for schedule(static KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE) \ - num_threads(OpenMP::impl_thread_pool_size()) - KOKKOS_PRAGMA_IVDEP_IF_ENABLED - for (auto iwork = m_policy.begin(); iwork < m_policy.end(); ++iwork) { - iterate_type(m_mdr_policy, m_functor)(iwork); - } - } - - public: - inline void execute() const { - if (OpenMP::in_parallel()) { - ParallelFor::exec_range(m_mdr_policy, m_functor, m_policy.begin(), - m_policy.end()); - return; - } - -#ifndef KOKKOS_INTERNAL_DISABLE_NATIVE_OPENMP - execute_parallel(); -#else - constexpr bool is_dynamic = - std::is_same::value; - -#pragma omp parallel num_threads(OpenMP::impl_thread_pool_size()) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - - data.set_work_partition(m_policy.end() - m_policy.begin(), - m_policy.chunk_size()); - - if (is_dynamic) { - // Make sure work partition is set before stealing - if (data.pool_rendezvous()) data.pool_rendezvous_release(); - } - - std::pair range(0, 0); - - do { - range = is_dynamic ? data.get_work_stealing_chunk() - : data.get_work_partition(); - - ParallelFor::exec_range(m_mdr_policy, m_functor, - range.first + m_policy.begin(), - range.second + m_policy.begin()); - - } while (is_dynamic && 0 <= range.first); - } - // END #pragma omp parallel -#endif - } - - inline ParallelFor(const FunctorType& arg_functor, MDRangePolicy arg_policy) - : m_instance(nullptr), - m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - } - template - static int max_tile_size_product(const Policy&, const Functor&) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelReduce, ReducerType, - Kokkos::OpenMP> { - private: - using Policy = Kokkos::RangePolicy; - - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - std::conditional_t::value, WorkTag, - void>; - - // Static Assert WorkTag void if ReducerType not InvalidType - using Analysis = - FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType& functor, const Member ibeg, const Member iend, - reference_type update) { - for (Member iwork = ibeg; iwork < iend; ++iwork) { - functor(iwork, update); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType& functor, const Member ibeg, const Member iend, - reference_type update) { - const TagType t{}; - for (Member iwork = ibeg; iwork < iend; ++iwork) { - functor(t, iwork, update); - } - } - - public: - inline void execute() const { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - if (m_policy.end() <= m_policy.begin()) { - if (m_result_ptr) { - final_reducer.init(m_result_ptr); - final_reducer.final(m_result_ptr); - } - return; - } - enum { - is_dynamic = std::is_same::value - }; - - const size_t pool_reduce_bytes = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - - m_instance->resize_thread_data(pool_reduce_bytes, 0 // team_reduce_bytes - , - 0 // team_shared_bytes - , - 0 // thread_local_bytes - ); - - const int pool_size = OpenMP::impl_thread_pool_size(); -#pragma omp parallel num_threads(pool_size) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - - data.set_work_partition(m_policy.end() - m_policy.begin(), - m_policy.chunk_size()); - - if (is_dynamic) { - // Make sure work partition is set before stealing - if (data.pool_rendezvous()) data.pool_rendezvous_release(); - } - - reference_type update = final_reducer.init( - reinterpret_cast(data.pool_reduce_local())); - - std::pair range(0, 0); - - do { - range = is_dynamic ? data.get_work_stealing_chunk() - : data.get_work_partition(); - - ParallelReduce::template exec_range( - m_functor, range.first + m_policy.begin(), - range.second + m_policy.begin(), update); - - } while (is_dynamic && 0 <= range.first); - } - - // Reduction: - - const pointer_type ptr = - pointer_type(m_instance->get_thread_data(0)->pool_reduce_local()); - - for (int i = 1; i < pool_size; ++i) { - final_reducer.join( - ptr, reinterpret_cast( - m_instance->get_thread_data(i)->pool_reduce_local())); - } - - final_reducer.final(ptr); - - if (m_result_ptr) { - const int n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - - for (int j = 0; j < n; ++j) { - m_result_ptr[j] = ptr[j]; - } - } - } - - //---------------------------------------- - - template - inline ParallelReduce( - const FunctorType& arg_functor, Policy arg_policy, - const ViewType& arg_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = nullptr) - : m_instance(nullptr), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_view.data()) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } - - inline ParallelReduce(const FunctorType& arg_functor, Policy arg_policy, - const ReducerType& reducer) - : m_instance(nullptr), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } -}; - -// MDRangePolicy impl -template -class ParallelReduce, ReducerType, - Kokkos::OpenMP> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - - using WorkTag = typename MDRangePolicy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - std::conditional_t::value, WorkTag, - void>; - - using Analysis = FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - - using iterate_type = - typename Kokkos::Impl::HostIterateTile; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; // construct as RangePolicy( 0, num_tiles - // ).set_chunk_size(1) in ctor - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - inline static void exec_range(const MDRangePolicy& mdr_policy, - const FunctorType& functor, const Member ibeg, - const Member iend, reference_type update) { - for (Member iwork = ibeg; iwork < iend; ++iwork) { - iterate_type(mdr_policy, functor, update)(iwork); - } - } - - public: - inline void execute() const { - enum { - is_dynamic = std::is_same::value - }; - - const size_t pool_reduce_bytes = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - - m_instance->resize_thread_data(pool_reduce_bytes, 0 // team_reduce_bytes - , - 0 // team_shared_bytes - , - 0 // thread_local_bytes - ); - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - const int pool_size = OpenMP::impl_thread_pool_size(); -#pragma omp parallel num_threads(pool_size) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - - data.set_work_partition(m_policy.end() - m_policy.begin(), - m_policy.chunk_size()); - - if (is_dynamic) { - // Make sure work partition is set before stealing - if (data.pool_rendezvous()) data.pool_rendezvous_release(); - } - - reference_type update = final_reducer.init( - reinterpret_cast(data.pool_reduce_local())); - - std::pair range(0, 0); - - do { - range = is_dynamic ? data.get_work_stealing_chunk() - : data.get_work_partition(); - - ParallelReduce::exec_range(m_mdr_policy, m_functor, - range.first + m_policy.begin(), - range.second + m_policy.begin(), update); - - } while (is_dynamic && 0 <= range.first); - } - // END #pragma omp parallel - - // Reduction: - - const pointer_type ptr = - pointer_type(m_instance->get_thread_data(0)->pool_reduce_local()); - - for (int i = 1; i < pool_size; ++i) { - final_reducer.join( - ptr, reinterpret_cast( - m_instance->get_thread_data(i)->pool_reduce_local())); - } - - final_reducer.final(ptr); - - if (m_result_ptr) { - const int n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - - for (int j = 0; j < n; ++j) { - m_result_ptr[j] = ptr[j]; - } - } - } - - //---------------------------------------- - - template - inline ParallelReduce( - const FunctorType& arg_functor, MDRangePolicy arg_policy, - const ViewType& arg_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = nullptr) - : m_instance(nullptr), - m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(InvalidType()), - m_result_ptr(arg_view.data()) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } - - inline ParallelReduce(const FunctorType& arg_functor, - MDRangePolicy arg_policy, const ReducerType& reducer) - : m_instance(nullptr), - m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(reducer), - m_result_ptr(reducer.view().data()) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } - template - static int max_tile_size_product(const Policy&, const Functor&) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelScan, - Kokkos::OpenMP> { - private: - using Policy = Kokkos::RangePolicy; - - using Analysis = - FunctorAnalysis; - - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const Policy m_policy; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType& functor, const Member ibeg, const Member iend, - reference_type update, const bool final) { - for (Member iwork = ibeg; iwork < iend; ++iwork) { - functor(iwork, update, final); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType& functor, const Member ibeg, const Member iend, - reference_type update, const bool final) { - const TagType t{}; - for (Member iwork = ibeg; iwork < iend; ++iwork) { - functor(t, iwork, update, final); - } - } - - public: - inline void execute() const { - const int value_count = Analysis::value_count(m_functor); - const size_t pool_reduce_bytes = 2 * Analysis::value_size(m_functor); - - m_instance->resize_thread_data(pool_reduce_bytes, 0 // team_reduce_bytes - , - 0 // team_shared_bytes - , - 0 // thread_local_bytes - ); - -#pragma omp parallel num_threads(OpenMP::impl_thread_pool_size()) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - typename Analysis::Reducer final_reducer(&m_functor); - - const WorkRange range(m_policy, omp_get_thread_num(), - omp_get_num_threads()); - - reference_type update_sum = final_reducer.init( - reinterpret_cast(data.pool_reduce_local())); - - ParallelScan::template exec_range( - m_functor, range.begin(), range.end(), update_sum, false); - - if (data.pool_rendezvous()) { - pointer_type ptr_prev = nullptr; - - const int n = omp_get_num_threads(); - - for (int i = 0; i < n; ++i) { - pointer_type ptr = - (pointer_type)data.pool_member(i)->pool_reduce_local(); - - if (i) { - for (int j = 0; j < value_count; ++j) { - ptr[j + value_count] = ptr_prev[j + value_count]; - } - final_reducer.join(ptr + value_count, ptr_prev); - } else { - final_reducer.init(ptr + value_count); - } - - ptr_prev = ptr; - } - - data.pool_rendezvous_release(); - } - - reference_type update_base = final_reducer.reference( - reinterpret_cast(data.pool_reduce_local()) + - value_count); - - ParallelScan::template exec_range( - m_functor, range.begin(), range.end(), update_base, true); - } - } - - //---------------------------------------- - - inline ParallelScan(const FunctorType& arg_functor, const Policy& arg_policy) - : m_instance(nullptr), m_functor(arg_functor), m_policy(arg_policy) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - } - - //---------------------------------------- -}; - -template -class ParallelScanWithTotal, - ReturnType, Kokkos::OpenMP> { - private: - using Policy = Kokkos::RangePolicy; - - using Analysis = - FunctorAnalysis; - - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const Policy m_policy; - ReturnType& m_returnvalue; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType& functor, const Member ibeg, const Member iend, - reference_type update, const bool final) { - for (Member iwork = ibeg; iwork < iend; ++iwork) { - functor(iwork, update, final); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType& functor, const Member ibeg, const Member iend, - reference_type update, const bool final) { - const TagType t{}; - for (Member iwork = ibeg; iwork < iend; ++iwork) { - functor(t, iwork, update, final); - } - } - - public: - inline void execute() const { - const int value_count = Analysis::value_count(m_functor); - const size_t pool_reduce_bytes = 2 * Analysis::value_size(m_functor); - - m_instance->resize_thread_data(pool_reduce_bytes, 0 // team_reduce_bytes - , - 0 // team_shared_bytes - , - 0 // thread_local_bytes - ); - -#pragma omp parallel num_threads(OpenMP::impl_thread_pool_size()) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - typename Analysis::Reducer final_reducer(&m_functor); - - const WorkRange range(m_policy, omp_get_thread_num(), - omp_get_num_threads()); - reference_type update_sum = final_reducer.init( - reinterpret_cast(data.pool_reduce_local())); - - ParallelScanWithTotal::template exec_range( - m_functor, range.begin(), range.end(), update_sum, false); - - if (data.pool_rendezvous()) { - pointer_type ptr_prev = nullptr; - - const int n = omp_get_num_threads(); - - for (int i = 0; i < n; ++i) { - pointer_type ptr = - (pointer_type)data.pool_member(i)->pool_reduce_local(); - - if (i) { - for (int j = 0; j < value_count; ++j) { - ptr[j + value_count] = ptr_prev[j + value_count]; - } - final_reducer.join(ptr + value_count, ptr_prev); - } else { - final_reducer.init(ptr + value_count); - } - - ptr_prev = ptr; - } - - data.pool_rendezvous_release(); - } - - reference_type update_base = final_reducer.reference( - reinterpret_cast(data.pool_reduce_local()) + - value_count); - - ParallelScanWithTotal::template exec_range( - m_functor, range.begin(), range.end(), update_base, true); - - if (omp_get_thread_num() == omp_get_num_threads() - 1) { - m_returnvalue = update_base; - } - } - } - - //---------------------------------------- - - inline ParallelScanWithTotal(const FunctorType& arg_functor, - const Policy& arg_policy, - ReturnType& arg_returnvalue) - : m_instance(nullptr), - m_functor(arg_functor), - m_policy(arg_policy), - m_returnvalue(arg_returnvalue) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - } - - //---------------------------------------- -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::OpenMP> { - private: - enum { TEAM_REDUCE_SIZE = 512 }; - - using Policy = - Kokkos::Impl::TeamPolicyInternal; - using WorkTag = typename Policy::work_tag; - using SchedTag = typename Policy::schedule_type::type; - using Member = typename Policy::member_type; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const Policy m_policy; - const size_t m_shmem_size; - - template - inline static std::enable_if_t<(std::is_void::value)> exec_team( - const FunctorType& functor, HostThreadTeamData& data, - const int league_rank_begin, const int league_rank_end, - const int league_size) { - for (int r = league_rank_begin; r < league_rank_end;) { - functor(Member(data, r, league_size)); - - if (++r < league_rank_end) { - // Don't allow team members to lap one another - // so that they don't overwrite shared memory. - if (data.team_rendezvous()) { - data.team_rendezvous_release(); - } - } - } - } - - template - inline static std::enable_if_t<(!std::is_void::value)> exec_team( - const FunctorType& functor, HostThreadTeamData& data, - const int league_rank_begin, const int league_rank_end, - const int league_size) { - const TagType t{}; - - for (int r = league_rank_begin; r < league_rank_end;) { - functor(t, Member(data, r, league_size)); - - if (++r < league_rank_end) { - // Don't allow team members to lap one another - // so that they don't overwrite shared memory. - if (data.team_rendezvous()) { - data.team_rendezvous_release(); - } - } - } - } - - public: - inline void execute() const { - enum { is_dynamic = std::is_same::value }; - - const size_t pool_reduce_size = 0; // Never shrinks - const size_t team_reduce_size = TEAM_REDUCE_SIZE * m_policy.team_size(); - const size_t team_shared_size = m_shmem_size; - const size_t thread_local_size = 0; // Never shrinks - - m_instance->resize_thread_data(pool_reduce_size, team_reduce_size, - team_shared_size, thread_local_size); - -#pragma omp parallel num_threads(OpenMP::impl_thread_pool_size()) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - - const int active = data.organize_team(m_policy.team_size()); - - if (active) { - data.set_work_partition( - m_policy.league_size(), - (0 < m_policy.chunk_size() ? m_policy.chunk_size() - : m_policy.team_iter())); - } - - if (is_dynamic) { - // Must synchronize to make sure each team has set its - // partition before beginning the work stealing loop. - if (data.pool_rendezvous()) data.pool_rendezvous_release(); - } - - if (active) { - std::pair range(0, 0); - - do { - range = is_dynamic ? data.get_work_stealing_chunk() - : data.get_work_partition(); - - ParallelFor::template exec_team(m_functor, data, range.first, - range.second, - m_policy.league_size()); - - } while (is_dynamic && 0 <= range.first); - } - - data.disband_team(); - } - } - - inline ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_instance(nullptr), - m_functor(arg_functor), - m_policy(arg_policy), - m_shmem_size(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - } -}; - -//---------------------------------------------------------------------------- - -template -class ParallelReduce, - ReducerType, Kokkos::OpenMP> { - private: - enum { TEAM_REDUCE_SIZE = 512 }; - - using Policy = - Kokkos::Impl::TeamPolicyInternal; - - using WorkTag = typename Policy::work_tag; - using SchedTag = typename Policy::schedule_type::type; - using Member = typename Policy::member_type; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - std::conditional_t::value, WorkTag, - void>; - - using Analysis = - FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - OpenMPInternal* m_instance; - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const int m_shmem_size; - - template - inline static std::enable_if_t<(std::is_void::value)> exec_team( - const FunctorType& functor, HostThreadTeamData& data, - reference_type& update, const int league_rank_begin, - const int league_rank_end, const int league_size) { - for (int r = league_rank_begin; r < league_rank_end;) { - functor(Member(data, r, league_size), update); - - if (++r < league_rank_end) { - // Don't allow team members to lap one another - // so that they don't overwrite shared memory. - if (data.team_rendezvous()) { - data.team_rendezvous_release(); - } - } - } - } - - template - inline static std::enable_if_t<(!std::is_void::value)> exec_team( - const FunctorType& functor, HostThreadTeamData& data, - reference_type& update, const int league_rank_begin, - const int league_rank_end, const int league_size) { - const TagType t{}; - - for (int r = league_rank_begin; r < league_rank_end;) { - functor(t, Member(data, r, league_size), update); - - if (++r < league_rank_end) { - // Don't allow team members to lap one another - // so that they don't overwrite shared memory. - if (data.team_rendezvous()) { - data.team_rendezvous_release(); - } - } - } - } - - public: - inline void execute() const { - enum { is_dynamic = std::is_same::value }; - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - if (m_policy.league_size() == 0 || m_policy.team_size() == 0) { - if (m_result_ptr) { - final_reducer.init(m_result_ptr); - final_reducer.final(m_result_ptr); - } - return; - } - - const size_t pool_reduce_size = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - - const size_t team_reduce_size = TEAM_REDUCE_SIZE * m_policy.team_size(); - const size_t team_shared_size = m_shmem_size + m_policy.scratch_size(1); - const size_t thread_local_size = 0; // Never shrinks - - m_instance->resize_thread_data(pool_reduce_size, team_reduce_size, - team_shared_size, thread_local_size); - - const int pool_size = OpenMP::impl_thread_pool_size(); -#pragma omp parallel num_threads(pool_size) - { - HostThreadTeamData& data = *(m_instance->get_thread_data()); - - const int active = data.organize_team(m_policy.team_size()); - - if (active) { - data.set_work_partition( - m_policy.league_size(), - (0 < m_policy.chunk_size() ? m_policy.chunk_size() - : m_policy.team_iter())); - } - - if (is_dynamic) { - // Must synchronize to make sure each team has set its - // partition before beginning the work stealing loop. - if (data.pool_rendezvous()) data.pool_rendezvous_release(); - } - - if (active) { - reference_type update = final_reducer.init( - reinterpret_cast(data.pool_reduce_local())); - - std::pair range(0, 0); - - do { - range = is_dynamic ? data.get_work_stealing_chunk() - : data.get_work_partition(); - - ParallelReduce::template exec_team(m_functor, data, update, - range.first, range.second, - m_policy.league_size()); - - } while (is_dynamic && 0 <= range.first); - } else { - final_reducer.init( - reinterpret_cast(data.pool_reduce_local())); - } - - data.disband_team(); - - // This thread has updated 'pool_reduce_local()' with its - // contributions to the reduction. The parallel region is - // about to terminate and the master thread will load and - // reduce each 'pool_reduce_local()' contribution. - // Must 'memory_fence()' to guarantee that storing the update to - // 'pool_reduce_local()' will complete before this thread - // exits the parallel region. - - memory_fence(); - } - - // Reduction: - - const pointer_type ptr = - pointer_type(m_instance->get_thread_data(0)->pool_reduce_local()); - - for (int i = 1; i < pool_size; ++i) { - final_reducer.join( - ptr, reinterpret_cast( - m_instance->get_thread_data(i)->pool_reduce_local())); - } - - final_reducer.final(ptr); - - if (m_result_ptr) { - const int n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - - for (int j = 0; j < n; ++j) { - m_result_ptr[j] = ptr[j]; - } - } - } - - //---------------------------------------- - - template - inline ParallelReduce( - const FunctorType& arg_functor, const Policy& arg_policy, - const ViewType& arg_result, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = nullptr) - : m_instance(nullptr), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_shmem_size(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - } - - inline ParallelReduce(const FunctorType& arg_functor, Policy arg_policy, - const ReducerType& reducer) - : m_instance(nullptr), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_shmem_size(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) { - if (t_openmp_instance) { - m_instance = t_openmp_instance; - } else { - m_instance = arg_policy.space().impl_internal_space_instance(); - } - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#undef KOKKOS_PRAGMA_IVDEP_IF_ENABLED -#undef KOKKOS_OPENMP_OPTIONAL_CHUNK_SIZE - -#endif -#endif /* KOKKOS_OPENMP_PARALLEL_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Task.cpp b/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Task.cpp deleted file mode 100644 index 4babcf03d9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_Task.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#if defined(KOKKOS_ENABLE_OPENMP) && defined(KOKKOS_ENABLE_TASKDAG) - -#include - -#include -#include -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template class TaskQueue; - -HostThreadTeamData& HostThreadTeamDataSingleton::singleton() { - static HostThreadTeamDataSingleton s; - return s; -} - -HostThreadTeamDataSingleton::HostThreadTeamDataSingleton() - : HostThreadTeamData() { - Kokkos::OpenMP::memory_space space; - const size_t num_pool_reduce_bytes = 32; - const size_t num_team_reduce_bytes = 32; - const size_t num_team_shared_bytes = 1024; - const size_t num_thread_local_bytes = 1024; - const size_t alloc_bytes = HostThreadTeamData::scratch_size( - num_pool_reduce_bytes, num_team_reduce_bytes, num_team_shared_bytes, - num_thread_local_bytes); - - void* ptr = nullptr; - try { - ptr = space.allocate(alloc_bytes); - } catch (Kokkos::Experimental::RawMemoryAllocationFailure const& f) { - // For now, just rethrow the error message with a note - // Note that this could, in turn, trigger an out of memory exception, - // but it's pretty unlikely, so we won't worry about it for now. - // TODO reasonable error message when `std::string` causes OOM error - Kokkos::Impl::throw_runtime_exception( - std::string("Failure to allocate scratch memory: ") + - f.get_error_message()); - } - - HostThreadTeamData::scratch_assign( - ptr, alloc_bytes, num_pool_reduce_bytes, num_team_reduce_bytes, - num_team_shared_bytes, num_thread_local_bytes); -} - -HostThreadTeamDataSingleton::~HostThreadTeamDataSingleton() { - Kokkos::OpenMP::memory_space space; - space.deallocate(HostThreadTeamData::scratch_buffer(), - static_cast(HostThreadTeamData::scratch_bytes())); -} - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -#else -void KOKKOS_CORE_SRC_OPENMP_KOKKOS_OPENMP_TASK_PREVENT_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_OPENMP ) && defined( \ - KOKKOS_ENABLE_TASKDAG ) */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_WorkGraphPolicy.hpp b/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_WorkGraphPolicy.hpp deleted file mode 100644 index 55d9c58607..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMP/Kokkos_OpenMP_WorkGraphPolicy.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMP_WORKGRAPHPOLICY_HPP -#define KOKKOS_OPENMP_WORKGRAPHPOLICY_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::OpenMP> { - private: - using Policy = Kokkos::WorkGraphPolicy; - - Policy m_policy; - FunctorType m_functor; - - template - std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - m_functor(w); - } - - template - std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - const TagType t{}; - m_functor(t, w); - } - - public: - inline void execute() { -#pragma omp parallel num_threads(OpenMP::impl_thread_pool_size()) - { - // Spin until COMPLETED_TOKEN. - // END_TOKEN indicates no work is currently available. - - for (std::int32_t w = Policy::END_TOKEN; - Policy::COMPLETED_TOKEN != (w = m_policy.pop_work());) { - if (Policy::END_TOKEN != w) { - exec_one(w); - m_policy.completed_work(w); - } - } - } - } - - inline ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_policy(arg_policy), m_functor(arg_functor) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif /* #define KOKKOS_OPENMP_WORKGRAPHPOLICY_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp deleted file mode 100644 index 5ff9bf32b6..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp +++ /dev/null @@ -1,249 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include - -/*--------------------------------------------------------------------------*/ - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Experimental { -/* Default allocation mechanism */ -OpenMPTargetSpace::OpenMPTargetSpace() {} - -void* OpenMPTargetSpace::impl_allocate( - - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - static_assert(sizeof(void*) == sizeof(uintptr_t), - "Error sizeof(void*) != sizeof(uintptr_t)"); - - void* ptr; - - ptr = omp_target_alloc(arg_alloc_size, omp_get_default_device()); - - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void* OpenMPTargetSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} - -void* OpenMPTargetSpace::allocate(const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} - -void OpenMPTargetSpace::impl_deallocate( - const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - if (arg_alloc_ptr) { - omp_target_free(arg_alloc_ptr, omp_get_default_device()); - } -} - -void OpenMPTargetSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void OpenMPTargetSpace::deallocate(const char* arg_label, - void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const - -{ - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} - -} // namespace Experimental -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -#ifdef KOKKOS_ENABLE_DEBUG -SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::OpenMPTargetSpace, void>::s_root_record; -#endif - -SharedAllocationRecord::~SharedAllocationRecord() { - auto alloc_size = SharedAllocationRecord::m_alloc_size; - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - alloc_size, (alloc_size - sizeof(SharedAllocationHeader))); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::OpenMPTargetSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, arg_label); - - // TODO DeepCopy - // DeepCopy - Kokkos::Impl::DeepCopy( - RecordBase::m_alloc_ptr, &header, sizeof(SharedAllocationHeader)); - Kokkos::fence( - "SharedAllocationRecord::SharedAllocationRecord(): fence after copying header from " - "HostSpace"); -} - -//---------------------------------------------------------------------------- - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ -/* -namespace Kokkos { -namespace { - const unsigned HOST_SPACE_ATOMIC_MASK = 0xFFFF; - const unsigned HOST_SPACE_ATOMIC_XOR_MASK = 0x5A39; - static int HOST_SPACE_ATOMIC_LOCKS[HOST_SPACE_ATOMIC_MASK+1]; -} - -namespace Impl { -void init_lock_array_host_space() { - static int is_initialized = 0; - if(! is_initialized) - for(int i = 0; i < static_cast (HOST_SPACE_ATOMIC_MASK+1); i++) - HOST_SPACE_ATOMIC_LOCKS[i] = 0; -} - -bool lock_address_host_space(void* ptr) { - return 0 == atomic_compare_exchange( &HOST_SPACE_ATOMIC_LOCKS[ - (( size_t(ptr) >> 2 ) & HOST_SPACE_ATOMIC_MASK) ^ -HOST_SPACE_ATOMIC_XOR_MASK] , 0 , 1); -} - -void unlock_address_host_space(void* ptr) { - atomic_exchange( &HOST_SPACE_ATOMIC_LOCKS[ - (( size_t(ptr) >> 2 ) & HOST_SPACE_ATOMIC_MASK) ^ -HOST_SPACE_ATOMIC_XOR_MASK] , 0); -} - -} -}*/ - -//============================================================================== -// {{{1 - -#include - -namespace Kokkos { -namespace Impl { - -// To avoid additional compilation cost for something that's (mostly?) not -// performance sensitive, we explicity instantiate these CRTP base classes here, -// where we have access to the associated *_timpl.hpp header files. -template class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::OpenMPTargetSpace>; -template class SharedAllocationRecordCommon< - Kokkos::Experimental::OpenMPTargetSpace>; - -} // end namespace Impl -} // end namespace Kokkos - -// end Explicit instantiations of CRTP Base classes }}}1 -//============================================================================== diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Abort.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Abort.hpp deleted file mode 100644 index ff07ce4f3a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Abort.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGET_ABORT_HPP -#define KOKKOS_OPENMPTARGET_ABORT_HPP - -#include -#ifdef KOKKOS_ENABLE_OPENMPTARGET - -namespace Kokkos { -namespace Impl { - -KOKKOS_INLINE_FUNCTION void OpenMPTarget_abort(char const *msg) { - fprintf(stderr, "%s.\n", msg); - std::abort(); -} - -} // namespace Impl -} // namespace Kokkos - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Error.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Error.hpp deleted file mode 100644 index 1ca30631af..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Error.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGET_ERROR_HPP -#define KOKKOS_OPENMPTARGET_ERROR_HPP - -#include -#include - -namespace Kokkos { -namespace Impl { - -inline void ompt_internal_safe_call(int e, const char* name, - const char* file = nullptr, - const int line = 0) { - if (e != 0) { - std::ostringstream out; - out << name << " return value of " << e << " indicates failure"; - if (file) { - out << " " << file << ":" << line; - } - throw_runtime_exception(out.str()); - } -} - -#define OMPT_SAFE_CALL(call) \ - Kokkos::Impl::ompt_internal_safe_call(call, #call, __FILE__, __LINE__) - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp deleted file mode 100644 index d3bec5aebf..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp +++ /dev/null @@ -1,175 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef KOKKOS_ENABLE_OPENMPTARGET - -// FIXME_OPENMPTARGET currently unused -/* -namespace Kokkos { -namespace Impl { -namespace { - -KOKKOS_INLINE_FUNCTION -int kokkos_omp_in_parallel(); - -KOKKOS_INLINE_FUNCTION -int kokkos_omp_in_parallel() { return omp_in_parallel(); } - -bool s_using_hwloc = false; - -} // namespace -} // namespace Impl -} // namespace Kokkos -*/ - -namespace Kokkos { -namespace Impl { - -void OpenMPTargetExec::verify_is_process(const char* const label) { - // Fails if the current task is in a parallel region or is not on the host. - if (omp_in_parallel() && (!omp_is_initial_device())) { - std::string msg(label); - msg.append(" ERROR: in parallel or on device"); - Kokkos::Impl::throw_runtime_exception(msg); - } -} - -void OpenMPTargetExec::verify_initialized(const char* const label) { - if (0 == Kokkos::Experimental::OpenMPTarget().impl_is_initialized()) { - std::string msg(label); - msg.append(" ERROR: not initialized"); - Kokkos::Impl::throw_runtime_exception(msg); - } -} - -void* OpenMPTargetExec::m_scratch_ptr = nullptr; -int64_t OpenMPTargetExec::m_scratch_size = 0; -int* OpenMPTargetExec::m_lock_array = nullptr; -int64_t OpenMPTargetExec::m_lock_size = 0; -uint32_t* OpenMPTargetExec::m_uniquetoken_ptr = nullptr; - -void OpenMPTargetExec::clear_scratch() { - Kokkos::Experimental::OpenMPTargetSpace space; - space.deallocate(m_scratch_ptr, m_scratch_size); - m_scratch_ptr = nullptr; - m_scratch_size = 0; -} - -void OpenMPTargetExec::clear_lock_array() { - if (m_lock_array != nullptr) { - Kokkos::Experimental::OpenMPTargetSpace space; - space.deallocate(m_lock_array, m_lock_size); - m_lock_array = nullptr; - m_lock_size = 0; - } -} - -void* OpenMPTargetExec::get_scratch_ptr() { return m_scratch_ptr; } - -void OpenMPTargetExec::resize_scratch(int64_t team_size, int64_t shmem_size_L0, - int64_t shmem_size_L1, - int64_t league_size) { - Kokkos::Experimental::OpenMPTargetSpace space; - const int64_t shmem_size = - shmem_size_L0 + shmem_size_L1; // L0 + L1 scratch memory per team. - const int64_t padding = shmem_size * 10 / 100; // Padding per team. - // Total amount of scratch memory allocated is depenedent - // on the maximum number of in-flight teams possible. - int64_t total_size = - (shmem_size + OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE + padding) * - std::min(MAX_ACTIVE_THREADS / team_size, league_size); - - if (total_size > m_scratch_size) { - space.deallocate(m_scratch_ptr, m_scratch_size); - m_scratch_size = total_size; - m_scratch_ptr = space.allocate(total_size); - } -} - -int* OpenMPTargetExec::get_lock_array(int num_teams) { - Kokkos::Experimental::OpenMPTargetSpace space; - int max_active_league_size = MAX_ACTIVE_THREADS / 32; - int lock_array_elem = - (num_teams > max_active_league_size) ? num_teams : max_active_league_size; - if (m_lock_size < (lock_array_elem * sizeof(int))) { - space.deallocate(m_lock_array, m_lock_size); - m_lock_size = lock_array_elem * sizeof(int); - m_lock_array = static_cast(space.allocate(m_lock_size)); - - // FIXME_OPENMPTARGET - Creating a target region here to initialize the - // lock_array with 0's fails. Hence creating an equivalent host array to - // achieve the same. Value of host array are then copied to the lock_array. - int* h_lock_array = static_cast( - omp_target_alloc(m_lock_size, omp_get_initial_device())); - - for (int i = 0; i < lock_array_elem; ++i) h_lock_array[i] = 0; - - OMPT_SAFE_CALL(omp_target_memcpy(m_lock_array, h_lock_array, m_lock_size, 0, - 0, omp_get_default_device(), - omp_get_initial_device())); - - omp_target_free(h_lock_array, omp_get_initial_device()); - } - - return m_lock_array; -} - -} // namespace Impl -} // namespace Kokkos - -#endif // KOKKOS_ENABLE_OPENMPTARGET diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.hpp deleted file mode 100644 index 52f5dcb83e..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.hpp +++ /dev/null @@ -1,1964 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGETEXEC_HPP -#define KOKKOS_OPENMPTARGETEXEC_HPP - -#include -#include - -#include -#include "Kokkos_OpenMPTarget_Abort.hpp" - -// FIXME_OPENMPTARGET - Using this macro to implement a workaround for -// hierarchical reducers. It avoids hitting the code path which we wanted to -// write but doesn't work. undef'ed at the end. -// Intel compilers prefer the non-workaround version. -#ifndef KOKKOS_ARCH_INTEL_GPU -#define KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND -#endif - -// FIXME_OPENMPTARGET - Using this macro to implement a workaround for -// hierarchical scan. It avoids hitting the code path which we wanted to -// write but doesn't work. undef'ed at the end. -#ifndef KOKKOS_ARCH_INTEL_GPU -#define KOKKOS_IMPL_TEAM_SCAN_WORKAROUND -#endif - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -struct OpenMPTargetReducerWrapper { - using value_type = typename Reducer::value_type; - - // Using a generic unknown Reducer for the OpenMPTarget backend is not - // implemented. - KOKKOS_INLINE_FUNCTION - static void join(value_type&, const value_type&) = delete; - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type&, const volatile value_type&) = delete; - - KOKKOS_INLINE_FUNCTION - static void init(value_type&) = delete; -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { dest += src; } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest += src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::sum(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { dest *= src; } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest *= src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::prod(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src < dest) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src < dest) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::min(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src > dest) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src > dest) dest = src; - } - - // Required - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::max(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest = dest && src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest && src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::land(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - using result_view_type = Kokkos::View; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest = dest || src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest || src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::lor(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest = dest & src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest & src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::band(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - public: - // Required - using value_type = std::remove_cv_t; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest = dest | src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest | src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val = reduction_identity::bor(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = ValLocScalar; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src.val < dest.val) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.val < dest.val) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.val = reduction_identity::min(); - val.loc = reduction_identity::min(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = ValLocScalar; - - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src.val > dest.val) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.val > dest.val) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.val = reduction_identity::max(); - val.loc = reduction_identity::min(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - - public: - // Required - using value_type = MinMaxScalar; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - } - if (src.max_val > dest.max_val) { - dest.max_val = src.max_val; - } - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - } - if (src.max_val > dest.max_val) { - dest.max_val = src.max_val; - } - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.max_val = reduction_identity::max(); - val.min_val = reduction_identity::min(); - } -}; - -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = MinMaxLocScalar; - - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - dest.min_loc = src.min_loc; - } - if (src.max_val > dest.max_val) { - dest.max_val = src.max_val; - dest.max_loc = src.max_loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - dest.min_loc = src.min_loc; - } - if (src.max_val > dest.max_val) { - dest.max_val = src.max_val; - dest.max_loc = src.max_loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.max_val = reduction_identity::max(); - val.min_val = reduction_identity::min(); - val.max_loc = reduction_identity::min(); - val.min_loc = reduction_identity::min(); - } -}; - -// -// specialize for MaxFirstLoc -// -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = ValLocScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (dest.val < src.val) { - dest = src; - } else if (!(src.val < dest.val)) { - dest.loc = (src.loc < dest.loc) ? src.loc : dest.loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (dest.val < src.val) { - dest = src; - } else if (!(src.val < dest.val)) { - dest.loc = (src.loc < dest.loc) ? src.loc : dest.loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.val = reduction_identity::max(); - val.loc = reduction_identity::min(); - } -#pragma omp end declare target -}; - -// -// specialize for MinFirstLoc -// -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = ValLocScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src.val < dest.val) { - dest = src; - } else if (!(dest.val < src.val)) { - dest.loc = (src.loc < dest.loc) ? src.loc : dest.loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.val < dest.val) { - dest = src; - } else if (!(dest.val < src.val)) { - dest.loc = (src.loc < dest.loc) ? src.loc : dest.loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.val = reduction_identity::min(); - val.loc = reduction_identity::min(); - } -#pragma omp end declare target -}; - -// -// specialize for MinMaxFirstLastLoc -// -template -struct OpenMPTargetReducerWrapper> { - private: - using scalar_type = std::remove_cv_t; - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = MinMaxLocScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - dest.min_loc = src.min_loc; - } else if (!(dest.min_val < src.min_val)) { - dest.min_loc = (src.min_loc < dest.min_loc) ? src.min_loc : dest.min_loc; - } - - if (dest.max_val < src.max_val) { - dest.max_val = src.max_val; - dest.max_loc = src.max_loc; - } else if (!(src.max_val < dest.max_val)) { - dest.max_loc = (src.max_loc > dest.max_loc) ? src.max_loc : dest.max_loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - dest.min_loc = src.min_loc; - } else if (!(dest.min_val < src.min_val)) { - dest.min_loc = (src.min_loc < dest.min_loc) ? src.min_loc : dest.min_loc; - } - - if (dest.max_val < src.max_val) { - dest.max_val = src.max_val; - dest.max_loc = src.max_loc; - } else if (!(src.max_val < dest.max_val)) { - dest.max_loc = (src.max_loc > dest.max_loc) ? src.max_loc : dest.max_loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.max_val = reduction_identity::max(); - val.min_val = reduction_identity::min(); - val.max_loc = reduction_identity::max(); - val.min_loc = reduction_identity::min(); - } -#pragma omp end declare target -}; - -// -// specialize for FirstLoc -// -template -struct OpenMPTargetReducerWrapper> { - private: - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = FirstLocScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest.min_loc_true = (src.min_loc_true < dest.min_loc_true) - ? src.min_loc_true - : dest.min_loc_true; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.min_loc_true = (src.min_loc_true < dest.min_loc_true) - ? src.min_loc_true - : dest.min_loc_true; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.min_loc_true = reduction_identity::min(); - } -#pragma omp end declare target -}; - -// -// specialize for LastLoc -// -template -struct OpenMPTargetReducerWrapper> { - private: - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = LastLocScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest.max_loc_true = (src.max_loc_true > dest.max_loc_true) - ? src.max_loc_true - : dest.max_loc_true; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.max_loc_true = (src.max_loc_true > dest.max_loc_true) - ? src.max_loc_true - : dest.max_loc_true; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.max_loc_true = reduction_identity::max(); - } -#pragma omp end declare target -}; - -// -// specialize for StdIsPartitioned -// -template -struct OpenMPTargetReducerWrapper> { - private: - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = StdIsPartScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest.max_loc_true = (dest.max_loc_true < src.max_loc_true) - ? src.max_loc_true - : dest.max_loc_true; - - dest.min_loc_false = (dest.min_loc_false < src.min_loc_false) - ? dest.min_loc_false - : src.min_loc_false; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.max_loc_true = (dest.max_loc_true < src.max_loc_true) - ? src.max_loc_true - : dest.max_loc_true; - - dest.min_loc_false = (dest.min_loc_false < src.min_loc_false) - ? dest.min_loc_false - : src.min_loc_false; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.max_loc_true = ::Kokkos::reduction_identity::max(); - val.min_loc_false = ::Kokkos::reduction_identity::min(); - } -#pragma omp end declare target -}; - -// -// specialize for StdPartitionPoint -// -template -struct OpenMPTargetReducerWrapper> { - private: - using index_type = std::remove_cv_t; - - public: - // Required - using value_type = StdPartPointScalar; - -// WORKAROUND OPENMPTARGET -// This pragma omp declare target should not be necessary, but Intel compiler -// fails without it -#pragma omp declare target - // Required - KOKKOS_INLINE_FUNCTION - static void join(value_type& dest, const value_type& src) { - dest.min_loc_false = (dest.min_loc_false < src.min_loc_false) - ? dest.min_loc_false - : src.min_loc_false; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.min_loc_false = (dest.min_loc_false < src.min_loc_false) - ? dest.min_loc_false - : src.min_loc_false; - } - - KOKKOS_INLINE_FUNCTION - static void init(value_type& val) { - val.min_loc_false = ::Kokkos::reduction_identity::min(); - } -#pragma omp end declare target -}; - -/* -template -class OpenMPTargetReducerWrapper { - public: - const ReducerType& reducer; - using value_type = typename ReducerType::value_type; - value_type& value; - - KOKKOS_INLINE_FUNCTION - void join(const value_type& upd) { - reducer.join(value,upd); - } - - KOKKOS_INLINE_FUNCTION - void init(const value_type& upd) { - reducer.init(value,upd); - } -};*/ - -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -//---------------------------------------------------------------------------- -/** \brief Data for OpenMPTarget thread execution */ - -class OpenMPTargetExec { - public: - // FIXME_OPENMPTARGET - Currently the maximum number of - // teams possible is calculated based on NVIDIA's Volta GPU. In - // future this value should be based on the chosen architecture for the - // OpenMPTarget backend. - enum { MAX_ACTIVE_THREADS = 2080 * 80 }; - enum { MAX_ACTIVE_TEAMS = MAX_ACTIVE_THREADS / 32 }; - - private: - static void* scratch_ptr; - - public: - static void verify_is_process(const char* const); - static void verify_initialized(const char* const); - - static int* get_lock_array(int num_teams); - static void* get_scratch_ptr(); - static void clear_scratch(); - static void clear_lock_array(); - static void resize_scratch(int64_t team_reduce_bytes, - int64_t team_shared_bytes, - int64_t thread_local_bytes, int64_t league_size); - - static void* m_scratch_ptr; - static int64_t m_scratch_size; - static int* m_lock_array; - static int64_t m_lock_size; - static uint32_t* m_uniquetoken_ptr; -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -class OpenMPTargetExecTeamMember { - public: - enum { TEAM_REDUCE_SIZE = 512 }; - - /** \brief Thread states for team synchronization */ - enum { Active = 0, Rendezvous = 1 }; - - using execution_space = Kokkos::Experimental::OpenMPTarget; - using scratch_memory_space = execution_space::scratch_memory_space; - - scratch_memory_space m_team_shared; - size_t m_team_scratch_size[2]; - int m_team_rank; - int m_team_size; - int m_league_rank; - int m_league_size; - int m_vector_length; - int m_vector_lane; - int m_shmem_block_index; - void* m_glb_scratch; - void* m_reduce_scratch; - - public: - KOKKOS_INLINE_FUNCTION - const execution_space::scratch_memory_space& team_shmem() const { - return m_team_shared.set_team_thread_mode(0, 1, 0); - } - - // set_team_thread_mode routine parameters for future understanding: - // first parameter - scratch level. - // second parameter - size multiplier for advancing scratch ptr after a - // request was serviced. third parameter - offset size multiplier from current - // scratch ptr when returning a ptr for a request. - KOKKOS_INLINE_FUNCTION - const execution_space::scratch_memory_space& team_scratch(int level) const { - return m_team_shared.set_team_thread_mode(level, 1, 0); - } - - KOKKOS_INLINE_FUNCTION - const execution_space::scratch_memory_space& thread_scratch(int level) const { - return m_team_shared.set_team_thread_mode(level, team_size(), team_rank()); - } - - KOKKOS_INLINE_FUNCTION int league_rank() const { return m_league_rank; } - KOKKOS_INLINE_FUNCTION int league_size() const { return m_league_size; } - KOKKOS_INLINE_FUNCTION int team_rank() const { return m_team_rank; } - KOKKOS_INLINE_FUNCTION int team_size() const { return m_team_size; } - KOKKOS_INLINE_FUNCTION void* impl_reduce_scratch() const { - return m_reduce_scratch; - } - - KOKKOS_INLINE_FUNCTION void team_barrier() const { -#pragma omp barrier - } - - template - KOKKOS_INLINE_FUNCTION void team_broadcast(ValueType& value, - int thread_id) const { - // Make sure there is enough scratch space: - using type = std::conditional_t<(sizeof(ValueType) < TEAM_REDUCE_SIZE), - ValueType, void>; - type* team_scratch = - reinterpret_cast(static_cast(m_glb_scratch) + - TEAM_REDUCE_SIZE * omp_get_team_num()); -#pragma omp barrier - if (team_rank() == thread_id) *team_scratch = value; -#pragma omp barrier - value = *team_scratch; - } - - template - KOKKOS_INLINE_FUNCTION void team_broadcast(const Closure& f, ValueType& value, - const int& thread_id) const { - f(value); - team_broadcast(value, thread_id); - } - - // FIXME_OPENMPTARGET this function has the wrong interface and currently - // ignores the reducer passed. - template - KOKKOS_INLINE_FUNCTION ValueType team_reduce(const ValueType& value, - const JoinOp&) const { -#pragma omp barrier - - using value_type = ValueType; - // const JoinLambdaAdapter op(op_in); - - // Make sure there is enough scratch space: - using type = std::conditional_t<(sizeof(value_type) < TEAM_REDUCE_SIZE), - value_type, void>; - - const int n_values = TEAM_REDUCE_SIZE / sizeof(value_type); - type* team_scratch = - reinterpret_cast(static_cast(m_glb_scratch) + - TEAM_REDUCE_SIZE * omp_get_team_num()); - for (int i = m_team_rank; i < n_values; i += m_team_size) { - team_scratch[i] = value_type(); - } - -#pragma omp barrier - - for (int k = 0; k < m_team_size; k += n_values) { - if ((k <= m_team_rank) && (k + n_values > m_team_rank)) - team_scratch[m_team_rank % n_values] += value; -#pragma omp barrier - } - - for (int d = 1; d < n_values; d *= 2) { - if ((m_team_rank + d < n_values) && (m_team_rank % (2 * d) == 0)) { - team_scratch[m_team_rank] += team_scratch[m_team_rank + d]; - } -#pragma omp barrier - } - return team_scratch[0]; - } - /** \brief Intra-team exclusive prefix sum with team_rank() ordering - * with intra-team non-deterministic ordering accumulation. - * - * The global inter-team accumulation value will, at the end of the - * league's parallel execution, be the scan's total. - * Parallel execution ordering of the league's teams is non-deterministic. - * As such the base value for each team's scan operation is similarly - * non-deterministic. - */ - template - KOKKOS_INLINE_FUNCTION ArgType - team_scan(const ArgType& /*value*/, ArgType* const /*global_accum*/) const { - // FIXME_OPENMPTARGET - /* // Make sure there is enough scratch space: - using type = - std::conditional_t<(sizeof(ArgType) < TEAM_REDUCE_SIZE), ArgType, void>; - - volatile type * const work_value = ((type*) m_exec.scratch_thread()); - - *work_value = value ; - - memory_fence(); - - if ( team_fan_in() ) { - // The last thread to synchronize returns true, all other threads wait - for team_fan_out() - // m_team_base[0] == highest ranking team member - // m_team_base[ m_team_size - 1 ] == lowest ranking team member - // - // 1) copy from lower to higher rank, initialize lowest rank to zero - // 2) prefix sum from lowest to highest rank, skipping lowest rank - - type accum = 0 ; - - if ( global_accum ) { - for ( int i = m_team_size ; i-- ; ) { - type & val = *((type*) m_exec.pool_rev( m_team_base_rev + i - )->scratch_thread()); accum += val ; - } - accum = atomic_fetch_add( global_accum , accum ); - } - - for ( int i = m_team_size ; i-- ; ) { - type & val = *((type*) m_exec.pool_rev( m_team_base_rev + i - )->scratch_thread()); const type offset = accum ; accum += val ; val = - offset ; - } - - memory_fence(); - } - - team_fan_out(); - - return *work_value ;*/ - return ArgType(); - } - - /** \brief Intra-team exclusive prefix sum with team_rank() ordering. - * - * The highest rank thread can compute the reduction total as - * reduction_total = dev.team_scan( value ) + value ; - */ - template - KOKKOS_INLINE_FUNCTION Type team_scan(const Type& value) const { - return this->template team_scan(value, 0); - } - - //---------------------------------------- - // Private for the driver - - private: - using space = execution_space::scratch_memory_space; - - public: - // FIXME_OPENMPTARGET - 512(16*32) bytes at the begining of the scratch space - // for each league is saved for reduction. It should actually be based on the - // ValueType of the reduction variable. - inline OpenMPTargetExecTeamMember( - const int league_rank, const int league_size, const int team_size, - const int vector_length // const TeamPolicyInternal< OpenMPTarget, - // Properties ...> & team - , - void* const glb_scratch, const int shmem_block_index, - const size_t shmem_size_L0, const size_t shmem_size_L1) - : m_team_scratch_size{shmem_size_L0, shmem_size_L1}, - m_team_rank(0), - m_team_size(team_size), - m_league_rank(league_rank), - m_league_size(league_size), - m_vector_length(vector_length), - m_shmem_block_index(shmem_block_index), - m_glb_scratch(glb_scratch) { - const int omp_tid = omp_get_thread_num(); - - // The scratch memory allocated is a sum of TEAM_REDUCE_SIZE, L0 shmem size - // and L1 shmem size. TEAM_REDUCE_SIZE = 512 bytes saved per team for - // hierarchical reduction. There is an additional 10% of the requested - // scratch memory allocated per team as padding. Hence the product with 0.1. - const int reduce_offset = - m_shmem_block_index * - (shmem_size_L0 + shmem_size_L1 + - ((shmem_size_L0 + shmem_size_L1) * 0.1) + TEAM_REDUCE_SIZE); - const int l0_offset = reduce_offset + TEAM_REDUCE_SIZE; - const int l1_offset = l0_offset + shmem_size_L0; - m_team_shared = scratch_memory_space( - (static_cast(glb_scratch) + l0_offset), shmem_size_L0, - static_cast(glb_scratch) + l1_offset, shmem_size_L1); - m_reduce_scratch = static_cast(glb_scratch) + reduce_offset; - m_league_rank = league_rank; - m_team_rank = omp_tid; - m_vector_lane = 0; - } - - static inline int team_reduce_size() { return TEAM_REDUCE_SIZE; } -}; - -template -class TeamPolicyInternal - : public PolicyTraits { - public: - //! Tag this class as a kokkos execution policy - using execution_policy = TeamPolicyInternal; - - using traits = PolicyTraits; - - //---------------------------------------- - - template - inline static int team_size_max(const FunctorType&, const ParallelForTag&) { - return 256; - } - - template - inline static int team_size_max(const FunctorType&, - const ParallelReduceTag&) { - return 256; - } - - template - inline static int team_size_max(const FunctorType&, const ReducerType&, - const ParallelReduceTag&) { - return 256; - } - - template - inline static int team_size_recommended(const FunctorType&, - const ParallelForTag&) { - return 128; - } - - template - inline static int team_size_recommended(const FunctorType&, - const ParallelReduceTag&) { - return 128; - } - - template - inline static int team_size_recommended(const FunctorType&, - const ReducerType&, - const ParallelReduceTag&) { - return 128; - } - - //---------------------------------------- - - private: - int m_league_size; - int m_team_size; - int m_vector_length; - int m_team_alloc; - int m_team_iter; - std::array m_team_scratch_size; - std::array m_thread_scratch_size; - bool m_tune_team_size; - bool m_tune_vector_length; - constexpr const static size_t default_team_size = 256; - int m_chunk_size; - - inline void init(const int league_size_request, const int team_size_request, - const int vector_length_request) { - m_league_size = league_size_request; - - // Minimum team size should be 32 for OpenMPTarget backend. - if (team_size_request < 32) { - Kokkos::Impl::OpenMPTarget_abort( - "OpenMPTarget backend requires a minimum of 32 threads per team.\n"); - } else - m_team_size = team_size_request; - - m_vector_length = vector_length_request; - set_auto_chunk_size(); - } - - template - friend class TeamPolicyInternal; - - public: - // FIXME_OPENMPTARGET : Currently this routine is a copy of the Cuda - // implementation, but this has to be tailored to be architecture specific. - inline static int scratch_size_max(int level) { - return ( - level == 0 ? 1024 * 40 : // 48kB is the max for CUDA, but we need some - // for team_member.reduce etc. - 20 * 1024 * - 1024); // arbitrarily setting this to 20MB, for a Volta V100 - // that would give us about 3.2GB for 2 teams per SM - } - inline bool impl_auto_team_size() const { return m_tune_team_size; } - inline bool impl_auto_vector_length() const { return m_tune_vector_length; } - inline void impl_set_team_size(const size_t size) { m_team_size = size; } - inline void impl_set_vector_length(const size_t length) { - m_tune_vector_length = length; - } - inline int impl_vector_length() const { return m_vector_length; } -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - KOKKOS_DEPRECATED inline int vector_length() const { - return impl_vector_length(); - } -#endif - inline int team_size() const { return m_team_size; } - inline int league_size() const { return m_league_size; } - inline size_t scratch_size(const int& level, int team_size_ = -1) const { - if (team_size_ < 0) team_size_ = m_team_size; - return m_team_scratch_size[level] + - team_size_ * m_thread_scratch_size[level]; - } - - inline Kokkos::Experimental::OpenMPTarget space() const { - return Kokkos::Experimental::OpenMPTarget(); - } - - template - TeamPolicyInternal(const TeamPolicyInternal& p) - : m_league_size(p.m_league_size), - m_team_size(p.m_team_size), - m_vector_length(p.m_vector_length), - m_team_alloc(p.m_team_alloc), - m_team_iter(p.m_team_iter), - m_team_scratch_size(p.m_team_scratch_size), - m_thread_scratch_size(p.m_thread_scratch_size), - m_tune_team_size(p.m_tune_team_size), - m_tune_vector_length(p.m_tune_vector_length), - m_chunk_size(p.m_chunk_size) {} - - /** \brief Specify league size, request team size */ - TeamPolicyInternal(const typename traits::execution_space&, - int league_size_request, int team_size_request, - int vector_length_request = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(false), - m_tune_vector_length(false), - m_chunk_size(0) { - init(league_size_request, team_size_request, vector_length_request); - } - - TeamPolicyInternal(const typename traits::execution_space&, - int league_size_request, - const Kokkos::AUTO_t& /* team_size_request */ - , - int vector_length_request = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(true), - m_tune_vector_length(false), - m_chunk_size(0) { - init(league_size_request, default_team_size / vector_length_request, - vector_length_request); - } - - TeamPolicyInternal(const typename traits::execution_space&, - int league_size_request, - const Kokkos::AUTO_t& /* team_size_request */ - , - const Kokkos::AUTO_t& /* vector_length_request */) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(true), - m_tune_vector_length(true), - m_chunk_size(0) { - init(league_size_request, default_team_size, 1); - } - TeamPolicyInternal(const typename traits::execution_space&, - int league_size_request, int team_size_request, - const Kokkos::AUTO_t& /* vector_length_request */) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(false), - m_tune_vector_length(true), - m_chunk_size(0) { - init(league_size_request, team_size_request, 1); - } - - TeamPolicyInternal(int league_size_request, int team_size_request, - int vector_length_request = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(false), - m_tune_vector_length(false), - m_chunk_size(0) { - init(league_size_request, team_size_request, vector_length_request); - } - - TeamPolicyInternal(int league_size_request, - const Kokkos::AUTO_t& /* team_size_request */ - , - int vector_length_request = 1) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(true), - m_tune_vector_length(false), - m_chunk_size(0) { - init(league_size_request, default_team_size / vector_length_request, - vector_length_request); - } - - TeamPolicyInternal(int league_size_request, - const Kokkos::AUTO_t& /* team_size_request */ - , - const Kokkos::AUTO_t& /* vector_length_request */) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(true), - m_tune_vector_length(true), - m_chunk_size(0) { - init(league_size_request, default_team_size, 1); - } - TeamPolicyInternal(int league_size_request, int team_size_request, - const Kokkos::AUTO_t& /* vector_length_request */) - : m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_tune_team_size(false), - m_tune_vector_length(true), - m_chunk_size(0) { - init(league_size_request, team_size_request, 1); - } - inline static size_t vector_length_max() { - return 32; /* TODO: this is bad. Need logic that is compiler and backend - aware */ - } - inline int team_alloc() const { return m_team_alloc; } - inline int team_iter() const { return m_team_iter; } - - inline int chunk_size() const { return m_chunk_size; } - - /** \brief set chunk_size to a discrete value*/ - inline TeamPolicyInternal& set_chunk_size( - typename traits::index_type chunk_size_) { - m_chunk_size = chunk_size_; - return *this; - } - - /** \brief set per team scratch size for a specific level of the scratch - * hierarchy */ - inline TeamPolicyInternal& set_scratch_size(const int& level, - const PerTeamValue& per_team) { - m_team_scratch_size[level] = per_team.value; - return *this; - } - - /** \brief set per thread scratch size for a specific level of the scratch - * hierarchy */ - inline TeamPolicyInternal& set_scratch_size( - const int& level, const PerThreadValue& per_thread) { - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - /** \brief set per thread and per team scratch size for a specific level of - * the scratch hierarchy */ - inline TeamPolicyInternal& set_scratch_size( - const int& level, const PerTeamValue& per_team, - const PerThreadValue& per_thread) { - m_team_scratch_size[level] = per_team.value; - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - private: - /** \brief finalize chunk_size if it was set to AUTO*/ - inline void set_auto_chunk_size() { - int concurrency = 2048 * 128; - - if (concurrency == 0) concurrency = 1; - - if (m_chunk_size > 0) { - if (!Impl::is_integral_power_of_two(m_chunk_size)) - Kokkos::abort("TeamPolicy blocking granularity must be power of two"); - } - - int new_chunk_size = 1; - while (new_chunk_size * 100 * concurrency < m_league_size) - new_chunk_size *= 2; - if (new_chunk_size < 128) { - new_chunk_size = 1; - while ((new_chunk_size * 40 * concurrency < m_league_size) && - (new_chunk_size < 128)) - new_chunk_size *= 2; - } - m_chunk_size = new_chunk_size; - } - - public: - using member_type = Impl::OpenMPTargetExecTeamMember; -}; -} // namespace Impl - -} // namespace Kokkos - -namespace Kokkos { - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember> -TeamThreadRange(const Impl::OpenMPTargetExecTeamMember& thread, - const iType& count) { - return Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>(thread, count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - std::common_type_t, Impl::OpenMPTargetExecTeamMember> -TeamThreadRange(const Impl::OpenMPTargetExecTeamMember& thread, - const iType1& begin, const iType2& end) { - using iType = std::common_type_t; - return Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>(thread, iType(begin), - iType(end)); -} - -template -KOKKOS_INLINE_FUNCTION Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember> -ThreadVectorRange(const Impl::OpenMPTargetExecTeamMember& thread, - const iType& count) { - return Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>(thread, count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::ThreadVectorRangeBoundariesStruct< - std::common_type_t, Impl::OpenMPTargetExecTeamMember> -ThreadVectorRange(const Impl::OpenMPTargetExecTeamMember& thread, - const iType1& arg_begin, const iType2& arg_end) { - using iType = std::common_type_t; - return Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>(thread, iType(arg_begin), - iType(arg_end)); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember> -TeamVectorRange(const Impl::OpenMPTargetExecTeamMember& thread, - const iType& count) { - return Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>(thread, count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamVectorRangeBoundariesStruct< - std::common_type_t, Impl::OpenMPTargetExecTeamMember> -TeamVectorRange(const Impl::OpenMPTargetExecTeamMember& thread, - const iType1& arg_begin, const iType2& arg_end) { - using iType = std::common_type_t; - return Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>(thread, iType(arg_begin), - iType(arg_end)); -} - -KOKKOS_INLINE_FUNCTION -Impl::ThreadSingleStruct PerTeam( - const Impl::OpenMPTargetExecTeamMember& thread) { - return Impl::ThreadSingleStruct(thread); -} - -KOKKOS_INLINE_FUNCTION -Impl::VectorSingleStruct PerThread( - const Impl::OpenMPTargetExecTeamMember& thread) { - return Impl::VectorSingleStruct(thread); -} -} // namespace Kokkos - -namespace Kokkos { - -/** \brief Inter-thread parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all threads of the the calling thread team. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda) { -#pragma omp for nowait schedule(static, 1) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) lambda(i); -} - -/** \brief Inter-thread vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all threads of the the calling thread team - * and a summation of val is performed and put into result. - */ - -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ValueType& result) { - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - - ValueType* TeamThread_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - -#pragma omp barrier - TeamThread_scratch[0] = ValueType(); -#pragma omp barrier - - if constexpr (std::is_arithmetic::value) { -#pragma omp for reduction(+ : TeamThread_scratch[:1]) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - ValueType tmp = ValueType(); - lambda(i, tmp); - TeamThread_scratch[0] += tmp; - } - } else { -#pragma omp declare reduction(custom:ValueType : omp_out += omp_in) - -#pragma omp for reduction(custom : TeamThread_scratch[:1]) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - ValueType tmp = ValueType(); - lambda(i, tmp); - TeamThread_scratch[0] += tmp; - } - } - - result = TeamThread_scratch[0]; -} - -#if !defined(KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND) -// For some reason the actual version we wanted to write doesn't work -// and crashes. We should try this with every new compiler -// This is the variant we actually wanted to write -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ReducerType result) { - using ValueType = typename ReducerType::value_type; - -#pragma omp declare reduction( \ - custominner:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) - - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - - ValueType* TeamThread_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - -#pragma omp barrier - Impl::OpenMPTargetReducerWrapper::init(TeamThread_scratch[0]); -#pragma omp barrier - -#pragma omp for reduction(custominner : TeamThread_scratch[:1]) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - lambda(i, TeamThread_scratch[0]); - } - result.reference() = TeamThread_scratch[0]; -} -#else -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ReducerType result) { - using ValueType = typename ReducerType::value_type; - - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - - ValueType* TeamThread_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - -#pragma omp declare reduction( \ - omp_red_teamthread_reducer:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp barrier - ValueType tmp; - result.init(tmp); - TeamThread_scratch[0] = tmp; -#pragma omp barrier - - iType team_size = iType(omp_get_num_threads()); -#pragma omp for reduction(omp_red_teamthread_reducer \ - : TeamThread_scratch[:1]) schedule(static, 1) - for (iType t = 0; t < team_size; t++) { - ValueType tmp2; - result.init(tmp2); - - for (iType i = loop_boundaries.start + t; i < loop_boundaries.end; - i += team_size) { - lambda(i, tmp2); - } - - // FIXME_OPENMPTARGET: Join should work but doesn't. Every threads gets a - // private TeamThread_scratch[0] and at the end of the for-loop the `join` - // operation is performed by OpenMP itself and hence the simple assignment - // works. - // result.join(TeamThread_scratch[0], tmp2); - TeamThread_scratch[0] = tmp2; - } - - result.reference() = TeamThread_scratch[0]; -} -#endif // KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND - -/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling thread - * and a reduction of val is performed using JoinType(ValueType& val, const - * ValueType& update) and put into init_result. The input value of init_result - * is used as initializer for temporary variables of ValueType. Therefore the - * input value should be the neutral element with respect to the join operation - * (e.g. '0 for +-' or '1 for *'). - */ -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, const JoinType& join, ValueType& init_result) { - ValueType* TeamThread_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - - // FIXME_OPENMPTARGET: Still need to figure out how to get value_count here. - const int value_count = 1; - -#pragma omp barrier - TeamThread_scratch[0] = init_result; -#pragma omp barrier - -#pragma omp for - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - lambda(i, TeamThread_scratch[omp_get_num_threads() * value_count]); - } - - // Reduce all partial results within a team. - const int team_size = omp_get_num_threads(); - int tree_neighbor_offset = 1; - do { -#pragma omp for - for (int i = 0; i < team_size - tree_neighbor_offset; - i += 2 * tree_neighbor_offset) { - const int neighbor = i + tree_neighbor_offset; - join(lambda, &TeamThread_scratch[i * value_count], - &TeamThread_scratch[neighbor * value_count]); - } - tree_neighbor_offset *= 2; - } while (tree_neighbor_offset < team_size); - init_result = TeamThread_scratch[0]; -} - -// This is largely the same code as in HIP and CUDA except for the member name -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_bounds, - const FunctorType& lambda) { - using Analysis = Impl::FunctorAnalysis, - FunctorType>; - using value_type = typename Analysis::value_type; - - const auto start = loop_bounds.start; - const auto end = loop_bounds.end; - // Note this thing is called .member in the CUDA specialization of - // TeamThreadRangeBoundariesStruct - auto& member = loop_bounds.team; - const auto team_size = member.team_size(); - const auto team_rank = member.team_rank(); - -#if defined(KOKKOS_IMPL_TEAM_SCAN_WORKAROUND) - value_type scan_val = value_type(); - - if (team_rank == 0) { - for (iType i = start; i < end; ++i) { - lambda(i, scan_val, true); - } - } -#pragma omp barrier -#else - const auto nchunk = (end - start + team_size - 1) / team_size; - value_type accum = 0; - // each team has to process one or - // more chunks of the prefix scan - for (iType i = 0; i < nchunk; ++i) { - auto ii = start + i * team_size + team_rank; - // local accumulation for this chunk - value_type local_accum = 0; - // user updates value with prefix value - if (ii < loop_bounds.end) lambda(ii, local_accum, false); - // perform team scan - local_accum = member.team_scan(local_accum); - // add this blocks accum to total accumulation - auto val = accum + local_accum; - // user updates their data with total accumulation - if (ii < loop_bounds.end) lambda(ii, val, true); - // the last value needs to be propogated to next chunk - if (team_rank == team_size - 1) accum = val; - // broadcast last value to rest of the team - member.team_broadcast(accum, team_size - 1); - } -#endif -} - -} // namespace Kokkos - -namespace Kokkos { -/** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling thread. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda) { -#pragma omp simd - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) lambda(i); -} - -/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling thread - * and a summation of val is performed and put into result. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ValueType& result) { - ValueType vector_reduce = ValueType(); - - if constexpr (std::is_arithmetic::value) { -#pragma omp simd reduction(+ : vector_reduce) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - ValueType tmp = ValueType(); - lambda(i, tmp); - vector_reduce += tmp; - } - } else { -#pragma omp declare reduction(custom:ValueType : omp_out += omp_in) - -#pragma omp simd reduction(custom : vector_reduce) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - lambda(i, vector_reduce); - } - } - - result = vector_reduce; -} - -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ReducerType const& result) { - using ValueType = typename ReducerType::value_type; - -#pragma omp declare reduction( \ - custom:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) - - ValueType vector_reduce; - Impl::OpenMPTargetReducerWrapper::init(vector_reduce); - -#pragma omp simd reduction(custom : vector_reduce) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - lambda(i, vector_reduce); - } - - result.reference() = vector_reduce; -} - -/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling thread - * and a reduction of val is performed using JoinType(ValueType& val, const - * ValueType& update) and put into init_result. The input value of init_result - * is used as initializer for temporary variables of ValueType. Therefore the - * input value should be the neutral element with respect to the join operation - * (e.g. '0 for +-' or '1 for *'). - */ -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, const JoinType& join, ValueType& init_result) { - ValueType result = init_result; - - // FIXME_OPENMPTARGET think about omp simd - // join does not work with omp reduction clause - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - ValueType tmp = ValueType(); - lambda(i, tmp); - join(result, tmp); - } - - init_result = result; -} - -/** \brief Intra-thread vector parallel exclusive prefix sum. Executes - * lambda(iType i, ValueType & val, bool final) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes in the thread and a scan - * operation is performed. Depending on the target execution space the operator - * might be called twice: once with final=false and once with final=true. When - * final==true val contains the prefix sum value. The contribution of this "i" - * needs to be added to val no matter whether final==true or not. In a serial - * execution (i.e. team_size==1) the operator is only called once with - * final==true. Scan_val will be set to the final sum value over all vector - * lanes. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const FunctorType& lambda) { - using Analysis = Impl::FunctorAnalysis, - FunctorType>; - using value_type = typename Analysis::value_type; - - value_type scan_val = value_type(); - -#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP -#pragma ivdep -#endif - for (iType i = loop_boundaries.start; i < loop_boundaries.end; ++i) { - lambda(i, scan_val, true); - } -} - -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_TEAM_SCAN_WORKAROUND -#undef KOKKOS_IMPL_TEAM_SCAN_WORKAROUND -#endif - -namespace Kokkos { -/** \brief Intra-team vector parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling team. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda) { -#pragma omp for simd nowait schedule(static, 1) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) lambda(i); -} - -/** \brief Intra-team vector parallel_reduce. Executes lambda(iType i, - * ValueType & val) for each i=0..N-1. - * - * The range i=0..N-1 is mapped to all vector lanes of the the calling team - * and a summation of val is performed and put into result. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ValueType& result) { - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - - ValueType* TeamVector_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - -#pragma omp barrier - TeamVector_scratch[0] = ValueType(); -#pragma omp barrier - - if constexpr (std::is_arithmetic::value) { -#pragma omp for simd reduction(+ : TeamVector_scratch[:1]) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - ValueType tmp = ValueType(); - lambda(i, tmp); - TeamVector_scratch[0] += tmp; - } - } else { -#pragma omp declare reduction(custom:ValueType : omp_out += omp_in) - -#pragma omp for simd reduction(custom : TeamVector_scratch[:1]) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - ValueType tmp = ValueType(); - lambda(i, tmp); - TeamVector_scratch[0] += tmp; - } - } - - result = TeamVector_scratch[0]; -} - -#if !defined(KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND) -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ReducerType const& result) { - using ValueType = typename ReducerType::value_type; - - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - -#pragma omp declare reduction( \ - custom:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) - - ValueType* TeamVector_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - -#pragma omp barrier - Impl::OpenMPTargetReducerWrapper::init(TeamVector_scratch[0]); -#pragma omp barrier - -#pragma omp for simd reduction(custom : TeamVector_scratch[:1]) - for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { - lambda(i, TeamVector_scratch[0]); - } - - result.reference() = TeamVector_scratch[0]; -} -#else -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value> -parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< - iType, Impl::OpenMPTargetExecTeamMember>& loop_boundaries, - const Lambda& lambda, ReducerType const& result) { - using ValueType = typename ReducerType::value_type; - - // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of - // elements in the array <= 32. For reduction we allocate, 16 bytes per - // element in the scratch space, hence, 16*32 = 512. - static_assert(sizeof(ValueType) <= - Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); - - ValueType* TeamVector_scratch = - static_cast(loop_boundaries.team.impl_reduce_scratch()); - -#pragma omp declare reduction( \ - omp_red_teamthread_reducer:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp barrier - ValueType tmp; - result.init(tmp); - TeamVector_scratch[0] = tmp; -#pragma omp barrier - - iType team_size = iType(omp_get_num_threads()); -#pragma omp for simd reduction(omp_red_teamthread_reducer \ - : TeamVector_scratch[:1]) schedule(static, 1) - for (iType t = 0; t < team_size; t++) { - ValueType tmp2; - result.init(tmp2); - - for (iType i = loop_boundaries.start + t; i < loop_boundaries.end; - i += team_size) { - lambda(i, tmp2); - } - TeamVector_scratch[0] = tmp2; - } - - result.reference() = TeamVector_scratch[0]; -} -#endif // KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND -} // namespace Kokkos - -#ifdef KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND -#undef KOKKOS_IMPL_HIERARCHICAL_REDUCERS_WORKAROUND -#endif - -namespace Kokkos { - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::VectorSingleStruct& - /*single_struct*/, - const FunctorType& lambda) { - lambda(); -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::ThreadSingleStruct& - single_struct, - const FunctorType& lambda) { - if (single_struct.team_member.team_rank() == 0) lambda(); -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::VectorSingleStruct& - /*single_struct*/, - const FunctorType& lambda, ValueType& val) { - lambda(val); -} - -template -KOKKOS_INLINE_FUNCTION void single( - const Impl::ThreadSingleStruct& - single_struct, - const FunctorType& lambda, ValueType& val) { - if (single_struct.team_member.team_rank() == 0) { - lambda(val); - } - single_struct.team_member.team_broadcast(val, 0); -} -} // namespace Kokkos - -#endif /* #ifndef KOKKOS_OPENMPTARGETEXEC_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.hpp deleted file mode 100644 index 8e4baf8c05..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGET_INSTANCE_HPP -#define KOKKOS_OPENMPTARGET_INSTANCE_HPP - -#include - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -enum class openmp_fence_is_static { yes, no }; - -class OpenMPTargetInternal { - private: - OpenMPTargetInternal() = default; - OpenMPTargetInternal(const OpenMPTargetInternal&) = default; - OpenMPTargetInternal& operator=(const OpenMPTargetInternal&) = default; - - public: - void fence(openmp_fence_is_static is_static = openmp_fence_is_static::no); - void fence(const std::string& name, - openmp_fence_is_static is_static = openmp_fence_is_static::no); - - /** \brief Return the maximum amount of concurrency. */ - int concurrency(); - - //! Print configuration information to the given output stream. - void print_configuration(std::ostream& os, bool verbose) const; - - static const char* name(); - - //! Free any resources being consumed by the device. - void impl_finalize(); - - //! Has been initialized - int impl_is_initialized(); - uint32_t impl_get_instance_id() const noexcept; - //! Initialize, telling the CUDA run-time library which device to use. - void impl_initialize(); - - static OpenMPTargetInternal* impl_singleton(); - - private: - bool m_is_initialized = false; - uint32_t m_instance_id = Kokkos::Tools::Experimental::Impl::idForInstance< - Kokkos::Experimental::OpenMPTarget>(reinterpret_cast(this)); -}; -} // Namespace Impl -} // Namespace Experimental -} // Namespace Kokkos - -#endif // KOKKOS_OPENMPTARGET_INSTANCE_HPP diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp deleted file mode 100644 index dfb9ea70a1..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp +++ /dev/null @@ -1,1337 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGET_PARALLEL_HPP -#define KOKKOS_OPENMPTARGET_PARALLEL_HPP - -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Experimental::OpenMPTarget> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - const FunctorType m_functor; - const Policy m_policy; - - public: - void execute() const { execute_impl(); } - - template - void execute_impl() const { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - const auto begin = m_policy.begin(); - const auto end = m_policy.end(); - - if (end <= begin) return; - - FunctorType a_functor(m_functor); - -#pragma omp target teams distribute parallel for map(to : a_functor) - for (auto i = begin; i < end; ++i) { - if constexpr (std::is_void::value) { - a_functor(i); - } else { - a_functor(TagType(), i); - } - } - } - - ParallelFor(const FunctorType& arg_functor, Policy arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -// This class has the memcpy routine that is commonly used by ParallelReduce -// over RangePolicy and TeamPolicy. -template -struct ParallelReduceCommon { - // Copy the result back to device if the view is on the device. - static void memcpy_result(PointerType dest, PointerType src, size_t size, - bool ptr_on_device) { - if (ptr_on_device) { - OMPT_SAFE_CALL(omp_target_memcpy(dest, src, size, 0, 0, - omp_get_default_device(), - omp_get_initial_device())); - } else { - *dest = *src; - } - } -}; - -template -struct ParallelReduceSpecialize { - inline static void execute(const FunctorType& /*f*/, const PolicyType& /*p*/, - PointerType /*result_ptr*/) { - constexpr int FunctorHasJoin = - Impl::FunctorAnalysis::has_join_member_function; - constexpr int UseReducerType = is_reducer::value; - - std::stringstream error_message; - error_message << "Error: Invalid Specialization " << FunctorHasJoin << ' ' - << UseReducerType << '\n'; - // FIXME_OPENMPTARGET - OpenMPTarget_abort(error_message.str().c_str()); - } -}; - -template -struct ParallelReduceSpecialize, - ReducerType, PointerType, ValueType> { - using PolicyType = Kokkos::RangePolicy; - using TagType = typename PolicyType::work_tag; - using ReducerTypeFwd = - std::conditional_t::value, - FunctorType, ReducerType>; - using Analysis = Impl::FunctorAnalysis; - using ReferenceType = typename Analysis::reference_type; - - using ParReduceCommon = ParallelReduceCommon; - - static void execute_reducer(const FunctorType& f, const PolicyType& p, - PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - const auto begin = p.begin(); - const auto end = p.end(); - - ValueType result; - OpenMPTargetReducerWrapper::init(result); - - // Initialize and copy back the result even if it is a zero length - // reduction. - if (end <= begin) { - ParReduceCommon::memcpy_result(result_ptr, &result, sizeof(ValueType), - ptr_on_device); - return; - } - -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams distribute parallel for map(to \ - : f) reduction(custom \ - : result) - for (auto i = begin; i < end; ++i) { - if constexpr (std::is_void::value) { - f(i, result); - } else { - f(TagType(), i, result); - } - } - - ParReduceCommon::memcpy_result(result_ptr, &result, sizeof(ValueType), - ptr_on_device); - } - - template - static void execute_array(const FunctorType& f, const PolicyType& p, - PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - const auto begin = p.begin(); - const auto end = p.end(); - - // Enter the loop if the reduction is on a scalar type. - if constexpr (NumReductions == 1) { - ValueType result = ValueType(); - - // Initialize and copy back the result even if it is a zero length - // reduction. - if (end <= begin) { - ParReduceCommon::memcpy_result(result_ptr, &result, sizeof(ValueType), - ptr_on_device); - return; - } - // Case where reduction is on a native data type. - if constexpr (std::is_arithmetic::value) { -#pragma omp target teams distribute parallel for \ - map(to:f) reduction(+: result) - for (auto i = begin; i < end; ++i) - - if constexpr (std::is_void::value) { - f(i, result); - } else { - f(TagType(), i, result); - } - } else { -#pragma omp declare reduction(custom:ValueType : omp_out += omp_in) -#pragma omp target teams distribute parallel for map(to \ - : f) reduction(custom \ - : result) - for (auto i = begin; i < end; ++i) - - if constexpr (std::is_void::value) { - f(i, result); - } else { - f(TagType(), i, result); - } - } - - ParReduceCommon::memcpy_result(result_ptr, &result, sizeof(ValueType), - ptr_on_device); - } else { - ValueType result[NumReductions] = {}; - - // Initialize and copy back the result even if it is a zero length - // reduction. - if (end <= begin) { - ParReduceCommon::memcpy_result(result_ptr, result, - NumReductions * sizeof(ValueType), - ptr_on_device); - return; - } -#pragma omp target teams distribute parallel for map(to:f) reduction(+:result[:NumReductions]) - for (auto i = begin; i < end; ++i) { - if constexpr (std::is_void::value) { - f(i, result); - } else { - f(TagType(), i, result); - } - } - - ParReduceCommon::memcpy_result( - result_ptr, result, NumReductions * sizeof(ValueType), ptr_on_device); - } - } - - static void execute_init_join(const FunctorType& f, const PolicyType& p, - PointerType ptr, const bool ptr_on_device) { - const auto begin = p.begin(); - const auto end = p.end(); - - using FunctorAnalysis = - Impl::FunctorAnalysis; - constexpr int HasInit = FunctorAnalysis::has_init_member_function; - - // Initialize the result pointer. - - const auto size = end - begin; - - // FIXME_OPENMPTARGET: The team size and MAX_ACTIVE_THREADS are currently - // based on NVIDIA-V100 and should be modifid to be based on the - // architecture in the future. - const int max_team_threads = 32; - const int max_teams = - OpenMPTargetExec::MAX_ACTIVE_THREADS / max_team_threads; - // Number of elements in the reduction - const auto value_count = FunctorAnalysis::value_count(f); - - // Allocate scratch per active thread. Achieved by setting the first - // parameter of `resize_scratch=1`. - OpenMPTargetExec::resize_scratch(1, 0, value_count * sizeof(ValueType), - std::numeric_limits::max()); - ValueType* scratch_ptr = - static_cast(OpenMPTargetExec::get_scratch_ptr()); - -#pragma omp target map(to : f) is_device_ptr(scratch_ptr) - { - typename FunctorAnalysis::Reducer final_reducer(&f); - // Enter this loop if the functor has an `init` - if constexpr (HasInit) { - // The `init` routine needs to be called on the device since it might - // need device members. - final_reducer.init(scratch_ptr); - final_reducer.final(scratch_ptr); - } else { - for (int i = 0; i < value_count; ++i) { - static_cast(scratch_ptr)[i] = ValueType(); - } - - final_reducer.final(scratch_ptr); - } - } - - if (end <= begin) { - // If there is no work to be done, copy back the initialized values and - // exit. - if (!ptr_on_device) - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_initial_device(), omp_get_default_device())); - else - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_default_device(), omp_get_default_device())); - - return; - } - -#pragma omp target teams num_teams(max_teams) thread_limit(max_team_threads) \ - map(to \ - : f) is_device_ptr(scratch_ptr) - { - typename FunctorAnalysis::Reducer final_reducer(&f); -#pragma omp parallel - { - const int team_num = omp_get_team_num(); - const int num_teams = omp_get_num_teams(); - const auto chunk_size = size / num_teams; - const auto team_begin = begin + team_num * chunk_size; - const auto team_end = - (team_num == num_teams - 1) ? end : (team_begin + chunk_size); - ValueType* team_scratch = - scratch_ptr + team_num * max_team_threads * value_count; - ReferenceType result = final_reducer.init( - &team_scratch[omp_get_thread_num() * value_count]); - - // Accumulate partial results in thread specific storage. -#pragma omp for simd - for (auto i = team_begin; i < team_end; ++i) { - if constexpr (std::is_void::value) { - f(i, result); - } else { - f(TagType(), i, result); - } - } - - // Reduce all paritial results within a team. - const int team_size = max_team_threads; - int tree_neighbor_offset = 1; - do { -#pragma omp for simd - for (int i = 0; i < team_size - tree_neighbor_offset; - i += 2 * tree_neighbor_offset) { - const int neighbor = i + tree_neighbor_offset; - final_reducer.join(&team_scratch[i * value_count], - &team_scratch[neighbor * value_count]); - } - tree_neighbor_offset *= 2; - } while (tree_neighbor_offset < team_size); - } // end parallel - } // end target - - int tree_neighbor_offset = 1; - do { -#pragma omp target teams distribute parallel for simd map(to \ - : f) \ - is_device_ptr(scratch_ptr) - for (int i = 0; i < max_teams - tree_neighbor_offset; - i += 2 * tree_neighbor_offset) { - typename FunctorAnalysis::Reducer final_reducer(&f); - ValueType* team_scratch = scratch_ptr; - const int team_offset = max_team_threads * value_count; - final_reducer.join( - &team_scratch[i * team_offset], - &team_scratch[(i + tree_neighbor_offset) * team_offset]); - - // If `final` is provided by the functor. - // Do the final only once at the end. - if (tree_neighbor_offset * 2 >= max_teams && omp_get_team_num() == 0 && - omp_get_thread_num() == 0) { - final_reducer.final(scratch_ptr); - } - } - tree_neighbor_offset *= 2; - } while (tree_neighbor_offset < max_teams); - - // If the result view is on the host, copy back the values via memcpy. - if (!ptr_on_device) - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_initial_device(), omp_get_default_device())); - else - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_default_device(), omp_get_default_device())); - } -}; - -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::OpenMPTarget> { - private: - using Policy = Kokkos::RangePolicy; - - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - - using ReducerTypeFwd = - std::conditional_t::value, - FunctorType, ReducerType>; - using Analysis = Impl::FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - static constexpr int HasJoin = - Impl::FunctorAnalysis::has_join_member_function; - static constexpr int UseReducer = is_reducer::value; - static constexpr int IsArray = std::is_pointer::value; - - using ParReduceSpecialize = - ParallelReduceSpecialize; - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - bool m_result_ptr_on_device; - const int m_result_ptr_num_elems; - using TagType = typename Policy::work_tag; - - public: - void execute() const { - if constexpr (HasJoin) { - // Enter this loop if the Functor has a init-join. - ParReduceSpecialize::execute_init_join(m_functor, m_policy, m_result_ptr, - m_result_ptr_on_device); - } else if constexpr (UseReducer) { - // Enter this loop if the Functor is a reducer type. - ParReduceSpecialize::execute_reducer(m_functor, m_policy, m_result_ptr, - m_result_ptr_on_device); - } else if constexpr (IsArray) { - // Enter this loop if the reduction is on an array and the routine is - // templated over the size of the array. - if (m_result_ptr_num_elems <= 2) { - ParReduceSpecialize::template execute_array( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 4) { - ParReduceSpecialize::template execute_array( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 8) { - ParReduceSpecialize::template execute_array( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 16) { - ParReduceSpecialize::template execute_array( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 32) { - ParReduceSpecialize::template execute_array( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else { - Kokkos::abort("array reduction length must be <= 32"); - } - } else { - // This loop handles the basic scalar reduction. - ParReduceSpecialize::template execute_array( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } - } - - template - ParallelReduce(const FunctorType& arg_functor, Policy& arg_policy, - const ViewType& arg_result_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result_view.data()), - m_result_ptr_on_device( - MemorySpaceAccess::accessible), - m_result_ptr_num_elems(arg_result_view.size()) {} - - ParallelReduce(const FunctorType& arg_functor, Policy& arg_policy, - const ReducerType& reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_on_device( - MemorySpaceAccess::accessible), - m_result_ptr_num_elems(reducer.view().size()) {} -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelScan, - Kokkos::Experimental::OpenMPTarget> { - protected: - using Policy = Kokkos::RangePolicy; - - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - using idx_type = typename Policy::index_type; - - using Analysis = Impl::FunctorAnalysis; - - using value_type = typename Analysis::value_type; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - const FunctorType m_functor; - const Policy m_policy; - - template - std::enable_if_t::value> call_with_tag( - const FunctorType& f, const idx_type& idx, value_type& val, - const bool& is_final) const { - f(idx, val, is_final); - } - template - std::enable_if_t::value> call_with_tag( - const FunctorType& f, const idx_type& idx, value_type& val, - const bool& is_final) const { - f(WorkTag(), idx, val, is_final); - } - - public: - void impl_execute( - Kokkos::View - element_values, - Kokkos::View - chunk_values, - Kokkos::View count) - const { - const idx_type N = m_policy.end() - m_policy.begin(); - const idx_type chunk_size = 128; - const idx_type n_chunks = (N + chunk_size - 1) / chunk_size; - idx_type nteams = n_chunks > 512 ? 512 : n_chunks; - idx_type team_size = 128; - - FunctorType a_functor(m_functor); -#pragma omp target teams distribute map(to \ - : a_functor) num_teams(nteams) \ - thread_limit(team_size) - for (idx_type team_id = 0; team_id < n_chunks; ++team_id) { - typename Analysis::Reducer final_reducer(&a_functor); -#pragma omp parallel num_threads(team_size) - { - const idx_type local_offset = team_id * chunk_size; - -#pragma omp for - for (idx_type i = 0; i < chunk_size; ++i) { - const idx_type idx = local_offset + i; - value_type val; - final_reducer.init(&val); - if (idx < N) call_with_tag(a_functor, idx, val, false); - element_values(team_id, i) = val; - } -#pragma omp barrier - if (omp_get_thread_num() == 0) { - value_type sum; - final_reducer.init(&sum); - for (idx_type i = 0; i < chunk_size; ++i) { - final_reducer.join(&sum, &element_values(team_id, i)); - element_values(team_id, i) = sum; - } - chunk_values(team_id) = sum; - } -#pragma omp barrier - if (omp_get_thread_num() == 0) { - if (Kokkos::atomic_fetch_add(&count(), 1) == n_chunks - 1) { - value_type sum; - final_reducer.init(&sum); - for (idx_type i = 0; i < n_chunks; ++i) { - final_reducer.join(&sum, &chunk_values(i)); - chunk_values(i) = sum; - } - } - } - } - } - -#pragma omp target teams distribute map(to \ - : a_functor) num_teams(nteams) \ - thread_limit(team_size) - for (idx_type team_id = 0; team_id < n_chunks; ++team_id) { - typename Analysis::Reducer final_reducer(&a_functor); -#pragma omp parallel num_threads(team_size) - { - const idx_type local_offset = team_id * chunk_size; - value_type offset_value; - if (team_id > 0) - offset_value = chunk_values(team_id - 1); - else - final_reducer.init(&offset_value); - -#pragma omp for - for (idx_type i = 0; i < chunk_size; ++i) { - const idx_type idx = local_offset + i; - value_type local_offset_value; - if (i > 0) { - local_offset_value = element_values(team_id, i - 1); - // FIXME_OPENMPTARGET We seem to access memory illegaly on AMD GPUs -#ifdef KOKKOS_ARCH_VEGA - if constexpr (Analysis::has_join_member_function) { - if constexpr (std::is_void_v) - a_functor.join(local_offset_value, offset_value); - else - a_functor.join(WorkTag{}, local_offset_value, offset_value); - } else - local_offset_value += offset_value; -#else - final_reducer.join(&local_offset_value, &offset_value); -#endif - } else - local_offset_value = offset_value; - if (idx < N) - call_with_tag(a_functor, idx, local_offset_value, true); - } - } - } - } - - void execute() const { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - const idx_type N = m_policy.end() - m_policy.begin(); - const idx_type chunk_size = 128; - const idx_type n_chunks = (N + chunk_size - 1) / chunk_size; - - // This could be scratch memory per team - Kokkos::View - element_values("element_values", n_chunks, chunk_size); - Kokkos::View - chunk_values("chunk_values", n_chunks); - Kokkos::View count( - "Count"); - - impl_execute(element_values, chunk_values, count); - } - - //---------------------------------------- - - ParallelScan(const FunctorType& arg_functor, const Policy& arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} - - //---------------------------------------- -}; - -template -class ParallelScanWithTotal, - ReturnType, Kokkos::Experimental::OpenMPTarget> - : public ParallelScan, - Kokkos::Experimental::OpenMPTarget> { - using base_t = ParallelScan, - Kokkos::Experimental::OpenMPTarget>; - using value_type = typename base_t::value_type; - value_type& m_returnvalue; - - public: - void execute() const { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - const int64_t N = base_t::m_policy.end() - base_t::m_policy.begin(); - const int chunk_size = 128; - const int64_t n_chunks = (N + chunk_size - 1) / chunk_size; - - if (N > 0) { - // This could be scratch memory per team - Kokkos::View - element_values("element_values", n_chunks, chunk_size); - Kokkos::View - chunk_values("chunk_values", n_chunks); - Kokkos::View count( - "Count"); - - base_t::impl_execute(element_values, chunk_values, count); - - const int size = base_t::Analysis::value_size(base_t::m_functor); - DeepCopy( - &m_returnvalue, chunk_values.data() + (n_chunks - 1), size); - } else { - m_returnvalue = 0; - } - } - - ParallelScanWithTotal(const FunctorType& arg_functor, - const typename base_t::Policy& arg_policy, - ReturnType& arg_returnvalue) - : base_t(arg_functor, arg_policy), m_returnvalue(arg_returnvalue) {} -}; -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Experimental::OpenMPTarget> { - private: - using Policy = - Kokkos::Impl::TeamPolicyInternal; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - - const FunctorType m_functor; - const Policy m_policy; - const size_t m_shmem_size; - - public: - void execute() const { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - execute_impl(); - } - - private: - template - void execute_impl() const { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - const auto league_size = m_policy.league_size(); - const auto team_size = m_policy.team_size(); - const auto vector_length = m_policy.impl_vector_length(); - - const size_t shmem_size_L0 = m_policy.scratch_size(0, team_size); - const size_t shmem_size_L1 = m_policy.scratch_size(1, team_size); - OpenMPTargetExec::resize_scratch(team_size, shmem_size_L0, shmem_size_L1, - league_size); - - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); - FunctorType a_functor(m_functor); - - // FIXME_OPENMPTARGET - If the team_size is not a multiple of 32, the - // scratch implementation does not work in the Release or RelWithDebugInfo - // mode but works in the Debug mode. - - // Maximum active teams possible. - int max_active_teams = OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size; - // nteams should not exceed the maximum in-flight teams possible. - const auto nteams = - league_size < max_active_teams ? league_size : max_active_teams; - - // If the league size is <=0, do not launch the kernel. - if (nteams <= 0) return; - -// Performing our own scheduling of teams to avoid separation of code between -// teams-distribute and parallel. Gave a 2x performance boost in test cases with -// the clang compiler. atomic_compare_exchange can be avoided since the standard -// guarantees that the number of teams specified in the `num_teams` clause is -// always less than or equal to the maximum concurrently running teams. -#pragma omp target teams num_teams(nteams) thread_limit(team_size) \ - map(to \ - : a_functor) is_device_ptr(scratch_ptr) -#pragma omp parallel - { - const int blockIdx = omp_get_team_num(); - const int gridDim = omp_get_num_teams(); - - // Iterate through the number of teams until league_size and assign the - // league_id accordingly - // Guarantee that the compilers respect the `num_teams` clause - if (gridDim <= nteams) { - for (int league_id = blockIdx; league_id < league_size; - league_id += gridDim) { - typename Policy::member_type team( - league_id, league_size, team_size, vector_length, scratch_ptr, - blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void::value) - m_functor(team); - else - m_functor(TagType(), team); - } - } else - Kokkos::abort("`num_teams` clause was not respected.\n"); - } - } - - public: - ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_functor(arg_functor), - m_policy(arg_policy), - m_shmem_size(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) {} -}; - -template -struct ParallelReduceSpecialize, - ReducerType, PointerType, ValueType> { - using PolicyType = TeamPolicyInternal; - using TagType = typename PolicyType::work_tag; - using ReducerTypeFwd = - std::conditional_t::value, - FunctorType, ReducerType>; - using Analysis = Impl::FunctorAnalysis; - - using ReferenceType = typename Analysis::reference_type; - - using ParReduceCommon = ParallelReduceCommon; - - static void execute_reducer(const FunctorType& f, const PolicyType& p, - PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - - const int league_size = p.league_size(); - const int team_size = p.team_size(); - const int vector_length = p.impl_vector_length(); - - const size_t shmem_size_L0 = p.scratch_size(0, team_size); - const size_t shmem_size_L1 = p.scratch_size(1, team_size); - OpenMPTargetExec::resize_scratch(PolicyType::member_type::TEAM_REDUCE_SIZE, - shmem_size_L0, shmem_size_L1, league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); - - ValueType result = ValueType(); - - // Maximum active teams possible. - int max_active_teams = OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size; - const auto nteams = - league_size < max_active_teams ? league_size : max_active_teams; - - // If the league size is <=0, do not launch the kernel. - if (nteams <= 0) return; - -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams num_teams(nteams) thread_limit(team_size) map(to \ - : f) \ - is_device_ptr(scratch_ptr) reduction(custom \ - : result) -#pragma omp parallel reduction(custom : result) - { - const int blockIdx = omp_get_team_num(); - const int gridDim = omp_get_num_teams(); - - // Guarantee that the compilers respect the `num_teams` clause - if (gridDim <= nteams) { - for (int league_id = blockIdx; league_id < league_size; - league_id += gridDim) { - typename PolicyType::member_type team( - league_id, league_size, team_size, vector_length, scratch_ptr, - blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void::value) - f(team, result); - else - f(TagType(), team, result); - } - } else - Kokkos::abort("`num_teams` clause was not respected.\n"); - } - - // Copy results back to device if `parallel_reduce` is on a device view. - ParReduceCommon::memcpy_result(result_ptr, &result, sizeof(ValueType), - ptr_on_device); - } - - template - static void execute_array(const FunctorType& f, const PolicyType& p, - PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - - const int league_size = p.league_size(); - const int team_size = p.team_size(); - const int vector_length = p.impl_vector_length(); - - const size_t shmem_size_L0 = p.scratch_size(0, team_size); - const size_t shmem_size_L1 = p.scratch_size(1, team_size); - OpenMPTargetExec::resize_scratch(PolicyType::member_type::TEAM_REDUCE_SIZE, - shmem_size_L0, shmem_size_L1, league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); - - // Maximum active teams possible. - int max_active_teams = OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size; - const auto nteams = - league_size < max_active_teams ? league_size : max_active_teams; - - // If the league size is <=0, do not launch the kernel. - if (nteams <= 0) return; - - // Case where the number of reduction items is 1. - if constexpr (NumReductions == 1) { - ValueType result = ValueType(); - - // Case where reduction is on a native data type. - if constexpr (std::is_arithmetic::value) { -#pragma omp target teams num_teams(nteams) thread_limit(team_size) map(to \ - : f) \ - is_device_ptr(scratch_ptr) reduction(+: result) -#pragma omp parallel reduction(+ : result) - { - const int blockIdx = omp_get_team_num(); - const int gridDim = omp_get_num_teams(); - - // Guarantee that the compilers respect the `num_teams` clause - if (gridDim <= nteams) { - for (int league_id = blockIdx; league_id < league_size; - league_id += gridDim) { - typename PolicyType::member_type team( - league_id, league_size, team_size, vector_length, scratch_ptr, - blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void::value) - f(team, result); - else - f(TagType(), team, result); - } - } else - Kokkos::abort("`num_teams` clause was not respected.\n"); - } - } else { - // Case where the reduction is on a non-native data type. -#pragma omp declare reduction(custom:ValueType : omp_out += omp_in) -#pragma omp target teams num_teams(nteams) thread_limit(team_size) map(to \ - : f) \ - is_device_ptr(scratch_ptr) reduction(custom \ - : result) -#pragma omp parallel reduction(custom : result) - { - const int blockIdx = omp_get_team_num(); - const int gridDim = omp_get_num_teams(); - - // Guarantee that the compilers respect the `num_teams` clause - if (gridDim <= nteams) { - for (int league_id = blockIdx; league_id < league_size; - league_id += gridDim) { - typename PolicyType::member_type team( - league_id, league_size, team_size, vector_length, scratch_ptr, - blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void::value) - f(team, result); - else - f(TagType(), team, result); - } - } else - Kokkos::abort("`num_teams` clause was not respected.\n"); - } - } - - // Copy results back to device if `parallel_reduce` is on a device view. - ParReduceCommon::memcpy_result(result_ptr, &result, sizeof(ValueType), - ptr_on_device); - } else { - ValueType result[NumReductions] = {}; - // Case where the reduction is on an array. -#pragma omp target teams num_teams(nteams) thread_limit(team_size) map(to \ - : f) \ - is_device_ptr(scratch_ptr) reduction(+ : result[:NumReductions]) -#pragma omp parallel reduction(+ : result[:NumReductions]) - { - const int blockIdx = omp_get_team_num(); - const int gridDim = omp_get_num_teams(); - - // Guarantee that the compilers respect the `num_teams` clause - if (gridDim <= nteams) { - for (int league_id = blockIdx; league_id < league_size; - league_id += gridDim) { - typename PolicyType::member_type team( - league_id, league_size, team_size, vector_length, scratch_ptr, - blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void::value) - f(team, result); - else - f(TagType(), team, result); - } - } else - Kokkos::abort("`num_teams` clause was not respected.\n"); - } - - // Copy results back to device if `parallel_reduce` is on a device view. - ParReduceCommon::memcpy_result( - result_ptr, result, NumReductions * sizeof(ValueType), ptr_on_device); - } - } - - // FIXME_OPENMPTARGET : This routine is a copy from `parallel_reduce` over - // RangePolicy. Need a new implementation. - static void execute_init_join(const FunctorType& f, const PolicyType& p, - PointerType ptr, const bool ptr_on_device) { - using FunctorAnalysis = - Impl::FunctorAnalysis; - constexpr int HasInit = FunctorAnalysis::has_init_member_function; - - const int league_size = p.league_size(); - const int team_size = p.team_size(); - const int vector_length = p.impl_vector_length(); - - auto begin = 0; - auto end = league_size * team_size + team_size * vector_length; - - const size_t shmem_size_L0 = p.scratch_size(0, team_size); - const size_t shmem_size_L1 = p.scratch_size(1, team_size); - - // FIXME_OPENMPTARGET: This would oversubscribe scratch memory since we are - // already using the available scratch memory to create temporaries for each - // thread. - if ((shmem_size_L0 + shmem_size_L1) > 0) { - Kokkos::abort( - "OpenMPTarget: Scratch memory is not supported in `parallel_reduce` " - "over functors with init/join."); - } - - const auto nteams = league_size; - - // Number of elements in the reduction - const auto value_count = FunctorAnalysis::value_count(f); - - // Allocate scratch per active thread. - OpenMPTargetExec::resize_scratch(1, 0, value_count * sizeof(ValueType), - league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); - - // Enter this loop if the functor has an `init` - if constexpr (HasInit) { - // The `init` routine needs to be called on the device since it might need - // device members. -#pragma omp target map(to : f) is_device_ptr(scratch_ptr) - { - typename FunctorAnalysis::Reducer final_reducer(&f); - final_reducer.init(scratch_ptr); - final_reducer.final(scratch_ptr); - } - } else { -#pragma omp target map(to : f) is_device_ptr(scratch_ptr) - { - for (int i = 0; i < value_count; ++i) { - static_cast(scratch_ptr)[i] = ValueType(); - } - - typename FunctorAnalysis::Reducer final_reducer(&f); - final_reducer.final(static_cast(scratch_ptr)); - } - } - - if (end <= begin) { - // If there is no work to be done, copy back the initialized values and - // exit. - if (!ptr_on_device) - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_initial_device(), omp_get_default_device())); - else - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_default_device(), omp_get_default_device())); - - return; - } - -#pragma omp target teams num_teams(nteams) thread_limit(team_size) map(to \ - : f) \ - is_device_ptr(scratch_ptr) - { -#pragma omp parallel - { - const int team_num = omp_get_team_num(); - const int num_teams = omp_get_num_teams(); - ValueType* team_scratch = static_cast(scratch_ptr) + - team_num * team_size * value_count; - typename FunctorAnalysis::Reducer final_reducer(&f); - ReferenceType result = final_reducer.init(&team_scratch[0]); - - for (int league_id = team_num; league_id < league_size; - league_id += num_teams) { - typename PolicyType::member_type team( - league_id, league_size, team_size, vector_length, scratch_ptr, - team_num, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void::value) { - f(team, result); - } else { - f(TagType(), team, result); - } - } - } // end parallel - } // end target - - int tree_neighbor_offset = 1; - do { -#pragma omp target teams distribute parallel for simd map(to \ - : f) \ - is_device_ptr(scratch_ptr) - for (int i = 0; i < nteams - tree_neighbor_offset; - i += 2 * tree_neighbor_offset) { - ValueType* team_scratch = static_cast(scratch_ptr); - const int team_offset = team_size * value_count; - typename FunctorAnalysis::Reducer final_reducer(&f); - final_reducer.join( - &team_scratch[i * team_offset], - &team_scratch[(i + tree_neighbor_offset) * team_offset]); - - // If `final` is provided by the functor. - // Do the final only once at the end. - if (tree_neighbor_offset * 2 >= nteams && omp_get_team_num() == 0 && - omp_get_thread_num() == 0) { - final_reducer.final(scratch_ptr); - } - } - tree_neighbor_offset *= 2; - } while (tree_neighbor_offset < nteams); - - // If the result view is on the host, copy back the values via memcpy. - if (!ptr_on_device) - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_initial_device(), omp_get_default_device())); - else - OMPT_SAFE_CALL(omp_target_memcpy( - ptr, scratch_ptr, value_count * sizeof(ValueType), 0, 0, - omp_get_default_device(), omp_get_default_device())); - } -}; - -template -class ParallelReduce, - ReducerType, Kokkos::Experimental::OpenMPTarget> { - private: - using Policy = - Kokkos::Impl::TeamPolicyInternal; - - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using ReducerTypeFwd = - std::conditional_t::value, - FunctorType, ReducerType>; - using WorkTagFwd = - std::conditional_t::value, WorkTag, - void>; - using Analysis = Impl::FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - using value_type = typename Analysis::value_type; - - bool m_result_ptr_on_device; - const int m_result_ptr_num_elems; - - static constexpr int HasJoin = - Impl::FunctorAnalysis::has_join_member_function; - static constexpr int UseReducer = is_reducer::value; - static constexpr int IsArray = std::is_pointer::value; - - using ParReduceSpecialize = - ParallelReduceSpecialize; - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const size_t m_shmem_size; - - public: - void execute() const { - if constexpr (HasJoin) { - ParReduceSpecialize::execute_init_join(m_functor, m_policy, m_result_ptr, - m_result_ptr_on_device); - } else if constexpr (UseReducer) { - ParReduceSpecialize::execute_reducer(m_functor, m_policy, m_result_ptr, - m_result_ptr_on_device); - } else if constexpr (IsArray) { - if (m_result_ptr_num_elems <= 2) { - ParReduceSpecialize::template execute_array<2>( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 4) { - ParReduceSpecialize::template execute_array<4>( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 8) { - ParReduceSpecialize::template execute_array<8>( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 16) { - ParReduceSpecialize::template execute_array<16>( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else if (m_result_ptr_num_elems <= 32) { - ParReduceSpecialize::template execute_array<32>( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } else { - Kokkos::abort("array reduction length must be <= 32"); - } - } else { - ParReduceSpecialize::template execute_array<1>( - m_functor, m_policy, m_result_ptr, m_result_ptr_on_device); - } - } - - template - ParallelReduce(const FunctorType& arg_functor, const Policy& arg_policy, - const ViewType& arg_result, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = nullptr) - : m_result_ptr_on_device( - MemorySpaceAccess::accessible), - m_result_ptr_num_elems(arg_result.size()), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_shmem_size(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) {} - - ParallelReduce(const FunctorType& arg_functor, Policy& arg_policy, - const ReducerType& reducer) - : m_result_ptr_on_device( - MemorySpaceAccess::accessible), - m_result_ptr_num_elems(reducer.view().size()), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_shmem_size(arg_policy.scratch_size(0) + arg_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, arg_policy.team_size())) {} -}; - -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -template -struct TeamThreadRangeBoundariesStruct { - using index_type = iType; - const iType start; - const iType end; - const OpenMPTargetExecTeamMember& team; - - TeamThreadRangeBoundariesStruct(const OpenMPTargetExecTeamMember& thread_, - iType count) - : start(0), end(count), team(thread_) {} - TeamThreadRangeBoundariesStruct(const OpenMPTargetExecTeamMember& thread_, - iType begin_, iType end_) - : start(begin_), end(end_), team(thread_) {} -}; - -template -struct ThreadVectorRangeBoundariesStruct { - using index_type = iType; - const index_type start; - const index_type end; - const OpenMPTargetExecTeamMember& team; - - ThreadVectorRangeBoundariesStruct(const OpenMPTargetExecTeamMember& thread_, - index_type count) - : start(0), end(count), team(thread_) {} - ThreadVectorRangeBoundariesStruct(const OpenMPTargetExecTeamMember& thread_, - index_type begin_, index_type end_) - : start(begin_), end(end_), team(thread_) {} -}; - -template -struct TeamVectorRangeBoundariesStruct { - using index_type = iType; - const index_type start; - const index_type end; - const OpenMPTargetExecTeamMember& team; - - TeamVectorRangeBoundariesStruct(const OpenMPTargetExecTeamMember& thread_, - index_type count) - : start(0), end(count), team(thread_) {} - TeamVectorRangeBoundariesStruct(const OpenMPTargetExecTeamMember& thread_, - index_type begin_, index_type end_) - : start(begin_), end(end_), team(thread_) {} -}; - -} // namespace Impl - -} // namespace Kokkos -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* KOKKOS_OPENMPTARGET_PARALLEL_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel_MDRange.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel_MDRange.hpp deleted file mode 100644 index 2399b424f5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel_MDRange.hpp +++ /dev/null @@ -1,836 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGET_PARALLEL_MDRANGE_HPP -#define KOKKOS_OPENMPTARGET_PARALLEL_MDRANGE_HPP - -#include -#include -#include - -// WORKAROUND OPENMPTARGET: sometimes tile sizes don't make it correctly, -// this was tracked down to a bug in clang with regards of mapping structs -// with arrays of long in it. Arrays of int might be fine though ... -#define KOKKOS_IMPL_MDRANGE_USE_NO_TILES // undef EOF - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Experimental::OpenMPTarget> { - private: - using Policy = Kokkos::MDRangePolicy; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using Index = typename Policy::index_type; - - const FunctorType m_functor; - const Policy m_policy; - - public: - inline void execute() const { - OpenMPTargetExec::verify_is_process( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( - "Kokkos::Experimental::OpenMPTarget parallel_for"); - FunctorType functor(m_functor); - Policy policy = m_policy; - -#ifdef KOKKOS_IMPL_MDRANGE_USE_NO_TILES - typename Policy::point_type unused; - - execute_tile(unused, functor, policy); -#else - const int64_t begin = 0; - const int64_t end = m_policy.m_num_tiles; - -#pragma omp target teams distribute map(to : functor) num_teams(end - begin) - { - for (ptrdiff_t tile_idx = begin; tile_idx < end; ++tile_idx) { - -#pragma omp parallel - { - typename Policy::point_type offset; - if (Policy::outer_direction == Policy::Left) { - for (int i = 0; i < Policy::rank; ++i) { - offset[i] = (tile_idx % policy.m_tile_end[i]) * policy.m_tile[i] + - policy.m_lower[i]; - tile_idx /= policy.m_tile_end[i]; - } - } else { - for (int i = Policy::rank - 1; i >= 0; --i) { - offset[i] = (tile_idx % policy.m_tile_end[i]) * policy.m_tile[i] + - policy.m_lower[i]; - tile_idx /= policy.m_tile_end[i]; - } - } - execute_tile(offset, functor, policy); - } - } - } -#endif - } - - template - inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy) const { -#ifdef KOKKOS_IMPL_MDRANGE_USE_NO_TILES - (void)offset; - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - -#pragma omp target teams distribute parallel for collapse(2) map(to : functor) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1); - else - functor(typename Policy::work_tag(), i0, i1); - } - } -#else - const ptrdiff_t begin_0 = offset[0]; - ptrdiff_t end_0 = begin_0 + policy.m_tile[0]; - end_0 = end_0 < policy.m_upper[0] ? end_0 : policy.m_upper[0]; - - const ptrdiff_t begin_1 = offset[1]; - ptrdiff_t end_1 = begin_1 + policy.m_tile[1]; - end_1 = end_1 < policy.m_upper[1] ? end_1 : policy.m_upper[1]; - -#pragma omp for collapse(2) - for (ptrdiff_t i0 = begin_0; i0 < end_0; ++i0) - for (ptrdiff_t i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1); - else - functor(typename Policy::work_tag(), i0, i1); - } -#endif - } - - template - inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy) const { -#ifdef KOKKOS_IMPL_MDRANGE_USE_NO_TILES - (void)offset; - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - -#pragma omp target teams distribute parallel for collapse(3) map(to : functor) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2); - else - functor(typename Policy::work_tag(), i0, i1, i2); - } - } - } -#else - const ptrdiff_t begin_0 = offset[0]; - ptrdiff_t end_0 = begin_0 + policy.m_tile[0]; - end_0 = end_0 < policy.m_upper[0] ? end_0 : policy.m_upper[0]; - - const ptrdiff_t begin_1 = offset[1]; - ptrdiff_t end_1 = begin_1 + policy.m_tile[1]; - end_1 = end_1 < policy.m_upper[1] ? end_1 : policy.m_upper[1]; - - const ptrdiff_t begin_2 = offset[2]; - ptrdiff_t end_2 = begin_2 + policy.m_tile[2]; - end_2 = end_2 < policy.m_upper[2] ? end_2 : policy.m_upper[2]; - -#pragma omp for collapse(3) - for (ptrdiff_t i0 = begin_0; i0 < end_0; ++i0) - for (ptrdiff_t i1 = begin_1; i1 < end_1; ++i1) - for (ptrdiff_t i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2); - else - functor(typename Policy::work_tag(), i0, i1, i2); - } -#endif - } - - template - inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy) const { -#ifdef KOKKOS_IMPL_MDRANGE_USE_NO_TILES - (void)offset; - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - const Index begin_3 = policy.m_lower[3]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - const Index end_3 = policy.m_upper[3]; - -#pragma omp target teams distribute parallel for collapse(4) map(to : functor) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, i3); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3); - } - } - } - } -#else - const ptrdiff_t begin_0 = offset[0]; - ptrdiff_t end_0 = begin_0 + policy.m_tile[0]; - end_0 = end_0 < policy.m_upper[0] ? end_0 : policy.m_upper[0]; - - const ptrdiff_t begin_1 = offset[1]; - ptrdiff_t end_1 = begin_1 + policy.m_tile[1]; - end_1 = end_1 < policy.m_upper[1] ? end_1 : policy.m_upper[1]; - - const ptrdiff_t begin_2 = offset[2]; - ptrdiff_t end_2 = begin_2 + policy.m_tile[2]; - end_2 = end_2 < policy.m_upper[2] ? end_2 : policy.m_upper[2]; - - const ptrdiff_t begin_3 = offset[3]; - ptrdiff_t end_3 = begin_3 + policy.m_tile[3]; - end_3 = end_3 < policy.m_upper[3] ? end_3 : policy.m_upper[3]; - -#pragma omp for collapse(4) - for (ptrdiff_t i0 = begin_0; i0 < end_0; ++i0) - for (ptrdiff_t i1 = begin_1; i1 < end_1; ++i1) - for (ptrdiff_t i2 = begin_2; i2 < end_2; ++i2) - for (ptrdiff_t i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, i3); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3); - } -#endif - } - - template - inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy) const { -#ifdef KOKKOS_IMPL_MDRANGE_USE_NO_TILES - (void)offset; - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - const Index begin_3 = policy.m_lower[3]; - const Index begin_4 = policy.m_lower[4]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - const Index end_3 = policy.m_upper[3]; - const Index end_4 = policy.m_upper[4]; - -#pragma omp target teams distribute parallel for collapse(5) map(to : functor) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - for (auto i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4); - } - } - } - } - } -#else - const ptrdiff_t begin_0 = offset[0]; - ptrdiff_t end_0 = begin_0 + policy.m_tile[0]; - end_0 = end_0 < policy.m_upper[0] ? end_0 : policy.m_upper[0]; - - const ptrdiff_t begin_1 = offset[1]; - ptrdiff_t end_1 = begin_1 + policy.m_tile[1]; - end_1 = end_1 < policy.m_upper[1] ? end_1 : policy.m_upper[1]; - - const ptrdiff_t begin_2 = offset[2]; - ptrdiff_t end_2 = begin_2 + policy.m_tile[2]; - end_2 = end_2 < policy.m_upper[2] ? end_2 : policy.m_upper[2]; - - const ptrdiff_t begin_3 = offset[3]; - ptrdiff_t end_3 = begin_3 + policy.m_tile[3]; - end_3 = end_3 < policy.m_upper[3] ? end_3 : policy.m_upper[3]; - - const ptrdiff_t begin_4 = offset[4]; - ptrdiff_t end_4 = begin_4 + policy.m_tile[4]; - end_4 = end_4 < policy.m_upper[4] ? end_4 : policy.m_upper[4]; - -#pragma omp for collapse(5) - for (ptrdiff_t i0 = begin_0; i0 < end_0; ++i0) - for (ptrdiff_t i1 = begin_1; i1 < end_1; ++i1) - for (ptrdiff_t i2 = begin_2; i2 < end_2; ++i2) - for (ptrdiff_t i3 = begin_3; i3 < end_3; ++i3) - for (ptrdiff_t i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4); - } -#endif - } - - template - inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy) const { -#ifdef KOKKOS_IMPL_MDRANGE_USE_NO_TILES - (void)offset; - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - const Index begin_3 = policy.m_lower[3]; - const Index begin_4 = policy.m_lower[4]; - const Index begin_5 = policy.m_lower[5]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - const Index end_3 = policy.m_upper[3]; - const Index end_4 = policy.m_upper[4]; - const Index end_5 = policy.m_upper[5]; - -#pragma omp target teams distribute parallel for collapse(6) map(to : functor) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - for (auto i4 = begin_4; i4 < end_4; ++i4) { - for (auto i5 = begin_5; i5 < end_5; ++i5) { - { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - i5); - } - } - } - } - } - } - } -#else - const ptrdiff_t begin_0 = offset[0]; - ptrdiff_t end_0 = begin_0 + policy.m_tile[0]; - end_0 = end_0 < policy.m_upper[0] ? end_0 : policy.m_upper[0]; - - const ptrdiff_t begin_1 = offset[1]; - ptrdiff_t end_1 = begin_1 + policy.m_tile[1]; - end_1 = end_1 < policy.m_upper[1] ? end_1 : policy.m_upper[1]; - - const ptrdiff_t begin_2 = offset[2]; - ptrdiff_t end_2 = begin_2 + policy.m_tile[2]; - end_2 = end_2 < policy.m_upper[2] ? end_2 : policy.m_upper[2]; - - const ptrdiff_t begin_3 = offset[3]; - ptrdiff_t end_3 = begin_3 + policy.m_tile[3]; - end_3 = end_3 < policy.m_upper[3] ? end_3 : policy.m_upper[3]; - - const ptrdiff_t begin_4 = offset[4]; - ptrdiff_t end_4 = begin_4 + policy.m_tile[4]; - end_4 = end_4 < policy.m_upper[4] ? end_4 : policy.m_upper[4]; - - const ptrdiff_t begin_5 = offset[5]; - ptrdiff_t end_5 = begin_5 + policy.m_tile[5]; - end_5 = end_5 < policy.m_upper[5] ? end_5 : policy.m_upper[5]; - -#pragma omp for collapse(6) - for (ptrdiff_t i0 = begin_0; i0 < end_0; ++i0) - for (ptrdiff_t i1 = begin_1; i1 < end_1; ++i1) - for (ptrdiff_t i2 = begin_2; i2 < end_2; ++i2) - for (ptrdiff_t i3 = begin_3; i3 < end_3; ++i3) - for (ptrdiff_t i4 = begin_4; i4 < end_4; ++i4) - for (ptrdiff_t i5 = begin_5; i5 < end_5; ++i5) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5); - } -#endif - } - - inline ParallelFor(const FunctorType& arg_functor, Policy arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} - // TODO DZP: based on a conversation with Christian, we're using 256 as a - // heuristic here. We need something better once we can query these kinds of - // properties - template - static int max_tile_size_product(const Policy&, const Functor&) { - return 256; - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::OpenMPTarget> { - private: - using Policy = Kokkos::MDRangePolicy; - - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using Index = typename Policy::index_type; - - using ReducerConditional = - std::conditional::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = Impl::FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - enum { - HasJoin = - Impl::FunctorAnalysis::has_join_member_function - }; - enum { UseReducer = is_reducer::value }; - - const pointer_type m_result_ptr; - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - - using ParReduceCommon = ParallelReduceCommon; - - bool m_result_ptr_on_device; - - public: - inline void execute() const { - execute_tile( - m_functor, m_policy, m_result_ptr); - } - - template - inline ParallelReduce( - const FunctorType& arg_functor, Policy arg_policy, - const ViewType& arg_result_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = NULL) - : m_result_ptr(arg_result_view.data()), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr_on_device( - MemorySpaceAccess::accessible) {} - - inline ParallelReduce(const FunctorType& arg_functor, Policy arg_policy, - const ReducerType& reducer) - : m_result_ptr(reducer.view().data()), - m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr_on_device( - MemorySpaceAccess::accessible) {} - - template - inline std::enable_if_t execute_tile(const FunctorType& functor, - const Policy& policy, - pointer_type ptr) const { - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - - ValueType result = ValueType(); - - // FIXME_OPENMPTARGET: Unable to separate directives and their companion - // loops which leads to code duplication for different reduction types. - if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams distribute parallel for collapse(2) map(to \ - : functor) \ - reduction(custom \ - : result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1, result); - else - functor(typename Policy::work_tag(), i0, i1, result); - } - } - } else { -#pragma omp target teams distribute parallel for collapse(2) map(to : functor) \ -reduction(+:result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1, result); - else - functor(typename Policy::work_tag(), i0, i1, result); - } - } - } - - ParReduceCommon::memcpy_result(ptr, &result, sizeof(ValueType), - m_result_ptr_on_device); - } - - template - inline std::enable_if_t execute_tile(const FunctorType& functor, - const Policy& policy, - pointer_type ptr) const { - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - - ValueType result = ValueType(); - - // FIXME_OPENMPTARGET: Unable to separate directives and their companion - // loops which leads to code duplication for different reduction types. - if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams distribute parallel for collapse(3) map(to \ - : functor) \ - reduction(custom \ - : result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, result); - } - } - } - } else { -#pragma omp target teams distribute parallel for collapse(3) map(to : functor) \ -reduction(+:result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, result); - } - } - } - } - - ParReduceCommon::memcpy_result(ptr, &result, sizeof(ValueType), - m_result_ptr_on_device); - } - - template - inline std::enable_if_t execute_tile(const FunctorType& functor, - const Policy& policy, - pointer_type ptr) const { - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[3]; - const Index begin_3 = policy.m_lower[2]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - const Index end_3 = policy.m_upper[3]; - - ValueType result = ValueType(); - - // FIXME_OPENMPTARGET: Unable to separate directives and their companion - // loops which leads to code duplication for different reduction types. - if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams distribute parallel for collapse(4) map(to \ - : functor) \ - reduction(custom \ - : result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, result); - } - } - } - } - } else { -#pragma omp target teams distribute parallel for collapse(4) map(to : functor) \ -reduction(+:result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, result); - } - } - } - } - } - - ParReduceCommon::memcpy_result(ptr, &result, sizeof(ValueType), - m_result_ptr_on_device); - } - - template - inline std::enable_if_t execute_tile(const FunctorType& functor, - const Policy& policy, - pointer_type ptr) const { - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - const Index begin_3 = policy.m_lower[3]; - const Index begin_4 = policy.m_lower[4]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - const Index end_3 = policy.m_upper[3]; - const Index end_4 = policy.m_upper[4]; - - ValueType result = ValueType(); - - // FIXME_OPENMPTARGET: Unable to separate directives and their companion - // loops which leads to code duplication for different reduction types. - if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams distribute parallel for collapse(5) map(to \ - : functor) \ - reduction(custom \ - : result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - for (auto i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - result); - } - } - } - } - } - } else { -#pragma omp target teams distribute parallel for collapse(5) map(to : functor) \ -reduction(+:result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - for (auto i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - result); - } - } - } - } - } - } - - ParReduceCommon::memcpy_result(ptr, &result, sizeof(ValueType), - m_result_ptr_on_device); - } - - template - inline std::enable_if_t execute_tile(const FunctorType& functor, - const Policy& policy, - pointer_type ptr) const { - const Index begin_0 = policy.m_lower[0]; - const Index begin_1 = policy.m_lower[1]; - const Index begin_2 = policy.m_lower[2]; - const Index begin_3 = policy.m_lower[3]; - const Index begin_4 = policy.m_lower[4]; - const Index begin_5 = policy.m_lower[5]; - - const Index end_0 = policy.m_upper[0]; - const Index end_1 = policy.m_upper[1]; - const Index end_2 = policy.m_upper[2]; - const Index end_3 = policy.m_upper[3]; - const Index end_4 = policy.m_upper[4]; - const Index end_5 = policy.m_upper[5]; - - ValueType result = ValueType(); - - // FIXME_OPENMPTARGET: Unable to separate directives and their companion - // loops which leads to code duplication for different reduction types. - if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) - -#pragma omp target teams distribute parallel for collapse(6) map(to \ - : functor) \ - reduction(custom \ - : result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - for (auto i4 = begin_4; i4 < end_4; ++i4) { - for (auto i5 = begin_5; i5 < end_5; ++i5) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5, - result); - } - } - } - } - } - } - } else { -#pragma omp target teams distribute parallel for collapse(6) map(to : functor) \ -reduction(+:result) - for (auto i0 = begin_0; i0 < end_0; ++i0) { - for (auto i1 = begin_1; i1 < end_1; ++i1) { - for (auto i2 = begin_2; i2 < end_2; ++i2) { - for (auto i3 = begin_3; i3 < end_3; ++i3) { - for (auto i4 = begin_4; i4 < end_4; ++i4) { - for (auto i5 = begin_5; i5 < end_5; ++i5) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5, - result); - } - } - } - } - } - } - } - - ParReduceCommon::memcpy_result(ptr, &result, sizeof(ValueType), - m_result_ptr_on_device); - } - - template - static int max_tile_size_product(const Policy&, const Functor&) { - return 256; - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -#undef KOKKOS_IMPL_MDRANGE_USE_NO_TILES -#endif /* KOKKOS_OPENMPTARGET_PARALLEL_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp deleted file mode 100644 index 1ada2b1911..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp +++ /dev/null @@ -1,279 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_ENABLE_TASKPOLICY) - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template class TaskQueue; - -//---------------------------------------------------------------------------- - -TaskExec::TaskExec() - : m_self_exec(0), - m_team_exec(0), - m_sync_mask(0), - m_sync_value(0), - m_sync_step(0), - m_group_rank(0), - m_team_rank(0), - m_team_size(1) {} - -TaskExec::TaskExec( - Kokkos::Impl::OpenMPTargetExec &arg_exec, int const arg_team_size) - : m_self_exec(&arg_exec), - m_team_exec(arg_exec.pool_rev(arg_exec.pool_rank_rev() / arg_team_size)), - m_sync_mask(0), - m_sync_value(0), - m_sync_step(0), - m_group_rank(arg_exec.pool_rank_rev() / arg_team_size), - m_team_rank(arg_exec.pool_rank_rev() % arg_team_size), - m_team_size(arg_team_size) { - // This team spans - // m_self_exec->pool_rev( team_size * group_rank ) - // m_self_exec->pool_rev( team_size * ( group_rank + 1 ) - 1 ) - - int64_t volatile *const sync = (int64_t *)m_self_exec->scratch_reduce(); - - sync[0] = int64_t(0); - sync[1] = int64_t(0); - - for (int i = 0; i < m_team_size; ++i) { - m_sync_value |= int64_t(1) << (8 * i); - m_sync_mask |= int64_t(3) << (8 * i); - } - - Kokkos::memory_fence(); -} - -void TaskExec::team_barrier_impl() const { - if (m_team_exec->scratch_reduce_size() < int(2 * sizeof(int64_t))) { - Kokkos::abort("TaskQueue scratch_reduce memory too small"); - } - - // Use team shared memory to synchronize. - // Alternate memory locations between barriers to avoid a sequence - // of barriers overtaking one another. - - int64_t volatile *const sync = - ((int64_t *)m_team_exec->scratch_reduce()) + (m_sync_step & 0x01); - - // This team member sets one byte within the sync variable - int8_t volatile *const sync_self = ((int8_t *)sync) + m_team_rank; - - *sync_self = int8_t(m_sync_value & 0x03); // signal arrival - - while (m_sync_value != *sync) - ; // wait for team to arrive - - ++m_sync_step; - - if (0 == (0x01 & m_sync_step)) { // Every other step - m_sync_value ^= m_sync_mask; - if (1000 < m_sync_step) m_sync_step = 0; - } -} - -//---------------------------------------------------------------------------- - -void TaskQueueSpecialization::execute( - TaskQueue *const queue) { - using execution_space = Kokkos::Experimental::OpenMPTarget; - using queue_type = TaskQueue; - using task_root_type = TaskBase; - using PoolExec = Kokkos::Impl::OpenMPTargetExec; - using Member = TaskExec; - - task_root_type *const end = (task_root_type *)task_root_type::EndTag; - - // Required: team_size <= 8 - - const int team_size = PoolExec::pool_size(2); // Threads per core - // const int team_size = PoolExec::pool_size(1); // Threads per NUMA - - if (8 < team_size) { - Kokkos::abort("TaskQueue unsupported team size"); - } - -#pragma omp parallel - { - PoolExec &self = *PoolExec::get_thread_omp(); - - Member single_exec; - Member team_exec(self, team_size); - - // Team shared memory - task_root_type *volatile *const task_shared = - (task_root_type **)team_exec.m_team_exec->scratch_thread(); - -// Barrier across entire OpenMPTarget thread pool to insure initialization -#pragma omp barrier - - // Loop until all queues are empty and no tasks in flight - - do { - task_root_type *task = 0; - - // Each team lead attempts to acquire either a thread team task - // or a single thread task for the team. - - if (0 == team_exec.team_rank()) { - task = 0 < *((volatile int *)&queue->m_ready_count) ? end : 0; - - // Loop by priority and then type - for (int i = 0; i < queue_type::NumQueue && end == task; ++i) { - for (int j = 0; j < 2 && end == task; ++j) { - task = queue_type::pop_task(&queue->m_ready[i][j]); - } - } - } - - // Team lead broadcast acquired task to team members: - - if (1 < team_exec.team_size()) { - if (0 == team_exec.team_rank()) *task_shared = task; - - // Fence to be sure task_shared is stored before the barrier - Kokkos::memory_fence(); - - // Whole team waits for every team member to reach this statement - team_exec.team_barrier(); - - // Fence to be sure task_shared is stored - Kokkos::memory_fence(); - - task = *task_shared; - } - - if (0 == task) break; // 0 == m_ready_count - - if (end == task) { - // All team members wait for whole team to reach this statement. - // Is necessary to prevent task_shared from being updated - // before it is read by all threads. - team_exec.team_barrier(); - } else if (task_root_type::TaskTeam == task->m_task_type) { - // Thread Team Task - (*task->m_apply)(task, &team_exec); - - // The m_apply function performs a barrier - - if (0 == team_exec.team_rank()) { - // team member #0 completes the task, which may delete the task - queue->complete(task); - } - } else { - // Single Thread Task - - if (0 == team_exec.team_rank()) { - (*task->m_apply)(task, &single_exec); - - queue->complete(task); - } - - // All team members wait for whole team to reach this statement. - // Not necessary to complete the task. - // Is necessary to prevent task_shared from being updated - // before it is read by all threads. - team_exec.team_barrier(); - } - } while (1); - } - // END #pragma omp parallel -} - -void TaskQueueSpecialization:: - iff_single_thread_recursive_execute( - TaskQueue *const queue) { - using execution_space = Kokkos::Experimental::OpenMPTarget; - using queue_type = TaskQueue; - using task_root_type = TaskBase; - using Member = TaskExec; - - if (1 == omp_get_num_threads()) { - task_root_type *const end = (task_root_type *)task_root_type::EndTag; - - Member single_exec; - - task_root_type *task = end; - - do { - task = end; - - // Loop by priority and then type - for (int i = 0; i < queue_type::NumQueue && end == task; ++i) { - for (int j = 0; j < 2 && end == task; ++j) { - task = queue_type::pop_task(&queue->m_ready[i][j]); - } - } - - if (end == task) break; - - (*task->m_apply)(task, &single_exec); - - queue->complete(task); - - } while (1); - } -} - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -#endif /* #if defined( KOKKOS_ENABLE_OPENMPTARGET ) && defined( \ - KOKKOS_ENABLE_TASKPOLICY ) */ diff --git a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.hpp b/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.hpp deleted file mode 100644 index f7ba01dffe..0000000000 --- a/bundled/kokkos-3.7.00/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.hpp +++ /dev/null @@ -1,347 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_OPENMP_TASK_HPP -#define KOKKOS_IMPL_OPENMP_TASK_HPP - -#if defined(KOKKOS_ENABLE_TASKPOLICY) - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template <> -class TaskQueueSpecialization { - public: - using execution_space = Kokkos::Experimental::OpenMPTarget; - using queue_type = Kokkos::Impl::TaskQueue; - using task_base_type = Kokkos::Impl::TaskBase; - - // Must specify memory space - using memory_space = Kokkos::HostSpace; - - static void iff_single_thread_recursive_execute(queue_type* const); - - // Must provide task queue execution function - static void execute(queue_type* const); - - // Must provide mechanism to set function pointer in - // execution space from the host process. - template - static void proc_set_apply(task_base_type::function_type* ptr) { - using TaskType = TaskBase; - *ptr = TaskType::apply; - } -}; - -extern template class TaskQueue; - -//---------------------------------------------------------------------------- - -template <> -class TaskExec { - private: - TaskExec(TaskExec&&) = delete; - TaskExec(TaskExec const&) = delete; - TaskExec& operator=(TaskExec&&) = delete; - TaskExec& operator=(TaskExec const&) = delete; - - using PoolExec = Kokkos::Impl::OpenMPTargetExec; - - friend class Kokkos::Impl::TaskQueue; - friend class Kokkos::Impl::TaskQueueSpecialization< - Kokkos::Experimental::OpenMPTarget>; - - PoolExec* const m_self_exec; ///< This thread's thread pool data structure - PoolExec* const m_team_exec; ///< Team thread's thread pool data structure - int64_t m_sync_mask; - int64_t mutable m_sync_value; - int mutable m_sync_step; - int m_group_rank; ///< Which "team" subset of thread pool - int m_team_rank; ///< Which thread within a team - int m_team_size; - - TaskExec(); - TaskExec(PoolExec& arg_exec, int arg_team_size); - - void team_barrier_impl() const; - - public: - KOKKOS_FUNCTION void* team_shared() const { - KOKKOS_IF_ON_HOST( - (return m_team_exec ? m_team_exec->scratch_thread() : nullptr;)) - - KOKKOS_IF_ON_DEVICE((return nullptr;)) - } - - KOKKOS_FUNCTION int team_shared_size() const { - KOKKOS_IF_ON_HOST( - (return m_team_exec ? m_team_exec->scratch_thread_size() : 0;)) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /**\brief Whole team enters this function call - * before any teeam member returns from - * this function call. - */ - KOKKOS_FUNCTION void team_barrier() const { - KOKKOS_IF_ON_HOST((if (1 < m_team_size) { team_barrier_impl(); })) - } - - KOKKOS_INLINE_FUNCTION - int team_rank() const { return m_team_rank; } - - KOKKOS_INLINE_FUNCTION - int team_size() const { return m_team_size; } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec > -TeamThreadRange(Impl::TaskExec& thread, - const iType& count) { - return Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >(thread, - count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec > -TeamThreadRange(Impl::TaskExec& thread, - const iType& start, const iType& end) { - return Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >(thread, start, - end); -} - -/** \brief Inter-thread parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all threads of the the calling thread team. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda) { - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i); - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, ValueType& initialized_result) { - int team_rank = - loop_boundaries.thread.team_rank(); // member num within the team - ValueType result = initialized_result; - - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, result); - } - - if (1 < loop_boundaries.thread.team_size()) { - ValueType* shared = (ValueType*)loop_boundaries.thread.team_shared(); - - loop_boundaries.thread.team_barrier(); - shared[team_rank] = result; - - loop_boundaries.thread.team_barrier(); - - // reduce across threads to thread 0 - if (team_rank == 0) { - for (int i = 1; i < loop_boundaries.thread.team_size(); i++) { - shared[0] += shared[i]; - } - } - - loop_boundaries.thread.team_barrier(); - - // broadcast result - initialized_result = shared[0]; - } else { - initialized_result = result; - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, const JoinType& join, ValueType& initialized_result) { - int team_rank = - loop_boundaries.thread.team_rank(); // member num within the team - ValueType result = initialized_result; - - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, result); - } - - if (1 < loop_boundaries.thread.team_size()) { - ValueType* shared = (ValueType*)loop_boundaries.thread.team_shared(); - - loop_boundaries.thread.team_barrier(); - shared[team_rank] = result; - - loop_boundaries.thread.team_barrier(); - - // reduce across threads to thread 0 - if (team_rank == 0) { - for (int i = 1; i < loop_boundaries.thread.team_size(); i++) { - join(shared[0], shared[i]); - } - } - - loop_boundaries.thread.team_barrier(); - - // broadcast result - initialized_result = shared[0]; - } else { - initialized_result = result; - } -} - -// placeholder for future function -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, ValueType& initialized_result) {} - -// placeholder for future function -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, const JoinType& join, ValueType& initialized_result) { -} - -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda) { - ValueType accum = 0; - ValueType val, local_total; - ValueType* shared = (ValueType*)loop_boundaries.thread.team_shared(); - int team_size = loop_boundaries.thread.team_size(); - int team_rank = - loop_boundaries.thread.team_rank(); // member num within the team - - // Intra-member scan - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - local_total = 0; - lambda(i, local_total, false); - val = accum; - lambda(i, val, true); - accum += local_total; - } - - shared[team_rank] = accum; - loop_boundaries.thread.team_barrier(); - - // Member 0 do scan on accumulated totals - if (team_rank == 0) { - for (iType i = 1; i < team_size; i += 1) { - shared[i] += shared[i - 1]; - } - accum = 0; // Member 0 set accum to 0 in preparation for inter-member scan - } - - loop_boundaries.thread.team_barrier(); - - // Inter-member scan adding in accumulated totals - if (team_rank != 0) { - accum = shared[team_rank - 1]; - } - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - local_total = 0; - lambda(i, local_total, false); - val = accum; - lambda(i, val, true); - accum += local_total; - } -} - -// placeholder for future function -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda) {} - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #if defined( KOKKOS_ENABLE_TASKPOLICY ) */ -#endif /* #ifndef KOKKOS_IMPL_OPENMP_TASK_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Abort.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Abort.hpp deleted file mode 100644 index e376f012f7..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Abort.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCL_ABORT_HPP -#define KOKKOS_SYCL_ABORT_HPP - -#include -#if defined(KOKKOS_ENABLE_SYCL) -#include - -namespace Kokkos { -namespace Impl { - -inline void sycl_abort(char const* msg) { -#ifdef NDEBUG - KOKKOS_IMPL_DO_NOT_USE_PRINTF("Aborting with message %s.\n", msg); -#else - // Choosing "" here causes problems but a single whitespace character works. - const char* empty = " "; - __assert_fail(msg, empty, 0, empty); -#endif -} - -} // namespace Impl -} // namespace Kokkos - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp deleted file mode 100644 index 160f606848..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp +++ /dev/null @@ -1,194 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCLDEEPCOPY_HPP -#define KOKKOS_SYCLDEEPCOPY_HPP - -#include -#include - -#include - -#ifdef KOKKOS_ENABLE_SYCL - -namespace Kokkos { -namespace Impl { - -template -struct ZeroMemset { - ZeroMemset(const Kokkos::Experimental::SYCL& exec_space, - const View& dst, - typename View::const_value_type&) { - auto event = exec_space.impl_internal_space_instance()->m_queue->memset( - dst.data(), 0, - dst.size() * sizeof(typename View::value_type)); - exec_space.impl_internal_space_instance() - ->m_queue->ext_oneapi_submit_barrier(std::vector{event}); - } - - ZeroMemset(const View& dst, - typename View::const_value_type&) { - Experimental::Impl::SYCLInternal::singleton().m_queue->memset( - dst.data(), 0, - dst.size() * sizeof(typename View::value_type)); - } -}; - -void DeepCopySYCL(void* dst, const void* src, size_t n); -void DeepCopyAsyncSYCL(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n); -void DeepCopyAsyncSYCL(void* dst, const void* src, size_t n); - -template -struct DeepCopy::value>> { - DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } - DeepCopy(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { - DeepCopyAsyncSYCL(instance, dst, src, n); - } -}; - -template -struct DeepCopy::value>> { - DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } - DeepCopy(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { - DeepCopyAsyncSYCL(instance, dst, src, n); - } -}; - -template -struct DeepCopy::value && - is_sycl_type_space::value>> { - DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } - DeepCopy(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { - DeepCopyAsyncSYCL(instance, dst, src, n); - } -}; - -template -struct DeepCopy< - MemSpace1, MemSpace2, ExecutionSpace, - std::enable_if_t< - is_sycl_type_space::value && - is_sycl_type_space::value && - !std::is_same::value>> { - inline DeepCopy(void* dst, const void* src, size_t n) { - DeepCopySYCL(dst, src, n); - } - - inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, - size_t n) { - exec.fence(fence_string()); - DeepCopyAsyncSYCL(dst, src, n); - } - - private: - static const std::string& fence_string() { - static const std::string string = - std::string("Kokkos::Impl::DeepCopy<") + MemSpace1::name() + "Space, " + - MemSpace2::name() + - "Space, ExecutionSpace>::DeepCopy: fence before copy"; - return string; - } -}; - -template -struct DeepCopy< - MemSpace, HostSpace, ExecutionSpace, - std::enable_if_t< - is_sycl_type_space::value && - !std::is_same::value>> { - inline DeepCopy(void* dst, const void* src, size_t n) { - DeepCopySYCL(dst, src, n); - } - - inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, - size_t n) { - exec.fence(fence_string()); - DeepCopyAsyncSYCL(dst, src, n); - } - - private: - static const std::string& fence_string() { - static const std::string string = - std::string("Kokkos::Impl::DeepCopy<") + MemSpace::name() + - "Space, HostSpace, ExecutionSpace>::DeepCopy: fence before copy"; - return string; - } -}; - -template -struct DeepCopy< - HostSpace, MemSpace, ExecutionSpace, - std::enable_if_t< - is_sycl_type_space::value && - !std::is_same::value>> { - inline DeepCopy(void* dst, const void* src, size_t n) { - DeepCopySYCL(dst, src, n); - } - - inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, - size_t n) { - exec.fence(fence_string()); - DeepCopyAsyncSYCL(dst, src, n); - } - - private: - static const std::string& fence_string() { - static const std::string string = - std::string("Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"; - return string; - } -}; - -} // namespace Impl -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Half_Conversion.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Half_Conversion.hpp deleted file mode 100644 index 3adbb16268..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Half_Conversion.hpp +++ /dev/null @@ -1,159 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.5 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCL_HALF_HPP_ -#define KOKKOS_SYCL_HALF_HPP_ - -#ifdef KOKKOS_IMPL_SYCL_HALF_TYPE_DEFINED - -#include -#include // reduction_identity - -namespace Kokkos { -namespace Experimental { - -/************************** half conversions **********************************/ -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(half_t val) { return val; } - -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(float val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(double val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(short val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(unsigned short val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(int val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(unsigned int val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(long long val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(unsigned long long val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(long val) { return half_t::impl_type(val); } -KOKKOS_INLINE_FUNCTION -half_t cast_to_half(unsigned long val) { return half_t::impl_type(val); } - -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION - std::enable_if_t::value, T> - cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION - std::enable_if_t::value, T> - cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -cast_from_half(half_t val) { - return half_t::impl_type(val); -} -template -KOKKOS_INLINE_FUNCTION - std::enable_if_t::value, T> - cast_from_half(half_t val) { - return half_t::impl_type(val); -} -} // namespace Experimental - -template <> -struct reduction_identity { - KOKKOS_FORCEINLINE_FUNCTION constexpr static Kokkos::Experimental::half_t - sum() noexcept { - return Kokkos::Experimental::half_t::impl_type(0.0F); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static Kokkos::Experimental::half_t - prod() noexcept { - return Kokkos::Experimental::half_t::impl_type(1.0F); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static Kokkos::Experimental::half_t - max() noexcept { - return std::numeric_limits< - Kokkos::Experimental::half_t::impl_type>::lowest(); - } - KOKKOS_FORCEINLINE_FUNCTION constexpr static Kokkos::Experimental::half_t - min() noexcept { - return std::numeric_limits::max(); - } -}; - -} // namespace Kokkos -#endif // KOKKOS_IMPL_SYCL_HALF_TYPE_DEFINED -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Half_Impl_Type.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Half_Impl_Type.hpp deleted file mode 100644 index cf7d513b99..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Half_Impl_Type.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.5 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCL_HALF_IMPL_TYPE_HPP_ -#define KOKKOS_SYCL_HALF_IMPL_TYPE_HPP_ - -#include -#ifdef KOKKOS_ENABLE_SYCL - -#include - -#ifndef KOKKOS_IMPL_HALF_TYPE_DEFINED -// Make sure no one else tries to define half_t -#define KOKKOS_IMPL_HALF_TYPE_DEFINED -#define KOKKOS_IMPL_SYCL_HALF_TYPE_DEFINED - -namespace Kokkos { -namespace Impl { -struct half_impl_t { - using type = sycl::half; -}; -} // namespace Impl -} // namespace Kokkos -#endif // KOKKOS_IMPL_HALF_TYPE_DEFINED -#endif // KOKKOS_ENABLE_SYCL -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Instance.cpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Instance.cpp deleted file mode 100644 index 37721247a7..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Instance.cpp +++ /dev/null @@ -1,365 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include //kokkos_malloc - -namespace Kokkos { -namespace Experimental { -namespace Impl { - -std::vector*> SYCLInternal::all_queues; -std::mutex SYCLInternal::mutex; - -Kokkos::View sycl_global_unique_token_locks( - bool deallocate) { - static Kokkos::View locks = - Kokkos::View(); - if (!deallocate && locks.extent(0) == 0) - locks = Kokkos::View( - "Kokkos::UniqueToken::m_locks", SYCL().concurrency()); - if (deallocate) locks = Kokkos::View(); - return locks; -} - -SYCLInternal::~SYCLInternal() { - if (!was_finalized || m_scratchSpace || m_scratchFlags) { - std::cerr << "Kokkos::Experimental::SYCL ERROR: Failed to call " - "Kokkos::Experimental::SYCL::finalize()" - << std::endl; - std::cerr.flush(); - } -} - -int SYCLInternal::verify_is_initialized(const char* const label) const { - if (!is_initialized()) { - Kokkos::abort((std::string("Kokkos::Experimental::SYCL::") + label + - " : ERROR device not initialized\n") - .c_str()); - } - return is_initialized(); -} -SYCLInternal& SYCLInternal::singleton() { - static SYCLInternal self; - return self; -} - -void SYCLInternal::initialize(const sycl::device& d) { - auto exception_handler = [](sycl::exception_list exceptions) { - bool asynchronous_error = false; - for (std::exception_ptr const& e : exceptions) { - try { - std::rethrow_exception(e); - } catch (sycl::exception const& e) { - std::cerr << e.what() << '\n'; - asynchronous_error = true; - } - } - if (asynchronous_error) - Kokkos::Impl::throw_runtime_exception( - "There was an asynchronous SYCL error!\n"); - }; - initialize(sycl::queue{d, exception_handler}); -} - -// FIXME_SYCL -void SYCLInternal::initialize(const sycl::queue& q) { - if (was_finalized) - Kokkos::abort("Calling SYCL::initialize after SYCL::finalize is illegal\n"); - - if (is_initialized()) return; - - if (!HostSpace::execution_space::impl_is_initialized()) { - const std::string msg( - "SYCL::initialize ERROR : HostSpace::execution_space is not " - "initialized"); - Kokkos::Impl::throw_runtime_exception(msg); - } - - const bool ok_init = nullptr == m_scratchSpace || nullptr == m_scratchFlags; - const bool ok_dev = true; - if (ok_init && ok_dev) { - m_queue = q; - // guard pushing to all_queues - { - std::scoped_lock lock(mutex); - all_queues.push_back(&m_queue); - } - const sycl::device& d = m_queue->get_device(); - - m_maxWorkgroupSize = - d.template get_info(); - // FIXME_SYCL this should give the correct value for NVIDIA GPUs - m_maxConcurrency = - m_maxWorkgroupSize * 2 * - d.template get_info(); - - // Setup concurent bitset for obtaining unique tokens from within an - // executing kernel. - { - const int32_t buffer_bound = - Kokkos::Impl::concurrent_bitset::buffer_bound(m_maxConcurrency); - using Record = Kokkos::Impl::SharedAllocationRecord< - Kokkos::Experimental::SYCLDeviceUSMSpace, void>; - Record* const r = - Record::allocate(Kokkos::Experimental::SYCLDeviceUSMSpace(*m_queue), - "Kokkos::Experimental::SYCL::InternalScratchBitset", - sizeof(uint32_t) * buffer_bound); - Record::increment(r); - } - - m_maxShmemPerBlock = - d.template get_info(); - - for (auto& usm_mem : m_indirectKernelMem) { - usm_mem.reset(*m_queue, m_instance_id); - } - - } else { - std::ostringstream msg; - msg << "Kokkos::Experimental::SYCL::initialize(...) FAILED"; - - if (!ok_init) { - msg << " : Already initialized"; - } - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - - m_team_scratch_current_size = 0; - m_team_scratch_ptr = nullptr; -} - -sycl::device_ptr SYCLInternal::resize_team_scratch_space( - std::int64_t bytes, bool force_shrink) { - if (m_team_scratch_current_size == 0) { - m_team_scratch_current_size = bytes; - m_team_scratch_ptr = - Kokkos::kokkos_malloc( - "Kokkos::Experimental::SYCLDeviceUSMSpace::TeamScratchMemory", - m_team_scratch_current_size); - } - if ((bytes > m_team_scratch_current_size) || - ((bytes < m_team_scratch_current_size) && (force_shrink))) { - m_team_scratch_current_size = bytes; - m_team_scratch_ptr = - Kokkos::kokkos_realloc( - m_team_scratch_ptr, m_team_scratch_current_size); - } - return m_team_scratch_ptr; -} - -uint32_t SYCLInternal::impl_get_instance_id() const { return m_instance_id; } - -void SYCLInternal::finalize() { - SYCLInternal::fence(*m_queue, - "Kokkos::SYCLInternal::finalize: fence on finalization", - m_instance_id); - was_finalized = true; - - // The global_unique_token_locks array is static and should only be - // deallocated once by the defualt instance - if (this == &singleton()) Impl::sycl_global_unique_token_locks(true); - - using RecordSYCL = Kokkos::Impl::SharedAllocationRecord; - if (nullptr != m_scratchSpace) - RecordSYCL::decrement(RecordSYCL::get_record(m_scratchSpace)); - if (nullptr != m_scratchFlags) - RecordSYCL::decrement(RecordSYCL::get_record(m_scratchFlags)); - m_syclDev = -1; - m_scratchSpaceCount = 0; - m_scratchSpace = nullptr; - m_scratchFlagsCount = 0; - m_scratchFlags = nullptr; - - if (m_team_scratch_current_size > 0) - Kokkos::kokkos_free( - m_team_scratch_ptr); - m_team_scratch_current_size = 0; - m_team_scratch_ptr = nullptr; - - for (auto& usm_mem : m_indirectKernelMem) usm_mem.reset(); - // guard erasing from all_queues - { - std::scoped_lock lock(mutex); - all_queues.erase(std::find(all_queues.begin(), all_queues.end(), &m_queue)); - } - m_queue.reset(); -} - -sycl::device_ptr SYCLInternal::scratch_space(const std::size_t size) { - const size_type sizeScratchGrain = - sizeof(Kokkos::Experimental::SYCL::size_type); - if (verify_is_initialized("scratch_space") && - m_scratchSpaceCount * sizeScratchGrain < size) { - m_scratchSpaceCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = Kokkos::Impl::SharedAllocationRecord< - Kokkos::Experimental::SYCLDeviceUSMSpace, void>; - - if (nullptr != m_scratchSpace) - Record::decrement(Record::get_record(m_scratchSpace)); - - Record* const r = - Record::allocate(Kokkos::Experimental::SYCLDeviceUSMSpace(*m_queue), - "Kokkos::Experimental::SYCL::InternalScratchSpace", - (sizeScratchGrain * m_scratchSpaceCount)); - - Record::increment(r); - - m_scratchSpace = reinterpret_cast(r->data()); - } - - return m_scratchSpace; -} - -sycl::device_ptr SYCLInternal::scratch_flags(const std::size_t size) { - const size_type sizeScratchGrain = - sizeof(Kokkos::Experimental::SYCL::size_type); - if (verify_is_initialized("scratch_flags") && - m_scratchFlagsCount * sizeScratchGrain < size) { - m_scratchFlagsCount = (size + sizeScratchGrain - 1) / sizeScratchGrain; - - using Record = Kokkos::Impl::SharedAllocationRecord< - Kokkos::Experimental::SYCLDeviceUSMSpace, void>; - - if (nullptr != m_scratchFlags) - Record::decrement(Record::get_record(m_scratchFlags)); - - Record* const r = - Record::allocate(Kokkos::Experimental::SYCLDeviceUSMSpace(*m_queue), - "Kokkos::Experimental::SYCL::InternalScratchFlags", - (sizeScratchGrain * m_scratchFlagsCount)); - - Record::increment(r); - - m_scratchFlags = reinterpret_cast(r->data()); - } - m_queue->memset(m_scratchFlags, 0, m_scratchFlagsCount * sizeScratchGrain); - fence(*m_queue, - "Kokkos::Experimental::SYCLInternal::scratch_flags fence after " - "initializing m_scratchFlags", - m_instance_id); - - return m_scratchFlags; -} - -template -void SYCLInternal::fence_helper(WAT& wat, const std::string& name, - uint32_t instance_id) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::SYCL>( - name, Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{instance_id}, - [&]() { - try { - wat.wait_and_throw(); - } catch (sycl::exception const& e) { - Kokkos::Impl::throw_runtime_exception( - std::string("There was a synchronous SYCL error:\n") += e.what()); - } - }); -} -template void SYCLInternal::fence_helper(sycl::queue&, - const std::string&, - uint32_t); -template void SYCLInternal::fence_helper(sycl::event&, - const std::string&, - uint32_t); - -// This function cycles through a pool of USM allocations for functors -SYCLInternal::IndirectKernelMem& SYCLInternal::get_indirect_kernel_mem() { - // Thread safety: atomically increment round robin variable - // NB: atomic_fetch_inc_mod returns values in range [0-N], not - // [0-N) as might be expected. - size_t next_pool = desul::atomic_fetch_inc_mod( - &m_pool_next, m_usm_pool_size - 1, desul::MemoryOrderRelaxed(), - desul::MemoryScopeDevice()); - return m_indirectKernelMem[next_pool]; -} - -template -size_t SYCLInternal::USMObjectMem::reserve(size_t n) { - assert(m_q); - - if (m_capacity < n) { - using Record = Kokkos::Impl::SharedAllocationRecord; - // First free what we have (in case malloc can reuse it) - if (m_data) Record::decrement(Record::get_record(m_data)); - - Record* const r = Record::allocate( - AllocationSpace(*m_q), "Kokkos::Experimental::SYCL::USMObjectMem", n); - Record::increment(r); - - m_data = r->data(); - if constexpr (sycl::usm::alloc::device == Kind) - m_staging.reset(new char[n]); - m_capacity = n; - } - - return m_capacity; -} - -template -void SYCLInternal::USMObjectMem::reset() { - if (m_data) { - // This implies a fence since this class is not copyable - // and deallocating implies a fence across all registered queues. - using Record = Kokkos::Impl::SharedAllocationRecord; - Record::decrement(Record::get_record(m_data)); - - m_capacity = 0; - m_data = nullptr; - } - m_q.reset(); -} - -template class SYCLInternal::USMObjectMem; -template class SYCLInternal::USMObjectMem; -template class SYCLInternal::USMObjectMem; - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp deleted file mode 100644 index 3e90ec1fb5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef KOKKOS_SYCL_MDRANGEPOLICY_HPP_ -#define KOKKOS_SYCL_MDRANGEPOLICY_HPP_ - -#include - -namespace Kokkos { - -template <> -struct default_outer_direction { - using type = Iterate; - static constexpr Iterate value = Iterate::Left; -}; - -template <> -struct default_inner_direction { - using type = Iterate; - static constexpr Iterate value = Iterate::Left; -}; - -namespace Impl { - -// Settings for MDRangePolicy -template <> -inline TileSizeProperties get_tile_size_properties( - const Kokkos::Experimental::SYCL& space) { - TileSizeProperties properties; - properties.max_threads = - space.impl_internal_space_instance()->m_maxWorkgroupSize; - properties.default_largest_tile_size = 16; - properties.default_tile_size = 2; - properties.max_total_tile_size = properties.max_threads; - return properties; -} - -} // Namespace Impl -} // Namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Reduce.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Reduce.hpp deleted file mode 100644 index e980a82a58..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Reduce.hpp +++ /dev/null @@ -1,832 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCL_PARALLEL_REDUCE_HPP -#define KOKKOS_SYCL_PARALLEL_REDUCE_HPP - -#include - -#include -#if defined(KOKKOS_ENABLE_SYCL) -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { - -template -inline constexpr bool use_shuffle_based_algorithm = - std::is_reference_v; - -namespace SYCLReduction { -template -std::enable_if_t> workgroup_reduction( - sycl::nd_item& item, sycl::local_ptr local_mem, - sycl::device_ptr results_ptr, - sycl::global_ptr device_accessible_result_ptr, - const unsigned int value_count, const ReducerType& final_reducer, - bool final, unsigned int max_size) { - const auto local_id = item.get_local_linear_id(); - - // Perform the actual workgroup reduction in each subgroup - // separately. - auto sg = item.get_sub_group(); - auto* result = &local_mem[local_id * value_count]; - const auto id_in_sg = sg.get_local_id()[0]; - const auto local_range = - std::min(sg.get_local_range()[0], max_size); - const auto upper_stride_bound = - std::min(local_range - id_in_sg, max_size - local_id); - for (unsigned int stride = 1; stride < local_range; stride <<= 1) { - if (stride < upper_stride_bound) - final_reducer.join(result, &local_mem[(local_id + stride) * value_count]); - sycl::group_barrier(sg); - } - sycl::group_barrier(item.get_group()); - - // Copy the subgroup results into the first positions of the - // reduction array. - if (id_in_sg == 0) - final_reducer.copy(&local_mem[sg.get_group_id()[0] * value_count], result); - sycl::group_barrier(item.get_group()); - - // Do the final reduction only using the first subgroup. - if (sg.get_group_id()[0] == 0) { - const auto n_subgroups = sg.get_group_range()[0]; - auto* result_ = &local_mem[id_in_sg * value_count]; - // In case the number of subgroups is larger than the range of - // the first subgroup, we first combine the items with a higher - // index. - for (unsigned int offset = local_range; offset < n_subgroups; - offset += local_range) - if (id_in_sg + offset < n_subgroups) - final_reducer.join(result_, - &local_mem[(id_in_sg + offset) * value_count]); - sycl::group_barrier(sg); - - // Then, we proceed as before. - for (unsigned int stride = 1; stride < local_range; stride <<= 1) { - if (id_in_sg + stride < n_subgroups) - final_reducer.join(result_, - &local_mem[(id_in_sg + stride) * value_count]); - sycl::group_barrier(sg); - } - - // Finally, we copy the workgroup results back to global memory - // to be used in the next iteration. If this is the last - // iteration, i.e., there is only one workgroup also call - // final() if necessary. - if (id_in_sg == 0) { - if (final) { - final_reducer.final(&local_mem[0]); - if (device_accessible_result_ptr != nullptr) - final_reducer.copy(&device_accessible_result_ptr[0], &local_mem[0]); - else - final_reducer.copy(&results_ptr[0], &local_mem[0]); - } else - final_reducer.copy( - &results_ptr[(item.get_group_linear_id()) * value_count], - &local_mem[0]); - } - } -} - -template -std::enable_if_t> workgroup_reduction( - sycl::nd_item& item, sycl::local_ptr local_mem, - ValueType local_value, sycl::device_ptr results_ptr, - sycl::global_ptr device_accessible_result_ptr, - const ReducerType& final_reducer, bool final, unsigned int max_size) { - const auto local_id = item.get_local_linear_id(); - - // Perform the actual workgroup reduction in each subgroup - // separately. - auto sg = item.get_sub_group(); - const auto id_in_sg = sg.get_local_id()[0]; - const auto local_range = - std::min(sg.get_local_range()[0], max_size); - const auto upper_stride_bound = - std::min(local_range - id_in_sg, max_size - local_id); - for (unsigned int stride = 1; stride < local_range; stride <<= 1) { - auto tmp = sg.shuffle_down(local_value, stride); - if (stride < upper_stride_bound) final_reducer.join(&local_value, &tmp); - } - - // Copy the subgroup results into the first positions of the - // reduction array. - const auto max_subgroup_size = sg.get_max_local_range()[0]; - const auto n_active_subgroups = - (max_size + max_subgroup_size - 1) / max_subgroup_size; - if (id_in_sg == 0 && sg.get_group_id()[0] <= n_active_subgroups) - local_mem[sg.get_group_id()[0]] = local_value; - item.barrier(sycl::access::fence_space::local_space); - - // Do the final reduction only using the first subgroup. - if (sg.get_group_id()[0] == 0) { - auto sg_value = local_mem[id_in_sg < n_active_subgroups ? id_in_sg : 0]; - - // In case the number of subgroups is larger than the range of - // the first subgroup, we first combine the items with a higher - // index. - if (n_active_subgroups > local_range) { - for (unsigned int offset = local_range; offset < n_active_subgroups; - offset += local_range) - if (id_in_sg + offset < n_active_subgroups) { - final_reducer.join(&sg_value, &local_mem[(id_in_sg + offset)]); - } - sg.barrier(); - } - - // Then, we proceed as before. - for (unsigned int stride = 1; stride < local_range; stride <<= 1) { - auto tmp = sg.shuffle_down(sg_value, stride); - if (id_in_sg + stride < n_active_subgroups) - final_reducer.join(&sg_value, &tmp); - } - - // Finally, we copy the workgroup results back to global memory - // to be used in the next iteration. If this is the last - // iteration, i.e., there is only one workgroup also call - // final() if necessary. - if (id_in_sg == 0) { - if (final) { - final_reducer.final(&sg_value); - if (device_accessible_result_ptr != nullptr) - device_accessible_result_ptr[0] = sg_value; - else - results_ptr[0] = sg_value; - } else - results_ptr[(item.get_group_linear_id())] = sg_value; - } - } -} - -} // namespace SYCLReduction - -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::SYCL> { - public: - using Policy = Kokkos::RangePolicy; - - private: - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = - FunctorAnalysis; - using execution_space = typename Analysis::execution_space; - using value_type = typename Analysis::value_type; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - using WorkTag = typename Policy::work_tag; - - public: - // V - View - template - ParallelReduce(const FunctorType& f, const Policy& p, const V& v, - std::enable_if_t::value, void*> = nullptr) - : m_functor(f), - m_policy(p), - m_result_ptr(v.data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_shared_memory_lock( - p.space().impl_internal_space_instance()->m_mutexScratchSpace) {} - - ParallelReduce(const FunctorType& f, const Policy& p, - const ReducerType& reducer) - : m_functor(f), - m_policy(p), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_shared_memory_lock( - p.space().impl_internal_space_instance()->m_mutexScratchSpace) {} - - private: - template - sycl::event sycl_direct_launch( - const PolicyType& policy, const FunctorWrapper& functor_wrapper, - const ReducerWrapper& reducer_wrapper, - const std::vector& memcpy_events) const { - // Convenience references - const Kokkos::Experimental::SYCL& space = policy.space(); - Kokkos::Experimental::Impl::SYCLInternal& instance = - *space.impl_internal_space_instance(); - sycl::queue& q = space.sycl_queue(); - - // FIXME_SYCL optimize - constexpr size_t wgroup_size = 128; - constexpr size_t values_per_thread = 2; - std::size_t size = policy.end() - policy.begin(); - const auto init_size = std::max( - ((size + values_per_thread - 1) / values_per_thread + wgroup_size - 1) / - wgroup_size, - 1); - const unsigned int value_count = - Analysis::value_count(ReducerConditional::select(m_functor, m_reducer)); - const auto results_ptr = - static_cast>(instance.scratch_space( - sizeof(value_type) * std::max(value_count, 1u) * init_size)); - sycl::global_ptr device_accessible_result_ptr = - m_result_ptr_device_accessible ? m_result_ptr : nullptr; - auto scratch_flags = static_cast>( - instance.scratch_flags(sizeof(unsigned int))); - - sycl::event last_reduction_event; - - // If size<=1 we only call init(), the functor and possibly final once - // working with the global scratch memory but don't copy back to - // m_result_ptr yet. - if (size <= 1) { - auto parallel_reduce_event = q.submit([&](sycl::handler& cgh) { - const auto begin = policy.begin(); - cgh.depends_on(memcpy_events); - cgh.single_task([=]() { - const auto& functor = functor_wrapper.get_functor(); - const auto& selected_reducer = ReducerConditional::select( - static_cast(functor), - static_cast(reducer_wrapper.get_functor())); - typename Analysis::Reducer final_reducer(&selected_reducer); - reference_type update = final_reducer.init(results_ptr); - if (size == 1) { - if constexpr (std::is_void::value) - functor(begin, update); - else - functor(WorkTag(), begin, update); - } - final_reducer.final(results_ptr); - if (device_accessible_result_ptr != nullptr) - final_reducer.copy(device_accessible_result_ptr.get(), - results_ptr.get()); - }); - }); - q.ext_oneapi_submit_barrier( - std::vector{parallel_reduce_event}); - last_reduction_event = parallel_reduce_event; - } - - // Otherwise, we perform a reduction on the values in all workgroups - // separately, write the workgroup results back to global memory and recurse - // until only one workgroup does the reduction and thus gets the final - // value. - if (size > 1) { - auto n_wgroups = ((size + values_per_thread - 1) / values_per_thread + - wgroup_size - 1) / - wgroup_size; - auto parallel_reduce_event = q.submit([&](sycl::handler& cgh) { - sycl::accessor - local_mem(sycl::range<1>(wgroup_size) * std::max(value_count, 1u), - cgh); - sycl::accessor - num_teams_done(1, cgh); - - const auto begin = policy.begin(); - - cgh.depends_on(memcpy_events); - - cgh.parallel_for( - sycl::nd_range<1>(n_wgroups * wgroup_size, wgroup_size), - [=](sycl::nd_item<1> item) { - const auto local_id = item.get_local_linear_id(); - const auto global_id = - wgroup_size * item.get_group_linear_id() * values_per_thread + - local_id; - const auto& functor = functor_wrapper.get_functor(); - const auto& selected_reducer = ReducerConditional::select( - static_cast(functor), - static_cast( - reducer_wrapper.get_functor())); - typename Analysis::Reducer final_reducer(&selected_reducer); - - using index_type = typename Policy::index_type; - const auto upper_bound = std::min( - global_id + values_per_thread * wgroup_size, size); - - if constexpr (Analysis::StaticValueSize == 0) { - reference_type update = - final_reducer.init(&local_mem[local_id * value_count]); - for (index_type id = global_id; id < upper_bound; - id += wgroup_size) { - if constexpr (std::is_void::value) - functor(id + begin, update); - else - functor(WorkTag(), id + begin, update); - } - item.barrier(sycl::access::fence_space::local_space); - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), results_ptr, - device_accessible_result_ptr, value_count, final_reducer, - false, std::min(size, wgroup_size)); - - if (local_id == 0) { - sycl::atomic_ref - scratch_flags_ref(*scratch_flags); - num_teams_done[0] = ++scratch_flags_ref; - } - item.barrier(sycl::access::fence_space::local_space); - if (num_teams_done[0] == n_wgroups) { - if (local_id >= n_wgroups) - final_reducer.init(&local_mem[local_id * value_count]); - else { - final_reducer.copy(&local_mem[local_id * value_count], - &results_ptr[local_id * value_count]); - for (unsigned int id = local_id + wgroup_size; - id < n_wgroups; id += wgroup_size) { - final_reducer.join(&local_mem[local_id * value_count], - &results_ptr[id * value_count]); - } - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), results_ptr, - device_accessible_result_ptr, value_count, final_reducer, - true, std::min(n_wgroups, wgroup_size)); - } - } else { - value_type local_value; - reference_type update = final_reducer.init(&local_value); - for (index_type id = global_id; id < upper_bound; - id += wgroup_size) { - if constexpr (std::is_void::value) - functor(id + begin, update); - else - functor(WorkTag(), id + begin, update); - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), local_value, results_ptr, - device_accessible_result_ptr, final_reducer, false, - std::min(size, wgroup_size)); - - if (local_id == 0) { - sycl::atomic_ref - scratch_flags_ref(*scratch_flags); - num_teams_done[0] = ++scratch_flags_ref; - } - item.barrier(sycl::access::fence_space::local_space); - if (num_teams_done[0] == n_wgroups) { - if (local_id >= n_wgroups) - final_reducer.init(&local_value); - else { - local_value = results_ptr[local_id]; - for (unsigned int id = local_id + wgroup_size; - id < n_wgroups; id += wgroup_size) { - final_reducer.join(&local_value, &results_ptr[id]); - } - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), local_value, results_ptr, - device_accessible_result_ptr, final_reducer, true, - std::min(n_wgroups, wgroup_size)); - } - } - }); - }); - last_reduction_event = q.ext_oneapi_submit_barrier( - std::vector{parallel_reduce_event}); - } - - // At this point, the reduced value is written to the entry in results_ptr - // and all that is left is to copy it back to the given result pointer if - // necessary. - if (m_result_ptr && !m_result_ptr_device_accessible) { - Kokkos::Impl::DeepCopy( - space, m_result_ptr, results_ptr, - sizeof(*m_result_ptr) * value_count); - } - - return last_reduction_event; - } - - public: - void execute() const { - Kokkos::Experimental::Impl::SYCLInternal& instance = - *m_policy.space().impl_internal_space_instance(); - using IndirectKernelMem = - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem; - IndirectKernelMem& indirectKernelMem = instance.get_indirect_kernel_mem(); - IndirectKernelMem& indirectReducerMem = instance.get_indirect_kernel_mem(); - - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); - auto reducer_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_reducer, indirectReducerMem); - - sycl::event event = sycl_direct_launch( - m_policy, functor_wrapper, reducer_wrapper, - {functor_wrapper.get_copy_event(), reducer_wrapper.get_copy_event()}); - functor_wrapper.register_event(event); - reducer_wrapper.register_event(event); - } - - private: - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const bool m_result_ptr_device_accessible; - - // Only let one Parallel/Scan modify the shared memory. The - // constructor acquires the mutex which is released in the destructor. - std::scoped_lock m_shared_memory_lock; -}; - -template -class ParallelReduce, ReducerType, - Kokkos::Experimental::SYCL> { - public: - using Policy = Kokkos::MDRangePolicy; - - private: - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = - FunctorAnalysis; - using execution_space = typename Analysis::execution_space; - using value_type = typename Analysis::value_type; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - using WorkTag = typename Policy::work_tag; - - // MDRangePolicy is not trivially copyable. Hence, replicate the data we - // really need in DeviceIterateTile in a trivially copyable struct. - struct BarePolicy { - using index_type = typename Policy::index_type; - - BarePolicy(const Policy& policy) - : m_lower(policy.m_lower), - m_upper(policy.m_upper), - m_tile(policy.m_tile), - m_tile_end(policy.m_tile_end), - m_num_tiles(policy.m_num_tiles), - m_prod_tile_dims(policy.m_prod_tile_dims) {} - - const typename Policy::point_type m_lower; - const typename Policy::point_type m_upper; - const typename Policy::tile_type m_tile; - const typename Policy::point_type m_tile_end; - const typename Policy::index_type m_num_tiles; - const typename Policy::index_type m_prod_tile_dims; - static constexpr Iterate inner_direction = Policy::inner_direction; - static constexpr int rank = Policy::rank; - }; - - public: - // V - View - template - ParallelReduce(const FunctorType& f, const Policy& p, const V& v, - std::enable_if_t::value, void*> = nullptr) - : m_functor(f), - m_policy(p), - m_space(p.space()), - m_result_ptr(v.data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_shared_memory_lock( - m_space.impl_internal_space_instance()->m_mutexScratchSpace) {} - - ParallelReduce(const FunctorType& f, const Policy& p, - const ReducerType& reducer) - : m_functor(f), - m_policy(p), - m_space(p.space()), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_shared_memory_lock( - m_space.impl_internal_space_instance()->m_mutexScratchSpace) {} - - private: - template - sycl::event sycl_direct_launch( - const PolicyType& policy, const FunctorWrapper& functor_wrapper, - const ReducerWrapper& reducer_wrapper, - const std::vector& memcpy_events) const { - // Convenience references - Kokkos::Experimental::Impl::SYCLInternal& instance = - *m_space.impl_internal_space_instance(); - sycl::queue& q = m_space.sycl_queue(); - - const typename Policy::index_type nwork = m_policy.m_num_tiles; - const typename Policy::index_type block_size = - std::pow(2, std::ceil(std::log2(m_policy.m_prod_tile_dims))); - - const sycl::range<1> local_range(block_size); - // REMEMBER swap local x<->y to be conforming with Cuda/HIP implementation - const sycl::range<1> global_range(nwork * block_size); - const sycl::nd_range<1> range{global_range, local_range}; - - const size_t wgroup_size = range.get_local_range().size(); - size_t size = range.get_global_range().size(); - const auto init_size = - std::max((size + wgroup_size - 1) / wgroup_size, 1); - const unsigned int value_count = - Analysis::value_count(ReducerConditional::select(m_functor, m_reducer)); - const auto results_ptr = - static_cast>(instance.scratch_space( - sizeof(value_type) * std::max(value_count, 1u) * init_size)); - sycl::global_ptr device_accessible_result_ptr = - m_result_ptr_device_accessible ? m_result_ptr : nullptr; - auto scratch_flags = static_cast>( - instance.scratch_flags(sizeof(unsigned int))); - - sycl::event last_reduction_event; - - // If size<=1 we only call init(), the functor and possibly final once - // working with the global scratch memory but don't copy back to - // m_result_ptr yet. - if (size <= 1) { - auto parallel_reduce_event = q.submit([&](sycl::handler& cgh) { - cgh.depends_on(memcpy_events); - cgh.single_task([=]() { - const auto& functor = functor_wrapper.get_functor(); - const auto& selected_reducer = ReducerConditional::select( - static_cast(functor), - static_cast(reducer_wrapper.get_functor())); - typename Analysis::Reducer final_reducer(&selected_reducer); - - reference_type update = final_reducer.init(results_ptr); - if (size == 1) { - Kokkos::Impl::Reduce::DeviceIterateTile< - Policy::rank, BarePolicy, FunctorType, - typename Policy::work_tag, reference_type>( - policy, functor, update, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}) - .exec_range(); - } - final_reducer.final(results_ptr); - if (device_accessible_result_ptr) - final_reducer.copy(device_accessible_result_ptr.get(), - results_ptr.get()); - }); - }); - q.ext_oneapi_submit_barrier( - std::vector{parallel_reduce_event}); - last_reduction_event = parallel_reduce_event; - } - - // Otherwise, we perform a reduction on the values in all workgroups - // separately, write the workgroup results back to global memory and recurse - // until only one workgroup does the reduction and thus gets the final - // value. - if (size > 1) { - auto n_wgroups = (size + wgroup_size - 1) / wgroup_size; - auto parallel_reduce_event = q.submit([&](sycl::handler& cgh) { - sycl::accessor - local_mem(sycl::range<1>(wgroup_size) * std::max(value_count, 1u), - cgh); - sycl::accessor - num_teams_done(1, cgh); - - const BarePolicy bare_policy = m_policy; - - cgh.depends_on(memcpy_events); - - cgh.parallel_for(range, [=](sycl::nd_item<1> item) { - const auto local_id = item.get_local_linear_id(); - const auto& functor = functor_wrapper.get_functor(); - const auto& selected_reducer = ReducerConditional::select( - static_cast(functor), - static_cast(reducer_wrapper.get_functor())); - typename Analysis::Reducer final_reducer(&selected_reducer); - - // In the first iteration, we call functor to initialize the local - // memory. Otherwise, the local memory is initialized with the - // results from the previous iteration that are stored in global - // memory. - using index_type = typename Policy::index_type; - - // SWAPPED here to be conforming with CUDA implementation - const index_type local_x = 0; - const index_type local_y = item.get_local_id(0); - const index_type local_z = 0; - const index_type global_x = item.get_group(0); - const index_type global_y = 0; - const index_type global_z = 0; - const index_type n_global_x = item.get_group_range(0); - const index_type n_global_y = 1; - const index_type n_global_z = 1; - - if constexpr (Analysis::StaticValueSize == 0) { - reference_type update = - final_reducer.init(&local_mem[local_id * value_count]); - - Kokkos::Impl::Reduce::DeviceIterateTile< - Policy::rank, BarePolicy, FunctorType, - typename Policy::work_tag, reference_type>( - bare_policy, functor, update, - {n_global_x, n_global_y, n_global_z}, - {global_x, global_y, global_z}, {local_x, local_y, local_z}) - .exec_range(); - item.barrier(sycl::access::fence_space::local_space); - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), results_ptr, - device_accessible_result_ptr, value_count, final_reducer, false, - std::min(size, wgroup_size)); - - if (local_id == 0) { - sycl::atomic_ref - scratch_flags_ref(*scratch_flags); - num_teams_done[0] = ++scratch_flags_ref; - } - item.barrier(sycl::access::fence_space::local_space); - if (num_teams_done[0] == n_wgroups) { - if (local_id >= n_wgroups) - final_reducer.init(&local_mem[local_id * value_count]); - else { - final_reducer.copy(&local_mem[local_id * value_count], - &results_ptr[local_id * value_count]); - for (unsigned int id = local_id + wgroup_size; id < n_wgroups; - id += wgroup_size) { - final_reducer.join(&local_mem[local_id * value_count], - &results_ptr[id * value_count]); - } - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), results_ptr, - device_accessible_result_ptr, value_count, final_reducer, - true, std::min(n_wgroups, wgroup_size)); - } - } else { - value_type local_value; - reference_type update = final_reducer.init(&local_value); - - Kokkos::Impl::Reduce::DeviceIterateTile< - Policy::rank, BarePolicy, FunctorType, - typename Policy::work_tag, reference_type>( - bare_policy, functor, update, - {n_global_x, n_global_y, n_global_z}, - {global_x, global_y, global_z}, {local_x, local_y, local_z}) - .exec_range(); - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), local_value, results_ptr, - device_accessible_result_ptr, final_reducer, false, - std::min(size, wgroup_size)); - - if (local_id == 0) { - sycl::atomic_ref - scratch_flags_ref(*scratch_flags); - num_teams_done[0] = ++scratch_flags_ref; - } - item.barrier(sycl::access::fence_space::local_space); - if (num_teams_done[0] == n_wgroups) { - if (local_id >= n_wgroups) - final_reducer.init(&local_value); - else { - local_value = results_ptr[local_id]; - for (unsigned int id = local_id + wgroup_size; id < n_wgroups; - id += wgroup_size) { - final_reducer.join(&local_value, &results_ptr[id]); - } - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), local_value, results_ptr, - device_accessible_result_ptr, final_reducer, true, - std::min(n_wgroups, wgroup_size)); - } - } - }); - }); - last_reduction_event = q.ext_oneapi_submit_barrier( - std::vector{parallel_reduce_event}); - } - - // At this point, the reduced value is written to the entry in results_ptr - // and all that is left is to copy it back to the given result pointer if - // necessary. - if (m_result_ptr && !m_result_ptr_device_accessible) { - Kokkos::Impl::DeepCopy( - m_space, m_result_ptr, results_ptr, - sizeof(*m_result_ptr) * value_count); - } - - return last_reduction_event; - } - - public: - template - static int max_tile_size_product(const Policy& policy, const Functor&) { - return policy.space().impl_internal_space_instance()->m_maxWorkgroupSize; - } - - void execute() const { - Kokkos::Experimental::Impl::SYCLInternal& instance = - *m_space.impl_internal_space_instance(); - using IndirectKernelMem = - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem; - IndirectKernelMem& indirectKernelMem = instance.get_indirect_kernel_mem(); - IndirectKernelMem& indirectReducerMem = instance.get_indirect_kernel_mem(); - - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); - auto reducer_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_reducer, indirectReducerMem); - - sycl::event event = sycl_direct_launch( - m_policy, functor_wrapper, reducer_wrapper, - {functor_wrapper.get_copy_event(), reducer_wrapper.get_copy_event()}); - functor_wrapper.register_event(event); - reducer_wrapper.register_event(event); - } - - private: - const FunctorType m_functor; - const BarePolicy m_policy; - const Kokkos::Experimental::SYCL& m_space; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const bool m_result_ptr_device_accessible; - - // Only let one Parallel/Scan modify the shared memory. The - // constructor acquires the mutex which is released in the destructor. - std::scoped_lock m_shared_memory_lock; -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif -#endif /* KOKKOS_SYCL_PARALLEL_REDUCE_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Scan.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Scan.hpp deleted file mode 100644 index e2afc97839..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Scan.hpp +++ /dev/null @@ -1,377 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKO_SYCL_PARALLEL_SCAN_HPP -#define KOKKO_SYCL_PARALLEL_SCAN_HPP - -#include -#include -#include -#if defined(KOKKOS_ENABLE_SYCL) - -namespace Kokkos { -namespace Impl { - -// Perform a scan over a workgroup. -// At the end of this function, the subgroup scans are stored in the local array -// such that the last value (at position n_active_subgroups-1) contains the -// total sum. -template -void workgroup_scan(sycl::nd_item item, const FunctorType& final_reducer, - sycl::local_ptr local_mem, - ValueType& local_value, unsigned int global_range) { - // subgroup scans - auto sg = item.get_sub_group(); - const auto sg_group_id = sg.get_group_id()[0]; - const auto id_in_sg = sg.get_local_id()[0]; - for (unsigned int stride = 1; stride < global_range; stride <<= 1) { - auto tmp = sg.shuffle_up(local_value, stride); - if (id_in_sg >= stride) final_reducer.join(&local_value, &tmp); - } - - const auto max_subgroup_size = sg.get_max_local_range()[0]; - const auto n_active_subgroups = - (global_range + max_subgroup_size - 1) / max_subgroup_size; - - const auto local_range = sg.get_local_range()[0]; - if (id_in_sg == local_range - 1 && sg_group_id < n_active_subgroups) - local_mem[sg_group_id] = local_value; - local_value = sg.shuffle_up(local_value, 1); - if (id_in_sg == 0) final_reducer.init(&local_value); - sycl::group_barrier(item.get_group()); - - // scan subgroup results using the first subgroup - if (n_active_subgroups > 1) { - if (sg_group_id == 0) { - const auto n_rounds = - (n_active_subgroups + local_range - 1) / local_range; - for (unsigned int round = 0; round < n_rounds; ++round) { - const unsigned int idx = id_in_sg + round * local_range; - const auto upper_bound = - std::min(local_range, n_active_subgroups - round * local_range); - auto local_sg_value = local_mem[idx < n_active_subgroups ? idx : 0]; - for (unsigned int stride = 1; stride < upper_bound; stride <<= 1) { - auto tmp = sg.shuffle_up(local_sg_value, stride); - if (id_in_sg >= stride) { - if (idx < n_active_subgroups) - final_reducer.join(&local_sg_value, &tmp); - else - local_sg_value = tmp; - } - } - if (idx < n_active_subgroups) { - local_mem[idx] = local_sg_value; - if (round > 0) - final_reducer.join(&local_mem[idx], - &local_mem[round * local_range - 1]); - } - if (round + 1 < n_rounds) sycl::group_barrier(sg); - } - } - sycl::group_barrier(item.get_group()); - } - - // add results to all subgroups - if (sg_group_id > 0) - final_reducer.join(&local_value, &local_mem[sg_group_id - 1]); -} - -template -class ParallelScanSYCLBase { - public: - using Policy = Kokkos::RangePolicy; - - protected: - using Member = typename Policy::member_type; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using LaunchBounds = typename Policy::launch_bounds; - - public: - using Analysis = - FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - using functor_type = FunctorType; - using size_type = Kokkos::Experimental::SYCL::size_type; - using index_type = typename Policy::index_type; - - protected: - const FunctorType m_functor; - const Policy m_policy; - pointer_type m_scratch_space = nullptr; - - // Only let one Parallel/Scan modify the shared memory. The - // constructor acquires the mutex which is released in the destructor. - std::scoped_lock m_shared_memory_lock; - - private: - template - void scan_internal(sycl::queue& q, const FunctorWrapper& functor_wrapper, - pointer_type global_mem, std::size_t size) const { - // FIXME_SYCL optimize - constexpr size_t wgroup_size = 128; - auto n_wgroups = (size + wgroup_size - 1) / wgroup_size; - pointer_type group_results = global_mem + n_wgroups * wgroup_size; - - auto local_scans = q.submit([&](sycl::handler& cgh) { - // Store subgroup totals - const auto min_subgroup_size = - q.get_device() - .template get_info() - .front(); - sycl::accessor - local_mem(sycl::range<1>((wgroup_size + min_subgroup_size - 1) / - min_subgroup_size), - cgh); - - cgh.parallel_for( - sycl::nd_range<1>(n_wgroups * wgroup_size, wgroup_size), - [=](sycl::nd_item<1> item) { - const FunctorType& functor = functor_wrapper.get_functor(); - typename Analysis::Reducer final_reducer(&functor); - - const auto local_id = item.get_local_linear_id(); - const auto global_id = item.get_global_linear_id(); - - // Initialize local memory - value_type local_value; - if (global_id < size) - local_value = global_mem[global_id]; - else - final_reducer.init(&local_value); - - workgroup_scan<>(item, final_reducer, local_mem.get_pointer(), - local_value, wgroup_size); - - if (n_wgroups > 1 && local_id == wgroup_size - 1) - group_results[item.get_group_linear_id()] = - local_mem[item.get_sub_group().get_group_range()[0] - 1]; - - // Write results to global memory - if (global_id < size) global_mem[global_id] = local_value; - }); - }); - q.ext_oneapi_submit_barrier(std::vector{local_scans}); - - if (n_wgroups > 1) { - scan_internal(q, functor_wrapper, group_results, n_wgroups); - auto update_with_group_results = q.submit([&](sycl::handler& cgh) { - cgh.parallel_for( - sycl::nd_range<1>(n_wgroups * wgroup_size, wgroup_size), - [=](sycl::nd_item<1> item) { - const auto global_id = item.get_global_linear_id(); - const FunctorType& functor = functor_wrapper.get_functor(); - typename Analysis::Reducer final_reducer(&functor); - if (global_id < size) - final_reducer.join(&global_mem[global_id], - &group_results[item.get_group_linear_id()]); - }); - }); - q.ext_oneapi_submit_barrier( - std::vector{update_with_group_results}); - } - } - - template - sycl::event sycl_direct_launch(const FunctorWrapper& functor_wrapper, - sycl::event memcpy_event) const { - // Convenience references - const Kokkos::Experimental::SYCL& space = m_policy.space(); - sycl::queue& q = space.sycl_queue(); - - const std::size_t len = m_policy.end() - m_policy.begin(); - - // Initialize global memory - auto initialize_global_memory = q.submit([&](sycl::handler& cgh) { - auto global_mem = m_scratch_space; - auto begin = m_policy.begin(); - - cgh.depends_on(memcpy_event); - cgh.parallel_for(sycl::range<1>(len), [=](sycl::item<1> item) { - const typename Policy::index_type id = - static_cast(item.get_id()) + begin; - const FunctorType& functor = functor_wrapper.get_functor(); - typename Analysis::Reducer final_reducer(&functor); - - value_type update{}; - final_reducer.init(&update); - if constexpr (std::is_void::value) - functor_wrapper.get_functor()(id, update, false); - else - functor_wrapper.get_functor()(WorkTag(), id, update, false); - global_mem[id] = update; - }); - }); - q.ext_oneapi_submit_barrier( - std::vector{initialize_global_memory}); - - // Perform the actual exclusive scan - scan_internal(q, functor_wrapper, m_scratch_space, len); - - // Write results to global memory - auto update_global_results = q.submit([&](sycl::handler& cgh) { - auto global_mem = m_scratch_space; - cgh.parallel_for(sycl::range<1>(len), [=](sycl::item<1> item) { - auto global_id = item.get_id(0); - - value_type update = global_mem[global_id]; - if constexpr (std::is_void::value) - functor_wrapper.get_functor()(global_id, update, true); - else - functor_wrapper.get_functor()(WorkTag(), global_id, update, true); - global_mem[global_id] = update; - }); - }); - q.ext_oneapi_submit_barrier( - std::vector{update_global_results}); - return update_global_results; - } - - public: - template - void impl_execute(const PostFunctor& post_functor) { - if (m_policy.begin() == m_policy.end()) return; - - auto& instance = *m_policy.space().impl_internal_space_instance(); - const std::size_t len = m_policy.end() - m_policy.begin(); - - // Compute the total amount of memory we will need. We emulate the recursive - // structure that is used to do the actual scan. Essentially, we need to - // allocate memory for the whole range and then recursively for the reduced - // group results until only one group is left. - std::size_t total_memory = 0; - { - size_t wgroup_size = 128; - size_t n_nested_size = len; - size_t n_nested_wgroups; - do { - n_nested_wgroups = (n_nested_size + wgroup_size - 1) / wgroup_size; - n_nested_size = n_nested_wgroups; - total_memory += sizeof(value_type) * n_nested_wgroups * wgroup_size; - } while (n_nested_wgroups > 1); - total_memory += sizeof(value_type) * wgroup_size; - } - - // FIXME_SYCL consider only storing one value per block and recreate initial - // results in the end before doing the final pass - m_scratch_space = static_cast>( - instance.scratch_space(total_memory)); - - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem& - indirectKernelMem = instance.get_indirect_kernel_mem(); - - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); - - sycl::event event = - sycl_direct_launch(functor_wrapper, functor_wrapper.get_copy_event()); - functor_wrapper.register_event(event); - post_functor(); - } - - ParallelScanSYCLBase(const FunctorType& arg_functor, const Policy& arg_policy) - : m_functor(arg_functor), - m_policy(arg_policy), - m_shared_memory_lock(m_policy.space() - .impl_internal_space_instance() - ->m_mutexScratchSpace) {} -}; - -template -class ParallelScan, - Kokkos::Experimental::SYCL> - : private ParallelScanSYCLBase { - public: - using Base = ParallelScanSYCLBase; - - inline void execute() { - Base::impl_execute([]() {}); - } - - ParallelScan(const FunctorType& arg_functor, - const typename Base::Policy& arg_policy) - : Base(arg_functor, arg_policy) {} -}; - -//---------------------------------------------------------------------------- - -template -class ParallelScanWithTotal, - ReturnType, Kokkos::Experimental::SYCL> - : private ParallelScanSYCLBase { - public: - using Base = ParallelScanSYCLBase; - - ReturnType& m_returnvalue; - const Kokkos::Experimental::SYCL& m_exec; - - inline void execute() { - Base::impl_execute([&]() { - const long long nwork = Base::m_policy.end() - Base::m_policy.begin(); - if (nwork > 0) { - const int size = Base::Analysis::value_size(Base::m_functor); - DeepCopy( - m_exec, &m_returnvalue, Base::m_scratch_space + nwork - 1, size); - } - }); - } - - ParallelScanWithTotal(const FunctorType& arg_functor, - const typename Base::Policy& arg_policy, - ReturnType& arg_returnvalue) - : Base(arg_functor, arg_policy), - m_returnvalue(arg_returnvalue), - m_exec(arg_policy.space()) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Team.hpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Team.hpp deleted file mode 100644 index 5ac7d8af30..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Parallel_Team.hpp +++ /dev/null @@ -1,982 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCL_PARALLEL_TEAM_HPP -#define KOKKOS_SYCL_PARALLEL_TEAM_HPP - -#include - -#include // workgroup_reduction -#include - -#include - -namespace Kokkos { -namespace Impl { -template -class TeamPolicyInternal - : public PolicyTraits { - public: - using execution_policy = TeamPolicyInternal; - - using traits = PolicyTraits; - - template - friend class TeamPolicyInternal; - - private: - typename traits::execution_space m_space; - int m_league_size; - int m_team_size; - int m_vector_length; - size_t m_team_scratch_size[2]; - size_t m_thread_scratch_size[2]; - int m_chunk_size; - bool m_tune_team_size; - bool m_tune_vector_length; - - public: - using execution_space = Kokkos::Experimental::SYCL; - - template - TeamPolicyInternal(TeamPolicyInternal const& p) { - m_league_size = p.m_league_size; - m_team_size = p.m_team_size; - m_vector_length = p.m_vector_length; - m_team_scratch_size[0] = p.m_team_scratch_size[0]; - m_team_scratch_size[1] = p.m_team_scratch_size[1]; - m_thread_scratch_size[0] = p.m_thread_scratch_size[0]; - m_thread_scratch_size[1] = p.m_thread_scratch_size[1]; - m_chunk_size = p.m_chunk_size; - m_space = p.m_space; - m_tune_team_size = p.m_tune_team_size; - m_tune_vector_length = p.m_tune_vector_length; - } - - template - int team_size_max(FunctorType const& f, ParallelForTag const&) const { - return internal_team_size_max_for(f); - } - - template - inline int team_size_max(const FunctorType& f, - const ParallelReduceTag&) const { - return internal_team_size_max_reduce(f); - } - - template - inline int team_size_max(const FunctorType& f, const ReducerType& /*r*/, - const ParallelReduceTag&) const { - return internal_team_size_max_reduce(f); - } - - template - int team_size_recommended(FunctorType const& f, ParallelForTag const&) const { - return internal_team_size_recommended_for(f); - } - - template - inline int team_size_recommended(FunctorType const& f, - ParallelReduceTag const&) const { - return internal_team_size_recommended_reduce(f); - } - - template - int team_size_recommended(FunctorType const& f, ReducerType const&, - ParallelReduceTag const&) const { - return internal_team_size_recommended_reduce(f); - } - inline bool impl_auto_vector_length() const { return m_tune_vector_length; } - inline bool impl_auto_team_size() const { return m_tune_team_size; } - // FIXME_SYCL This is correct in most cases, but not necessarily in case a - // custom sycl::queue is used to initialize the execution space. - static int vector_length_max() { - std::vector sub_group_sizes = - execution_space{} - .impl_internal_space_instance() - ->m_queue->get_device() - .template get_info(); - return *std::max_element(sub_group_sizes.begin(), sub_group_sizes.end()); - } - - private: - static int verify_requested_vector_length(int requested_vector_length) { - int test_vector_length = - std::min(requested_vector_length, vector_length_max()); - - // Allow only power-of-two vector_length - if (!(is_integral_power_of_two(test_vector_length))) { - int test_pow2 = 1; - while (test_pow2 < test_vector_length) test_pow2 <<= 1; - test_vector_length = test_pow2 >> 1; - } - - return test_vector_length; - } - - public: - static int scratch_size_max(int level) { - return level == 0 ? 1024 * 32 - : // FIXME_SYCL arbitrarily setting this to 32kB - 20 * 1024 * 1024; // FIXME_SYCL arbitrarily setting this to 20MB - } - inline void impl_set_vector_length(size_t size) { m_vector_length = size; } - inline void impl_set_team_size(size_t size) { m_team_size = size; } - int impl_vector_length() const { return m_vector_length; } -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - KOKKOS_DEPRECATED int vector_length() const { return impl_vector_length(); } -#endif - - int team_size() const { return m_team_size; } - - int league_size() const { return m_league_size; } - - size_t scratch_size(int level, int team_size_ = -1) const { - if (team_size_ < 0) team_size_ = m_team_size; - return m_team_scratch_size[level] + - team_size_ * m_thread_scratch_size[level]; - } - - size_t team_scratch_size(int level) const { - return m_team_scratch_size[level]; - } - - size_t thread_scratch_size(int level) const { - return m_thread_scratch_size[level]; - } - - typename traits::execution_space space() const { return m_space; } - - TeamPolicyInternal() - : m_space(typename traits::execution_space()), - m_league_size(0), - m_team_size(-1), - m_vector_length(0), - m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(vector_length_max()), - m_tune_team_size(false), - m_tune_vector_length(false) {} - - /** \brief Specify league size, request team size */ - TeamPolicyInternal(const execution_space space_, int league_size_, - int team_size_request, int vector_length_request = 1) - : m_space(space_), - m_league_size(league_size_), - m_team_size(team_size_request), - m_vector_length( - (vector_length_request > 0) - ? verify_requested_vector_length(vector_length_request) - : (verify_requested_vector_length(1))), - m_team_scratch_size{0, 0}, - m_thread_scratch_size{0, 0}, - m_chunk_size(vector_length_max()), - m_tune_team_size(bool(team_size_request <= 0)), - m_tune_vector_length(bool(vector_length_request <= 0)) { - // FIXME_SYCL Check that league size is permissible, - // https://github.com/intel/llvm/pull/4064 - - // Make sure total block size is permissible - if (m_team_size * m_vector_length > - static_cast( - m_space.impl_internal_space_instance()->m_maxWorkgroupSize)) { - Impl::throw_runtime_exception( - std::string("Kokkos::TeamPolicy the team size is too large. " - "Team size x vector length is " + - std::to_string(m_team_size * m_vector_length) + - " but must be smaller than ") + - std::to_string( - m_space.impl_internal_space_instance()->m_maxWorkgroupSize)); - } - } - - /** \brief Specify league size, request team size */ - TeamPolicyInternal(const execution_space space_, int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - int vector_length_request = 1) - : TeamPolicyInternal(space_, league_size_, -1, vector_length_request) {} - // FLAG - /** \brief Specify league size and team size, request vector length*/ - TeamPolicyInternal(const execution_space space_, int league_size_, - int team_size_request, - const Kokkos::AUTO_t& /* vector_length_request */ - ) - : TeamPolicyInternal(space_, league_size_, team_size_request, -1) - - {} - - /** \brief Specify league size, request team size and vector length*/ - TeamPolicyInternal(const execution_space space_, int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - const Kokkos::AUTO_t& /* vector_length_request */ - - ) - : TeamPolicyInternal(space_, league_size_, -1, -1) - - {} - - TeamPolicyInternal(int league_size_, int team_size_request, - int vector_length_request = 1) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, - team_size_request, vector_length_request) {} - - TeamPolicyInternal(int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - int vector_length_request = 1) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, -1, - vector_length_request) {} - - /** \brief Specify league size and team size, request vector length*/ - TeamPolicyInternal(int league_size_, int team_size_request, - const Kokkos::AUTO_t& /* vector_length_request */ - - ) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, - team_size_request, -1) - - {} - - /** \brief Specify league size, request team size and vector length*/ - TeamPolicyInternal(int league_size_, - const Kokkos::AUTO_t& /* team_size_request */, - const Kokkos::AUTO_t& /* vector_length_request */ - - ) - : TeamPolicyInternal(typename traits::execution_space(), league_size_, -1, - -1) {} - - int chunk_size() const { return m_chunk_size; } - - TeamPolicyInternal& set_chunk_size(typename traits::index_type chunk_size_) { - m_chunk_size = chunk_size_; - return *this; - } - - /** \brief set per team scratch size for a specific level of the scratch - * hierarchy */ - TeamPolicyInternal& set_scratch_size(int level, - PerTeamValue const& per_team) { - m_team_scratch_size[level] = per_team.value; - return *this; - } - - /** \brief set per thread scratch size for a specific level of the scratch - * hierarchy */ - TeamPolicyInternal& set_scratch_size(int level, - PerThreadValue const& per_thread) { - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - /** \brief set per thread and per team scratch size for a specific level of - * the scratch hierarchy */ - TeamPolicyInternal& set_scratch_size(int level, PerTeamValue const& per_team, - PerThreadValue const& per_thread) { - m_team_scratch_size[level] = per_team.value; - m_thread_scratch_size[level] = per_thread.value; - return *this; - } - - using member_type = Kokkos::Impl::SYCLTeamMember; - - protected: - template - int internal_team_size_max_for(const FunctorType& /*f*/) const { - // nested_reducer_memsize = (sizeof(double) * (m_team_size + 2) - // custom: m_team_scratch_size[0] + m_thread_scratch_size[0] * m_team_size - // total: - // 2*sizeof(double)+m_team_scratch_size[0] - // + m_team_size(sizeof(double)+m_thread_scratch_size[0]) - const int max_threads_for_memory = - (space().impl_internal_space_instance()->m_maxShmemPerBlock - - 2 * sizeof(double) - m_team_scratch_size[0]) / - (sizeof(double) + m_thread_scratch_size[0]); - return std::min({ - int(m_space.impl_internal_space_instance()->m_maxWorkgroupSize), - // FIXME_SYCL Avoid requesting to many registers on NVIDIA GPUs. -#if defined(KOKKOS_ARCH_KEPLER) || defined(KOKKOS_ARCH_MAXWELL) || \ - defined(KOKKOS_ARCH_PASCAL) || defined(KOKKOS_ARCH_VOLTA) || \ - defined(KOKKOS_ARCH_TURING75) || defined(KOKKOS_ARCH_AMPERE) - 256, -#endif - max_threads_for_memory - }) / - impl_vector_length(); - } - - template - int internal_team_size_max_reduce(const FunctorType& f) const { - using Analysis = FunctorAnalysis; - using value_type = typename Analysis::value_type; - const int value_count = Analysis::value_count(f); - - // nested_reducer_memsize = (sizeof(double) * (m_team_size + 2) - // reducer_memsize = sizeof(value_type) * m_team_size * value_count - // custom: m_team_scratch_size[0] + m_thread_scratch_size[0] * m_team_size - // total: - // 2*sizeof(double)+m_team_scratch_size[0] - // + m_team_size(sizeof(double)+sizeof(value_type)*value_count - // +m_thread_scratch_size[0]) - const int max_threads_for_memory = - (space().impl_internal_space_instance()->m_maxShmemPerBlock - - 2 * sizeof(double) - m_team_scratch_size[0]) / - (sizeof(double) + sizeof(value_type) * value_count + - m_thread_scratch_size[0]); - return std::min({ - int(m_space.impl_internal_space_instance()->m_maxWorkgroupSize), - // FIXME_SYCL Avoid requesting to many registers on NVIDIA GPUs. -#if defined(KOKKOS_ARCH_KEPLER) || defined(KOKKOS_ARCH_MAXWELL) || \ - defined(KOKKOS_ARCH_PASCAL) || defined(KOKKOS_ARCH_VOLTA) || \ - defined(KOKKOS_ARCH_TURING75) || defined(KOKKOS_ARCH_AMPERE) - 256, -#endif - max_threads_for_memory - }) / - impl_vector_length(); - } - - template - int internal_team_size_recommended_for(const FunctorType& f) const { - // FIXME_SYCL improve - return 1 << Kokkos::Impl::int_log2(internal_team_size_max_for(f)); - } - - template - int internal_team_size_recommended_reduce(const FunctorType& f) const { - // FIXME_SYCL improve - return 1 << Kokkos::Impl::int_log2(internal_team_size_max_reduce(f)); - } -}; - -template -class ParallelFor, - Kokkos::Experimental::SYCL> { - public: - using Policy = TeamPolicyInternal; - using functor_type = FunctorType; - using size_type = ::Kokkos::Experimental::SYCL::size_type; - - private: - using member_type = typename Policy::member_type; - using work_tag = typename Policy::work_tag; - using launch_bounds = typename Policy::launch_bounds; - - FunctorType const m_functor; - Policy const m_policy; - size_type const m_league_size; - int m_team_size; - size_type const m_vector_size; - int m_shmem_begin; - int m_shmem_size; - sycl::device_ptr m_global_scratch_ptr; - size_t m_scratch_size[2]; - // Only let one ParallelFor/Reduce modify the team scratch memory. The - // constructor acquires the mutex which is released in the destructor. - std::scoped_lock m_scratch_lock; - - template - sycl::event sycl_direct_launch(const Policy& policy, - const FunctorWrapper& functor_wrapper, - const sycl::event& memcpy_events) const { - // Convenience references - const Kokkos::Experimental::SYCL& space = policy.space(); - sycl::queue& q = space.sycl_queue(); - - auto parallel_for_event = q.submit([&](sycl::handler& cgh) { - // FIXME_SYCL accessors seem to need a size greater than zero at least for - // host queues - sycl::accessor - team_scratch_memory_L0( - sycl::range<1>( - std::max(m_scratch_size[0] + m_shmem_begin, size_t(1))), - cgh); - - // Avoid capturing *this since it might not be trivially copyable - const auto shmem_begin = m_shmem_begin; - const size_t scratch_size[2] = {m_scratch_size[0], m_scratch_size[1]}; - sycl::device_ptr const global_scratch_ptr = m_global_scratch_ptr; - - auto lambda = [=](sycl::nd_item<2> item) { - const member_type team_member( - team_scratch_memory_L0.get_pointer(), shmem_begin, scratch_size[0], - global_scratch_ptr + item.get_group(1) * scratch_size[1], - scratch_size[1], item); - if constexpr (std::is_void::value) - functor_wrapper.get_functor()(team_member); - else - functor_wrapper.get_functor()(work_tag(), team_member); - }; - - static sycl::kernel kernel = [&] { - sycl::kernel_id functor_kernel_id = - sycl::get_kernel_id(); - auto kernel_bundle = - sycl::get_kernel_bundle( - q.get_context(), std::vector{functor_kernel_id}); - return kernel_bundle.get_kernel(functor_kernel_id); - }(); - auto max_sg_size = - kernel - .get_info( - q.get_device(), - sycl::range<3>(m_team_size, m_vector_size, 1)); - auto final_vector_size = std::min(m_vector_size, max_sg_size); - // FIXME_SYCL For some reason, explicitly enforcing the kernel bundle to - // be used gives a runtime error. - // cgh.use_kernel_bundle(kernel_bundle); - - cgh.depends_on(memcpy_events); - cgh.parallel_for( - sycl::nd_range<2>( - sycl::range<2>(m_team_size, m_league_size * final_vector_size), - sycl::range<2>(m_team_size, final_vector_size)), - lambda); - }); - q.ext_oneapi_submit_barrier(std::vector{parallel_for_event}); - return parallel_for_event; - } - - public: - inline void execute() const { - if (m_league_size == 0) return; - - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem& - indirectKernelMem = m_policy.space() - .impl_internal_space_instance() - ->get_indirect_kernel_mem(); - - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); - - sycl::event event = sycl_direct_launch(m_policy, functor_wrapper, - functor_wrapper.get_copy_event()); - functor_wrapper.register_event(event); - } - - ParallelFor(FunctorType const& arg_functor, Policy const& arg_policy) - : m_functor(arg_functor), - m_policy(arg_policy), - m_league_size(arg_policy.league_size()), - m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()), - m_scratch_lock(arg_policy.space() - .impl_internal_space_instance() - ->m_team_scratch_mutex) { - // FIXME_SYCL optimize - if (m_team_size < 0) - m_team_size = - m_policy.team_size_recommended(arg_functor, ParallelForTag{}); - - m_shmem_begin = (sizeof(double) * (m_team_size + 2)); - m_shmem_size = - (m_policy.scratch_size(0, m_team_size) + - FunctorTeamShmemSize::value(m_functor, m_team_size)); - m_scratch_size[0] = m_shmem_size; - m_scratch_size[1] = m_policy.scratch_size(1, m_team_size); - - // Functor's reduce memory, team scan memory, and team shared memory depend - // upon team size. - auto& space = *m_policy.space().impl_internal_space_instance(); - m_global_scratch_ptr = - static_cast>(space.resize_team_scratch_space( - static_cast(m_scratch_size[1]) * m_league_size)); - - if (static_cast(space.m_maxShmemPerBlock) < - m_shmem_size - m_shmem_begin) { - std::stringstream out; - out << "Kokkos::Impl::ParallelFor insufficient shared memory! " - "Requested " - << m_shmem_size - m_shmem_begin << " bytes but maximum is " - << space.m_maxShmemPerBlock << '\n'; - Kokkos::Impl::throw_runtime_exception(out.str()); - } - - const auto max_team_size = - m_policy.team_size_max(arg_functor, ParallelForTag{}); - if (m_team_size > m_policy.team_size_max(arg_functor, ParallelForTag{})) - Kokkos::Impl::throw_runtime_exception( - "Kokkos::Impl::ParallelFor requested too large team size. The " - "maximal team_size is " + - std::to_string(max_team_size) + '!'); - } -}; - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -class ParallelReduce, - ReducerType, Kokkos::Experimental::SYCL> { - public: - using Policy = TeamPolicyInternal; - - private: - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using Analysis = - FunctorAnalysis; - using member_type = typename Policy::member_type; - using WorkTag = typename Policy::work_tag; - using launch_bounds = typename Policy::launch_bounds; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - using value_type = typename Analysis::value_type; - - public: - using functor_type = FunctorType; - using size_type = Kokkos::Experimental::SYCL::size_type; - - private: - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const bool m_result_ptr_device_accessible; - size_type m_shmem_begin; - size_type m_shmem_size; - sycl::device_ptr m_global_scratch_ptr; - size_t m_scratch_size[2]; - const size_type m_league_size; - int m_team_size; - const size_type m_vector_size; - // Only let one ParallelFor/Reduce modify the team scratch memory. The - // constructor acquires the mutex which is released in the destructor. - std::scoped_lock m_scratch_lock; - - template - sycl::event sycl_direct_launch( - const PolicyType& policy, const FunctorWrapper& functor_wrapper, - const ReducerWrapper& reducer_wrapper, - const std::vector& memcpy_events) const { - // Convenience references - const Kokkos::Experimental::SYCL& space = policy.space(); - Kokkos::Experimental::Impl::SYCLInternal& instance = - *space.impl_internal_space_instance(); - sycl::queue& q = space.sycl_queue(); - - const unsigned int value_count = - Analysis::value_count(ReducerConditional::select(m_functor, m_reducer)); - std::size_t size = std::size_t(m_league_size) * m_team_size * m_vector_size; - value_type* results_ptr = nullptr; - - sycl::event last_reduction_event; - - // If size<=1 we only call init(), the functor and possibly final once - // working with the global scratch memory but don't copy back to - // m_result_ptr yet. - if (size <= 1) { - results_ptr = - static_cast>(instance.scratch_space( - sizeof(value_type) * std::max(value_count, 1u))); - sycl::global_ptr device_accessible_result_ptr = - m_result_ptr_device_accessible ? m_result_ptr : nullptr; - - auto parallel_reduce_event = q.submit([&](sycl::handler& cgh) { - // FIXME_SYCL accessors seem to need a size greater than zero at least - // for host queues - sycl::accessor - team_scratch_memory_L0( - sycl::range<1>( - std::max(m_scratch_size[0] + m_shmem_begin, size_t(1))), - cgh); - - // Avoid capturing *this since it might not be trivially copyable - const auto shmem_begin = m_shmem_begin; - const size_t scratch_size[2] = {m_scratch_size[0], m_scratch_size[1]}; - sycl::device_ptr const global_scratch_ptr = m_global_scratch_ptr; - - cgh.depends_on(memcpy_events); - cgh.parallel_for( - sycl::nd_range<2>(sycl::range<2>(1, 1), sycl::range<2>(1, 1)), - [=](sycl::nd_item<2> item) { - const auto& functor = functor_wrapper.get_functor(); - const auto& selected_reducer = ReducerConditional::select( - static_cast(functor), - static_cast( - reducer_wrapper.get_functor())); - typename Analysis::Reducer final_reducer(&selected_reducer); - - reference_type update = final_reducer.init(results_ptr); - if (size == 1) { - const member_type team_member( - team_scratch_memory_L0.get_pointer(), shmem_begin, - scratch_size[0], global_scratch_ptr, scratch_size[1], item); - if constexpr (std::is_void::value) - functor(team_member, update); - else - functor(WorkTag(), team_member, update); - } - final_reducer.final(results_ptr); - if (device_accessible_result_ptr) - final_reducer.copy(device_accessible_result_ptr, - &results_ptr[0]); - }); - }); - q.ext_oneapi_submit_barrier( - std::vector{parallel_reduce_event}); - last_reduction_event = parallel_reduce_event; - } else { - // Otherwise, (if the total range has more than one element) we perform a - // reduction on the values in all workgroups separately, write the - // workgroup results back to global memory and recurse until only one - // workgroup does the reduction and thus gets the final value. - auto parallel_reduce_event = q.submit([&](sycl::handler& cgh) { - auto scratch_flags = static_cast>( - instance.scratch_flags(sizeof(unsigned int))); - - // FIXME_SYCL accessors seem to need a size greater than zero at least - // for host queues - sycl::accessor - team_scratch_memory_L0( - sycl::range<1>( - std::max(m_scratch_size[0] + m_shmem_begin, size_t(1))), - cgh); - - // Avoid capturing *this since it might not be trivially copyable - const auto shmem_begin = m_shmem_begin; - const size_t scratch_size[2] = {m_scratch_size[0], m_scratch_size[1]}; - sycl::device_ptr const global_scratch_ptr = m_global_scratch_ptr; - - auto team_reduction_factory = - [&](sycl::accessor - local_mem, - sycl::device_ptr results_ptr) mutable { - sycl::global_ptr device_accessible_result_ptr = - m_result_ptr_device_accessible ? m_result_ptr : nullptr; - auto lambda = [=](sycl::nd_item<2> item) { - auto n_wgroups = - item.get_group_range()[0] * item.get_group_range()[1]; - auto wgroup_size = - item.get_local_range()[0] * item.get_local_range()[1]; - auto size = n_wgroups * wgroup_size; - - auto& num_teams_done = reinterpret_cast( - local_mem[wgroup_size * std::max(value_count, 1u)]); - const auto local_id = item.get_local_linear_id(); - const auto& functor = functor_wrapper.get_functor(); - const auto& selected_reducer = ReducerConditional::select( - static_cast(functor), - static_cast( - reducer_wrapper.get_functor())); - typename Analysis::Reducer final_reducer(&selected_reducer); - - if constexpr (Analysis::StaticValueSize == 0) { - reference_type update = - final_reducer.init(&local_mem[local_id * value_count]); - const member_type team_member( - team_scratch_memory_L0.get_pointer(), shmem_begin, - scratch_size[0], - global_scratch_ptr + item.get_group(1) * scratch_size[1], - scratch_size[1], item); - if constexpr (std::is_void::value) - functor(team_member, update); - else - functor(WorkTag(), team_member, update); - item.barrier(sycl::access::fence_space::local_space); - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), results_ptr, - device_accessible_result_ptr, value_count, - selected_reducer, false, - std::min(size, - item.get_local_range()[0] * - item.get_local_range()[1])); - - if (local_id == 0) { - sycl::atomic_ref - scratch_flags_ref(*scratch_flags); - num_teams_done = ++scratch_flags_ref; - } - sycl::group_barrier(item.get_group()); - if (num_teams_done == n_wgroups) { - if (local_id >= n_wgroups) - final_reducer.init(&local_mem[local_id * value_count]); - else { - final_reducer.copy(&local_mem[local_id * value_count], - &results_ptr[local_id * value_count]); - for (unsigned int id = local_id + wgroup_size; - id < n_wgroups; id += wgroup_size) { - final_reducer.join(&local_mem[local_id * value_count], - &results_ptr[id * value_count]); - } - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), results_ptr, - device_accessible_result_ptr, value_count, - selected_reducer, true, - std::min(n_wgroups, item.get_local_range()[0] * - item.get_local_range()[1])); - } - } else { - value_type local_value; - reference_type update = final_reducer.init(&local_value); - const member_type team_member( - team_scratch_memory_L0.get_pointer(), shmem_begin, - scratch_size[0], - global_scratch_ptr + item.get_group(1) * scratch_size[1], - scratch_size[1], item); - if constexpr (std::is_void::value) - functor(team_member, update); - else - functor(WorkTag(), team_member, update); - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), local_value, results_ptr, - device_accessible_result_ptr, final_reducer, false, - std::min(size, - item.get_local_range()[0] * - item.get_local_range()[1])); - - if (local_id == 0) { - sycl::atomic_ref - scratch_flags_ref(*scratch_flags); - num_teams_done = ++scratch_flags_ref; - } - item.barrier(sycl::access::fence_space::local_space); - if (num_teams_done == n_wgroups) { - if (local_id >= n_wgroups) - final_reducer.init(&local_value); - else { - local_value = results_ptr[local_id]; - for (unsigned int id = local_id + wgroup_size; - id < n_wgroups; id += wgroup_size) { - final_reducer.join(&local_value, &results_ptr[id]); - } - } - - SYCLReduction::workgroup_reduction<>( - item, local_mem.get_pointer(), local_value, results_ptr, - device_accessible_result_ptr, final_reducer, true, - std::min(n_wgroups, item.get_local_range()[0] * - item.get_local_range()[1])); - } - } - }; - return lambda; - }; - - auto dummy_reduction_lambda = team_reduction_factory({1, cgh}, nullptr); - - static sycl::kernel kernel = [&] { - sycl::kernel_id functor_kernel_id = - sycl::get_kernel_id(); - auto kernel_bundle = - sycl::get_kernel_bundle( - q.get_context(), std::vector{functor_kernel_id}); - return kernel_bundle.get_kernel(functor_kernel_id); - }(); - auto max_sg_size = kernel.get_info< - sycl::info::kernel_device_specific::max_sub_group_size>( - q.get_device(), sycl::range<3>(m_team_size, m_vector_size, 1)); - auto final_vector_size = std::min(m_vector_size, max_sg_size); - // FIXME_SYCL For some reason, explicitly enforcing the kernel bundle to - // be used gives a runtime error. - - // cgh.use_kernel_bundle(kernel_bundle); - - auto wgroup_size = m_team_size * final_vector_size; - std::size_t size = std::size_t(m_league_size) * wgroup_size; - sycl::accessor - local_mem(sycl::range<1>(wgroup_size) * std::max(value_count, 1u) + - (sizeof(unsigned int) + sizeof(value_type) - 1) / - sizeof(value_type), - cgh); - - const auto init_size = - std::max((size + wgroup_size - 1) / wgroup_size, 1); - results_ptr = - static_cast>(instance.scratch_space( - sizeof(value_type) * std::max(value_count, 1u) * init_size)); - - auto reduction_lambda = team_reduction_factory(local_mem, results_ptr); - - cgh.depends_on(memcpy_events); - - cgh.parallel_for( - sycl::nd_range<2>( - sycl::range<2>(m_team_size, m_league_size * m_vector_size), - sycl::range<2>(m_team_size, m_vector_size)), - reduction_lambda); - }); - last_reduction_event = q.ext_oneapi_submit_barrier( - std::vector{parallel_reduce_event}); - } - - // At this point, the reduced value is written to the entry in results_ptr - // and all that is left is to copy it back to the given result pointer if - // necessary. - if (m_result_ptr && !m_result_ptr_device_accessible) { - Kokkos::Impl::DeepCopy( - space, m_result_ptr, results_ptr, - sizeof(*m_result_ptr) * value_count); - } - - return last_reduction_event; - } - - public: - inline void execute() { - Kokkos::Experimental::Impl::SYCLInternal& instance = - *m_policy.space().impl_internal_space_instance(); - using IndirectKernelMem = - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem; - IndirectKernelMem& indirectKernelMem = instance.get_indirect_kernel_mem(); - IndirectKernelMem& indirectReducerMem = instance.get_indirect_kernel_mem(); - - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); - auto reducer_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_reducer, indirectReducerMem); - - sycl::event event = sycl_direct_launch( - m_policy, functor_wrapper, reducer_wrapper, - {functor_wrapper.get_copy_event(), reducer_wrapper.get_copy_event()}); - functor_wrapper.register_event(event); - reducer_wrapper.register_event(event); - } - - private: - void initialize() { - // FIXME_SYCL optimize - if (m_team_size < 0) - m_team_size = - m_policy.team_size_recommended(m_functor, ParallelReduceTag{}); - // Must be a power of two greater than two, get the one not bigger than the - // requested one. - if ((m_team_size & m_team_size - 1) || m_team_size < 2) { - int temp_team_size = 2; - while ((temp_team_size << 1) < m_team_size) temp_team_size <<= 1; - m_team_size = temp_team_size; - } - - m_shmem_begin = (sizeof(double) * (m_team_size + 2)); - m_shmem_size = - (m_policy.scratch_size(0, m_team_size) + - FunctorTeamShmemSize::value(m_functor, m_team_size)); - m_scratch_size[0] = m_shmem_size; - m_scratch_size[1] = m_policy.scratch_size(1, m_team_size); - - // Functor's reduce memory, team scan memory, and team shared memory depend - // upon team size. - auto& space = *m_policy.space().impl_internal_space_instance(); - m_global_scratch_ptr = - static_cast>(space.resize_team_scratch_space( - static_cast(m_scratch_size[1]) * m_league_size)); - - if (static_cast(space.m_maxShmemPerBlock) < - m_shmem_size - m_shmem_begin) { - std::stringstream out; - out << "Kokkos::Impl::ParallelFor insufficient shared memory! " - "Requested " - << m_shmem_size - m_shmem_begin << " bytes but maximum is " - << space.m_maxShmemPerBlock << '\n'; - Kokkos::Impl::throw_runtime_exception(out.str()); - } - - if (m_team_size > m_policy.team_size_max(m_functor, ParallelReduceTag{})) - Kokkos::Impl::throw_runtime_exception( - "Kokkos::Impl::ParallelFor requested too large team size."); - } - - public: - template - ParallelReduce( - FunctorType const& arg_functor, Policy const& arg_policy, - ViewType const& arg_result, - std::enable_if_t::value, void*> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_league_size(arg_policy.league_size()), - m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()), - m_scratch_lock(arg_policy.space() - .impl_internal_space_instance() - ->m_team_scratch_mutex) { - initialize(); - } - - ParallelReduce(FunctorType const& arg_functor, Policy const& arg_policy, - ReducerType const& reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_result_ptr_device_accessible( - MemorySpaceAccess::accessible), - m_league_size(arg_policy.league_size()), - m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()), - m_scratch_lock(arg_policy.space() - .impl_internal_space_instance() - ->m_team_scratch_mutex) { - initialize(); - } -}; -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Space.cpp b/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Space.cpp deleted file mode 100644 index 07ca907fa5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/SYCL/Kokkos_SYCL_Space.cpp +++ /dev/null @@ -1,498 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include -#include -#include -#include -#include - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ -namespace Kokkos { -namespace Impl { - -void DeepCopySYCL(void* dst, const void* src, size_t n) { - Experimental::Impl::SYCLInternal::singleton().m_queue->memcpy(dst, src, n); -} - -void DeepCopyAsyncSYCL(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { - // FIXME_SYCL memcpy doesn't respect submit_barrier which means that we need - // to actually fence the execution space to make sure the memcpy is properly - // enqueued when using out-of-order queues. - sycl::queue& q = *instance.impl_internal_space_instance()->m_queue; - q.wait_and_throw(); - auto event = q.memcpy(dst, src, n); - q.ext_oneapi_submit_barrier(std::vector{event}); -} - -void DeepCopyAsyncSYCL(void* dst, const void* src, size_t n) { - Experimental::Impl::SYCLInternal::singleton().m_queue->memcpy(dst, src, n); - Experimental::SYCL().fence( - "Kokkos::Impl::DeepCopyAsyncSYCL: fence after memcpy"); -} - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Experimental { - -SYCLDeviceUSMSpace::SYCLDeviceUSMSpace() - : m_queue(*SYCL().impl_internal_space_instance()->m_queue) {} -SYCLDeviceUSMSpace::SYCLDeviceUSMSpace(sycl::queue queue) - : m_queue(std::move(queue)) {} - -SYCLSharedUSMSpace::SYCLSharedUSMSpace() - : m_queue(*SYCL().impl_internal_space_instance()->m_queue) {} -SYCLSharedUSMSpace::SYCLSharedUSMSpace(sycl::queue queue) - : m_queue(std::move(queue)) {} - -SYCLHostUSMSpace::SYCLHostUSMSpace() - : m_queue(*SYCL().impl_internal_space_instance()->m_queue) {} -SYCLHostUSMSpace::SYCLHostUSMSpace(sycl::queue queue) - : m_queue(std::move(queue)) {} - -void* allocate_sycl( - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, const Kokkos::Tools::SpaceHandle arg_handle, - const RawMemoryAllocationFailure::AllocationMechanism failure_tag, - const sycl::usm::alloc allocation_kind, const sycl::queue& queue) { - void* const hostPtr = sycl::malloc(arg_alloc_size, queue, allocation_kind); - - if (hostPtr == nullptr) - throw RawMemoryAllocationFailure( - arg_alloc_size, 1, RawMemoryAllocationFailure::FailureMode::Unknown, - failure_tag); - - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, hostPtr, - reported_size); - } - - return hostPtr; -} - -void* SYCLDeviceUSMSpace::allocate(const Kokkos::Experimental::SYCL& exec_space, - const size_t arg_alloc_size) const { - return allocate(exec_space, "[unlabeled]", arg_alloc_size); -} - -void* SYCLDeviceUSMSpace::allocate(const Kokkos::Experimental::SYCL& exec_space, - const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return allocate_sycl( - arg_label, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), - RawMemoryAllocationFailure::AllocationMechanism::SYCLMallocDevice, - sycl::usm::alloc::device, - *exec_space.impl_internal_space_instance()->m_queue); -} - -void* SYCLDeviceUSMSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} - -void* SYCLDeviceUSMSpace::allocate(const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return allocate_sycl( - arg_label, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), - RawMemoryAllocationFailure::AllocationMechanism::SYCLMallocDevice, - sycl::usm::alloc::device, m_queue); -} - -void* SYCLSharedUSMSpace::allocate(const SYCL& exec_space, - const size_t arg_alloc_size) const { - return allocate(exec_space, "[unlabeled]", arg_alloc_size); -} -void* SYCLSharedUSMSpace::allocate(const SYCL& exec_space, - const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return allocate_sycl( - arg_label, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), - RawMemoryAllocationFailure::AllocationMechanism::SYCLMallocShared, - sycl::usm::alloc::shared, - *exec_space.impl_internal_space_instance()->m_queue); -} - -void* SYCLSharedUSMSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void* SYCLSharedUSMSpace::allocate(const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return allocate_sycl( - arg_label, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), - RawMemoryAllocationFailure::AllocationMechanism::SYCLMallocShared, - sycl::usm::alloc::shared, m_queue); -} - -void* SYCLHostUSMSpace::allocate(const SYCL& exec_space, - const size_t arg_alloc_size) const { - return allocate(exec_space, "[unlabeled]", arg_alloc_size); -} -void* SYCLHostUSMSpace::allocate(const SYCL& exec_space, const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return allocate_sycl( - arg_label, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), - RawMemoryAllocationFailure::AllocationMechanism::SYCLMallocHost, - sycl::usm::alloc::host, - *exec_space.impl_internal_space_instance()->m_queue); -} - -void* SYCLHostUSMSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void* SYCLHostUSMSpace::allocate(const char* arg_label, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return allocate_sycl( - arg_label, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), - RawMemoryAllocationFailure::AllocationMechanism::SYCLMallocHost, - sycl::usm::alloc::host, m_queue); -} - -void sycl_deallocate(const char* arg_label, void* const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle, - const sycl::queue& queue) { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - - SYCL::impl_static_fence( - "Kokkos::Impl::sycl_deallocate: fence before deallocate"); - sycl::free(arg_alloc_ptr, queue); -} - -void SYCLDeviceUSMSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} -void SYCLDeviceUSMSpace::deallocate(const char* arg_label, - void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - sycl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), m_queue); -} - -void SYCLSharedUSMSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void SYCLSharedUSMSpace::deallocate(const char* arg_label, - void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - sycl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), m_queue); -} - -void SYCLHostUSMSpace::deallocate(void* const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void SYCLHostUSMSpace::deallocate(const char* arg_label, - void* const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - sycl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size, - Kokkos::Tools::make_space_handle(name()), m_queue); -} - -} // namespace Experimental -} // namespace Kokkos - -namespace Kokkos { -namespace Impl { - -#ifdef KOKKOS_ENABLE_DEBUG -SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::SYCLDeviceUSMSpace, void>::s_root_record; - -SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::SYCLSharedUSMSpace, void>::s_root_record; - -SharedAllocationRecord SharedAllocationRecord< - Kokkos::Experimental::SYCLHostUSMSpace, void>::s_root_record; -#endif - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::SYCLDeviceUSMSpace& space, - const std::string& label, const size_t size, - const SharedAllocationRecord::function_type dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(space, label, size), - sizeof(SharedAllocationHeader) + size, dealloc, label), - m_space(space) { - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, label); - - // Copy to device memory - Kokkos::Experimental::SYCL exec; - Kokkos::Impl::DeepCopy( - exec, RecordBase::m_alloc_ptr, &header, sizeof(SharedAllocationHeader)); - exec.fence( - "SharedAllocationRecord::SharedAllocationRecord(): fence after copying header from " - "HostSpace"); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::SYCL& arg_exec_space, - const Kokkos::Experimental::SYCLDeviceUSMSpace& space, - const std::string& label, const size_t size, - const SharedAllocationRecord::function_type dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Kokkos::Impl::checked_allocation_with_header(arg_exec_space, space, - label, size), - sizeof(SharedAllocationHeader) + size, dealloc, label), - m_space(space) { - SharedAllocationHeader header; - - this->base_t::_fill_host_accessible_header_info(header, label); - - // Copy to device memory - Kokkos::Impl::DeepCopy( - arg_exec_space, RecordBase::m_alloc_ptr, &header, - sizeof(SharedAllocationHeader)); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::SYCL& exec_space, - const Kokkos::Experimental::SYCLSharedUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(exec_space, arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - - this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, - arg_label); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::SYCLSharedUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - - this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, - arg_label); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::SYCL& exec_space, - const Kokkos::Experimental::SYCLHostUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(exec_space, arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - - this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, - arg_label); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::SYCLHostUSMSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - - this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, - arg_label); -} - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -SharedAllocationRecord::~SharedAllocationRecord() { - const auto alloc_size = SharedAllocationRecord::m_alloc_size; - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - alloc_size, alloc_size - sizeof(SharedAllocationHeader)); -} - -SharedAllocationRecord::~SharedAllocationRecord() { - const auto alloc_size = SharedAllocationRecord::m_alloc_size; - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - alloc_size, alloc_size - sizeof(SharedAllocationHeader)); -} - -SharedAllocationRecord::~SharedAllocationRecord() { - const auto alloc_size = SharedAllocationRecord::m_alloc_size; - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - alloc_size, alloc_size - sizeof(SharedAllocationHeader)); -} - -//---------------------------------------------------------------------------- - -} // namespace Impl -} // namespace Kokkos - -//============================================================================== -// {{{1 - -#include - -namespace Kokkos { -namespace Impl { - -// To avoid additional compilation cost for something that's (mostly?) not -// performance sensitive, we explicity instantiate these CRTP base classes here, -// where we have access to the associated *_timpl.hpp header files. -template class HostInaccessibleSharedAllocationRecordCommon< - Kokkos::Experimental::SYCLDeviceUSMSpace>; -template class SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLDeviceUSMSpace>; -template class SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLSharedUSMSpace>; -template class SharedAllocationRecordCommon< - Kokkos::Experimental::SYCLHostUSMSpace>; - -} // namespace Impl -} // namespace Kokkos - -// end Explicit instantiations of CRTP Base classes }}}1 -//============================================================================== diff --git a/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp b/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp deleted file mode 100644 index d726a86f76..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp +++ /dev/null @@ -1,213 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKO_SERIAL_PARALLEL_MDRANGE_HPP -#define KOKKO_SERIAL_PARALLEL_MDRANGE_HPP - -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Serial> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - - using iterate_type = typename Kokkos::Impl::HostIterateTile< - MDRangePolicy, FunctorType, typename MDRangePolicy::work_tag, void>; - - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; - - void exec() const { - const typename Policy::member_type e = m_policy.end(); - for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { - iterate_type(m_mdr_policy, m_functor)(i); - } - } - - public: - inline void execute() const { this->exec(); } - template - static int max_tile_size_product(const Policy&, const Functor&) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } - inline ParallelFor(const FunctorType& arg_functor, - const MDRangePolicy& arg_policy) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)) {} -}; - -template -class ParallelReduce, ReducerType, - Kokkos::Serial> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - - using WorkTag = typename MDRangePolicy::work_tag; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - std::conditional_t::value, WorkTag, - void>; - - using Analysis = FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - - using iterate_type = - typename Kokkos::Impl::HostIterateTile; - - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - inline void exec(reference_type update) const { - const typename Policy::member_type e = m_policy.end(); - for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { - iterate_type(m_mdr_policy, m_functor, update)(i); - } - } - - public: - template - static int max_tile_size_product(const Policy&, const Functor&) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } - inline void execute() const { - const size_t pool_reduce_size = - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)); - const size_t team_reduce_size = 0; // Never shrinks - const size_t team_shared_size = 0; // Never shrinks - const size_t thread_local_size = 0; // Never shrinks - - auto* internal_instance = m_policy.space().impl_internal_space_instance(); - // Need to lock resize_thread_team_data - std::lock_guard lock( - internal_instance->m_thread_team_data_mutex); - internal_instance->resize_thread_team_data( - pool_reduce_size, team_reduce_size, team_shared_size, - thread_local_size); - - pointer_type ptr = - m_result_ptr - ? m_result_ptr - : pointer_type( - internal_instance->m_thread_team_data.pool_reduce_local()); - - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - - reference_type update = final_reducer.init(ptr); - - this->exec(update); - - final_reducer.final(ptr); - } - - template - ParallelReduce(const FunctorType& arg_functor, - const MDRangePolicy& arg_policy, - const HostViewType& arg_result_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void*> = nullptr) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(InvalidType()), - m_result_ptr(arg_result_view.data()) { - static_assert(Kokkos::is_view::value, - "Kokkos::Serial reduce result must be a View"); - - static_assert( - Kokkos::Impl::MemorySpaceAccess::accessible, - "Kokkos::Serial reduce result must be a View in HostSpace"); - } - - inline ParallelReduce(const FunctorType& arg_functor, - MDRangePolicy arg_policy, const ReducerType& reducer) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(reducer), - m_result_ptr(reducer.view().data()) { - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } -}; - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_Task.cpp b/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_Task.cpp deleted file mode 100644 index 468f27eebd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_Task.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#if defined(KOKKOS_ENABLE_TASKDAG) - -#include - -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template class TaskQueue; - -} -} // namespace Kokkos - -#else -void KOKKOS_CORE_SRC_IMPL_SERIAL_TASK_PREVENT_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_TASKDAG ) */ diff --git a/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_WorkGraphPolicy.hpp b/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_WorkGraphPolicy.hpp deleted file mode 100644 index 05980170b8..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Serial/Kokkos_Serial_WorkGraphPolicy.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SERIAL_WORKGRAPHPOLICY_HPP -#define KOKKOS_SERIAL_WORKGRAPHPOLICY_HPP - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Serial> { - private: - using Policy = Kokkos::WorkGraphPolicy; - - Policy m_policy; - FunctorType m_functor; - - template - std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - m_functor(w); - } - - template - std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - const TagType t{}; - m_functor(t, w); - } - - public: - inline void execute() const noexcept { - // Spin until COMPLETED_TOKEN. - // END_TOKEN indicates no work is currently available. - - for (std::int32_t w = Policy::END_TOKEN; - Policy::COMPLETED_TOKEN != (w = m_policy.pop_work());) { - if (Policy::END_TOKEN != w) { - exec_one(w); - m_policy.completed_work(w); - } - } - } - - inline ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_policy(arg_policy), m_functor(arg_functor) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif /* #define KOKKOS_SERIAL_WORKGRAPHPOLICY_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_ThreadsExec.cpp b/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_ThreadsExec.cpp deleted file mode 100644 index 346eb1dc08..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_ThreadsExec.cpp +++ /dev/null @@ -1,902 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { -namespace { -std::mutex host_internal_cppthread_mutex; - -// std::thread compatible driver. -// Recovery from an exception would require constant intra-thread health -// verification; which would negatively impact runtime. As such simply -// abort the process. -void internal_cppthread_driver() { - try { - ThreadsExec::driver(); - } catch (const std::exception &x) { - std::cerr << "Exception thrown from worker thread: " << x.what() - << std::endl; - std::cerr.flush(); - std::abort(); - } catch (...) { - std::cerr << "Exception thrown from worker thread" << std::endl; - std::cerr.flush(); - std::abort(); - } -} - -ThreadsExec s_threads_process; -ThreadsExec *s_threads_exec[ThreadsExec::MAX_THREAD_COUNT] = {nullptr}; -std::thread::id s_threads_pid[ThreadsExec::MAX_THREAD_COUNT]; -std::pair s_threads_coord[ThreadsExec::MAX_THREAD_COUNT]; - -int s_thread_pool_size[3] = {0, 0, 0}; - -unsigned s_current_reduce_size = 0; -unsigned s_current_shared_size = 0; - -void (*volatile s_current_function)(ThreadsExec &, const void *); -const void *volatile s_current_function_arg = nullptr; - -struct Sentinel { - ~Sentinel() { - if (s_thread_pool_size[0] || s_thread_pool_size[1] || - s_thread_pool_size[2] || s_current_reduce_size || - s_current_shared_size || s_current_function || s_current_function_arg || - s_threads_exec[0]) { - std::cerr << "ERROR : Process exiting while Kokkos::Threads is still " - "initialized" - << std::endl; - } - } -}; - -inline unsigned fan_size(const unsigned rank, const unsigned size) { - const unsigned rank_rev = size - (rank + 1); - unsigned count = 0; - for (unsigned n = 1; (rank_rev + n < size) && !(rank_rev & n); n <<= 1) { - ++count; - } - return count; -} - -} // namespace -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -//---------------------------------------------------------------------------- -// Spawn a thread - -void ThreadsExec::spawn() { - std::thread t(internal_cppthread_driver); - t.detach(); -} - -//---------------------------------------------------------------------------- - -bool ThreadsExec::is_process() { - static const std::thread::id master_pid = std::this_thread::get_id(); - - return master_pid == std::this_thread::get_id(); -} - -void ThreadsExec::global_lock() { host_internal_cppthread_mutex.lock(); } - -void ThreadsExec::global_unlock() { host_internal_cppthread_mutex.unlock(); } - -//---------------------------------------------------------------------------- - -void ThreadsExec::wait_yield(volatile int &flag, const int value) { - while (value == flag) { - std::this_thread::yield(); - } -} - -void execute_function_noop(ThreadsExec &, const void *) {} - -void ThreadsExec::driver() { - SharedAllocationRecord::tracking_enable(); - - ThreadsExec this_thread; - - while (ThreadsExec::Active == this_thread.m_pool_state) { - (*s_current_function)(this_thread, s_current_function_arg); - - // Deactivate thread and wait for reactivation - this_thread.m_pool_state = ThreadsExec::Inactive; - - wait_yield(this_thread.m_pool_state, ThreadsExec::Inactive); - } -} - -ThreadsExec::ThreadsExec() - : m_pool_base(nullptr), - m_scratch(nullptr), - m_scratch_reduce_end(0), - m_scratch_thread_end(0), - m_numa_rank(0), - m_numa_core_rank(0), - m_pool_rank(0), - m_pool_size(0), - m_pool_fan_size(0), - m_pool_state(ThreadsExec::Terminating) { - if (&s_threads_process != this) { - // A spawned thread - - ThreadsExec *const nil = nullptr; - - // Which entry in 's_threads_exec', possibly determined from hwloc binding - const int entry = reinterpret_cast(s_current_function_arg) < - size_t(s_thread_pool_size[0]) - ? reinterpret_cast(s_current_function_arg) - : size_t(Kokkos::hwloc::bind_this_thread( - s_thread_pool_size[0], s_threads_coord)); - - // Given a good entry set this thread in the 's_threads_exec' array - if (entry < s_thread_pool_size[0] && - nil == atomic_compare_exchange(s_threads_exec + entry, nil, this)) { - const std::pair coord = - Kokkos::hwloc::get_this_thread_coordinate(); - - m_numa_rank = coord.first; - m_numa_core_rank = coord.second; - m_pool_base = s_threads_exec; - m_pool_rank = s_thread_pool_size[0] - (entry + 1); - m_pool_rank_rev = s_thread_pool_size[0] - (pool_rank() + 1); - m_pool_size = s_thread_pool_size[0]; - m_pool_fan_size = fan_size(m_pool_rank, m_pool_size); - m_pool_state = ThreadsExec::Active; - - s_threads_pid[m_pool_rank] = std::this_thread::get_id(); - - // Inform spawning process that the threads_exec entry has been set. - s_threads_process.m_pool_state = ThreadsExec::Active; - } else { - // Inform spawning process that the threads_exec entry could not be set. - s_threads_process.m_pool_state = ThreadsExec::Terminating; - } - } else { - // Enables 'parallel_for' to execute on unitialized Threads device - m_pool_rank = 0; - m_pool_size = 1; - m_pool_state = ThreadsExec::Inactive; - - s_threads_pid[m_pool_rank] = std::this_thread::get_id(); - } -} - -ThreadsExec::~ThreadsExec() { - const unsigned entry = m_pool_size - (m_pool_rank + 1); - - using Record = Kokkos::Impl::SharedAllocationRecord; - - if (m_scratch) { - Record *const r = Record::get_record(m_scratch); - - m_scratch = nullptr; - - Record::decrement(r); - } - - m_pool_base = nullptr; - m_scratch_reduce_end = 0; - m_scratch_thread_end = 0; - m_numa_rank = 0; - m_numa_core_rank = 0; - m_pool_rank = 0; - m_pool_size = 0; - m_pool_fan_size = 0; - - m_pool_state = ThreadsExec::Terminating; - - if (&s_threads_process != this && entry < MAX_THREAD_COUNT) { - ThreadsExec *const nil = nullptr; - - atomic_compare_exchange(s_threads_exec + entry, this, nil); - - s_threads_process.m_pool_state = ThreadsExec::Terminating; - } -} - -int ThreadsExec::get_thread_count() { return s_thread_pool_size[0]; } - -ThreadsExec *ThreadsExec::get_thread(const int init_thread_rank) { - ThreadsExec *const th = - init_thread_rank < s_thread_pool_size[0] - ? s_threads_exec[s_thread_pool_size[0] - (init_thread_rank + 1)] - : nullptr; - - if (nullptr == th || th->m_pool_rank != init_thread_rank) { - std::ostringstream msg; - msg << "Kokkos::Impl::ThreadsExec::get_thread ERROR : " - << "thread " << init_thread_rank << " of " << s_thread_pool_size[0]; - if (nullptr == th) { - msg << " does not exist"; - } else { - msg << " has wrong thread_rank " << th->m_pool_rank; - } - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - - return th; -} - -//---------------------------------------------------------------------------- - -void ThreadsExec::execute_sleep(ThreadsExec &exec, const void *) { - ThreadsExec::global_lock(); - ThreadsExec::global_unlock(); - - const int n = exec.m_pool_fan_size; - const int rank_rev = exec.m_pool_size - (exec.m_pool_rank + 1); - - for (int i = 0; i < n; ++i) { - Impl::spinwait_while_equal( - exec.m_pool_base[rank_rev + (1 << i)]->m_pool_state, - ThreadsExec::Active); - } - - exec.m_pool_state = ThreadsExec::Inactive; -} - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -void ThreadsExec::verify_is_process(const std::string &name, - const bool initialized) { - if (!is_process()) { - std::string msg(name); - msg.append( - " FAILED : Called by a worker thread, can only be called by the master " - "process."); - Kokkos::Impl::throw_runtime_exception(msg); - } - - if (initialized && 0 == s_thread_pool_size[0]) { - std::string msg(name); - msg.append(" FAILED : Threads not initialized."); - Kokkos::Impl::throw_runtime_exception(msg); - } -} - -int ThreadsExec::in_parallel() { - // A thread function is in execution and - // the function argument is not the special threads process argument and - // the master process is a worker or is not the master process. - return s_current_function && (&s_threads_process != s_current_function_arg) && - (s_threads_process.m_pool_base || !is_process()); -} -void ThreadsExec::fence() { internal_fence(Impl::fence_is_static::yes); } -void ThreadsExec::fence(const std::string &name) { - internal_fence(name, Impl::fence_is_static::yes); -} - -void ThreadsExec::internal_fence(Impl::fence_is_static is_static) { - internal_fence((is_static == Impl::fence_is_static::no) - ? "Kokkos::ThreadsExec::fence: Unnamed Instance Fence" - : "Kokkos::ThreadsExec::fence: Unnamed Static Fence", - is_static); -} - -// Wait for root thread to become inactive -void ThreadsExec::internal_fence(const std::string &name, - Impl::fence_is_static is_static) { - const auto &fence_lam = [&]() { - if (s_thread_pool_size[0]) { - // Wait for the root thread to complete: - Impl::spinwait_while_equal(s_threads_exec[0]->m_pool_state, - ThreadsExec::Active); - } - - s_current_function = nullptr; - s_current_function_arg = nullptr; - - // Make sure function and arguments are cleared before - // potentially re-activating threads with a subsequent launch. - memory_fence(); - }; - if (is_static == Impl::fence_is_static::yes) { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, - Kokkos::Tools::Experimental::SpecialSynchronizationCases:: - GlobalDeviceSynchronization, - fence_lam); - } else { - Kokkos::Tools::Experimental::Impl::profile_fence_event( - name, Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{1}, - fence_lam); - } -} - -/** \brief Begin execution of the asynchronous functor */ -void ThreadsExec::start(void (*func)(ThreadsExec &, const void *), - const void *arg) { - verify_is_process("ThreadsExec::start", true); - - if (s_current_function || s_current_function_arg) { - Kokkos::Impl::throw_runtime_exception( - std::string("ThreadsExec::start() FAILED : already executing")); - } - - s_current_function = func; - s_current_function_arg = arg; - - // Make sure function and arguments are written before activating threads. - memory_fence(); - - // Activate threads: - for (int i = s_thread_pool_size[0]; 0 < i--;) { - s_threads_exec[i]->m_pool_state = ThreadsExec::Active; - } - - if (s_threads_process.m_pool_size) { - // Master process is the root thread, run it: - (*func)(s_threads_process, arg); - s_threads_process.m_pool_state = ThreadsExec::Inactive; - } -} - -//---------------------------------------------------------------------------- - -bool ThreadsExec::sleep() { - verify_is_process("ThreadsExec::sleep", true); - - if (&execute_sleep == s_current_function) return false; - - fence(); - - ThreadsExec::global_lock(); - - s_current_function = &execute_sleep; - - // Activate threads: - for (unsigned i = s_thread_pool_size[0]; 0 < i;) { - s_threads_exec[--i]->m_pool_state = ThreadsExec::Active; - } - - return true; -} - -bool ThreadsExec::wake() { - verify_is_process("ThreadsExec::wake", true); - - if (&execute_sleep != s_current_function) return false; - - ThreadsExec::global_unlock(); - - if (s_threads_process.m_pool_base) { - execute_sleep(s_threads_process, nullptr); - s_threads_process.m_pool_state = ThreadsExec::Inactive; - } - - fence(); - - return true; -} - -//---------------------------------------------------------------------------- - -void ThreadsExec::execute_resize_scratch_in_serial() { - const unsigned begin = s_threads_process.m_pool_base ? 1 : 0; - - auto deallocate_scratch_memory = [](ThreadsExec &exec) { - if (exec.m_scratch) { - using Record = - Kokkos::Impl::SharedAllocationRecord; - Record *const r = Record::get_record(exec.m_scratch); - exec.m_scratch = nullptr; - Record::decrement(r); - } - }; - if (s_threads_process.m_pool_base) { - for (unsigned i = s_thread_pool_size[0]; begin < i;) { - deallocate_scratch_memory(*s_threads_exec[--i]); - } - } - - s_current_function = &first_touch_allocate_thread_private_scratch; - s_current_function_arg = &s_threads_process; - - // Make sure function and arguments are written before activating threads. - memory_fence(); - - for (unsigned i = s_thread_pool_size[0]; begin < i;) { - ThreadsExec &th = *s_threads_exec[--i]; - - th.m_pool_state = ThreadsExec::Active; - - wait_yield(th.m_pool_state, ThreadsExec::Active); - } - - if (s_threads_process.m_pool_base) { - deallocate_scratch_memory(s_threads_process); - s_threads_process.m_pool_state = ThreadsExec::Active; - first_touch_allocate_thread_private_scratch(s_threads_process, nullptr); - s_threads_process.m_pool_state = ThreadsExec::Inactive; - } - - s_current_function_arg = nullptr; - s_current_function = nullptr; - - // Make sure function and arguments are cleared before proceeding. - memory_fence(); -} - -//---------------------------------------------------------------------------- - -void *ThreadsExec::root_reduce_scratch() { - return s_threads_process.reduce_memory(); -} - -void ThreadsExec::first_touch_allocate_thread_private_scratch(ThreadsExec &exec, - const void *) { - exec.m_scratch_reduce_end = s_threads_process.m_scratch_reduce_end; - exec.m_scratch_thread_end = s_threads_process.m_scratch_thread_end; - - if (s_threads_process.m_scratch_thread_end) { - // Allocate tracked memory: - { - using Record = - Kokkos::Impl::SharedAllocationRecord; - Record *const r = - Record::allocate(Kokkos::HostSpace(), "Kokkos::thread_scratch", - s_threads_process.m_scratch_thread_end); - - Record::increment(r); - - exec.m_scratch = r->data(); - } - - unsigned *ptr = reinterpret_cast(exec.m_scratch); - - unsigned *const end = - ptr + s_threads_process.m_scratch_thread_end / sizeof(unsigned); - - // touch on this thread - while (ptr < end) *ptr++ = 0; - } -} - -void *ThreadsExec::resize_scratch(size_t reduce_size, size_t thread_size) { - enum { ALIGN_MASK = Kokkos::Impl::MEMORY_ALIGNMENT - 1 }; - - fence(); - - const size_t old_reduce_size = s_threads_process.m_scratch_reduce_end; - const size_t old_thread_size = s_threads_process.m_scratch_thread_end - - s_threads_process.m_scratch_reduce_end; - - reduce_size = (reduce_size + ALIGN_MASK) & ~ALIGN_MASK; - thread_size = (thread_size + ALIGN_MASK) & ~ALIGN_MASK; - - // Increase size or deallocate completely. - - if ((old_reduce_size < reduce_size) || (old_thread_size < thread_size) || - ((reduce_size == 0 && thread_size == 0) && - (old_reduce_size != 0 || old_thread_size != 0))) { - verify_is_process("ThreadsExec::resize_scratch", true); - - s_threads_process.m_scratch_reduce_end = reduce_size; - s_threads_process.m_scratch_thread_end = reduce_size + thread_size; - - execute_resize_scratch_in_serial(); - - s_threads_process.m_scratch = s_threads_exec[0]->m_scratch; - } - - return s_threads_process.m_scratch; -} - -//---------------------------------------------------------------------------- - -void ThreadsExec::print_configuration(std::ostream &s, const bool detail) { - verify_is_process("ThreadsExec::print_configuration", false); - - fence(); - - const unsigned numa_count = Kokkos::hwloc::get_available_numa_count(); - const unsigned cores_per_numa = Kokkos::hwloc::get_available_cores_per_numa(); - const unsigned threads_per_core = - Kokkos::hwloc::get_available_threads_per_core(); - - // Forestall compiler warnings for unused variables. - (void)numa_count; - (void)cores_per_numa; - (void)threads_per_core; - - s << "Kokkos::Threads"; - -#if defined(KOKKOS_ENABLE_THREADS) - s << " KOKKOS_ENABLE_THREADS"; -#endif -#if defined(KOKKOS_ENABLE_HWLOC) - s << " hwloc[" << numa_count << "x" << cores_per_numa << "x" - << threads_per_core << "]"; -#endif - - if (s_thread_pool_size[0]) { - s << " threads[" << s_thread_pool_size[0] << "]" - << " threads_per_numa[" << s_thread_pool_size[1] << "]" - << " threads_per_core[" << s_thread_pool_size[2] << "]"; - if (nullptr == s_threads_process.m_pool_base) { - s << " Asynchronous"; - } - s << " ReduceScratch[" << s_current_reduce_size << "]" - << " SharedScratch[" << s_current_shared_size << "]"; - s << std::endl; - - if (detail) { - for (int i = 0; i < s_thread_pool_size[0]; ++i) { - ThreadsExec *const th = s_threads_exec[i]; - - if (th) { - const int rank_rev = th->m_pool_size - (th->m_pool_rank + 1); - - s << " Thread[ " << th->m_pool_rank << " : " << th->m_numa_rank << "." - << th->m_numa_core_rank << " ]"; - - s << " Fan{"; - for (int j = 0; j < th->m_pool_fan_size; ++j) { - ThreadsExec *const thfan = th->m_pool_base[rank_rev + (1 << j)]; - s << " [ " << thfan->m_pool_rank << " : " << thfan->m_numa_rank - << "." << thfan->m_numa_core_rank << " ]"; - } - s << " }"; - - if (th == &s_threads_process) { - s << " is_process"; - } - } - s << std::endl; - } - } - } else { - s << " not initialized" << std::endl; - } -} - -//---------------------------------------------------------------------------- - -int ThreadsExec::is_initialized() { return nullptr != s_threads_exec[0]; } - -void ThreadsExec::initialize(int thread_count_arg) { - // legacy arguments - unsigned thread_count = thread_count_arg == -1 ? 0 : thread_count_arg; - unsigned use_numa_count = 0; - unsigned use_cores_per_numa = 0; - bool allow_asynchronous_threadpool = false; - // need to provide an initializer for Intel compilers - static const Sentinel sentinel = {}; - - const bool is_initialized = 0 != s_thread_pool_size[0]; - - unsigned thread_spawn_failed = 0; - - for (int i = 0; i < ThreadsExec::MAX_THREAD_COUNT; i++) - s_threads_exec[i] = nullptr; - - if (!is_initialized) { - // If thread_count, use_numa_count, or use_cores_per_numa are zero - // then they will be given default values based upon hwloc detection - // and allowed asynchronous execution. - - const bool hwloc_avail = Kokkos::hwloc::available(); - const bool hwloc_can_bind = - hwloc_avail && Kokkos::hwloc::can_bind_threads(); - - if (thread_count == 0) { - thread_count = hwloc_avail - ? Kokkos::hwloc::get_available_numa_count() * - Kokkos::hwloc::get_available_cores_per_numa() * - Kokkos::hwloc::get_available_threads_per_core() - : 1; - } - - const unsigned thread_spawn_begin = hwloc::thread_mapping( - "Kokkos::Threads::initialize", allow_asynchronous_threadpool, - thread_count, use_numa_count, use_cores_per_numa, s_threads_coord); - - const std::pair proc_coord = s_threads_coord[0]; - - if (thread_spawn_begin) { - // Synchronous with s_threads_coord[0] as the process core - // Claim entry #0 for binding the process core. - s_threads_coord[0] = std::pair(~0u, ~0u); - } - - s_thread_pool_size[0] = thread_count; - s_thread_pool_size[1] = s_thread_pool_size[0] / use_numa_count; - s_thread_pool_size[2] = s_thread_pool_size[1] / use_cores_per_numa; - s_current_function = - &execute_function_noop; // Initialization work function - - for (unsigned ith = thread_spawn_begin; ith < thread_count; ++ith) { - s_threads_process.m_pool_state = ThreadsExec::Inactive; - - // If hwloc available then spawned thread will - // choose its own entry in 's_threads_coord' - // otherwise specify the entry. - s_current_function_arg = - reinterpret_cast(hwloc_can_bind ? ~0u : ith); - - // Make sure all outstanding memory writes are complete - // before spawning the new thread. - memory_fence(); - - // Spawn thread executing the 'driver()' function. - // Wait until spawned thread has attempted to initialize. - // If spawning and initialization is successful then - // an entry in 's_threads_exec' will be assigned. - ThreadsExec::spawn(); - wait_yield(s_threads_process.m_pool_state, ThreadsExec::Inactive); - if (s_threads_process.m_pool_state == ThreadsExec::Terminating) break; - } - - // Wait for all spawned threads to deactivate before zeroing the function. - - for (unsigned ith = thread_spawn_begin; ith < thread_count; ++ith) { - // Try to protect against cache coherency failure by casting to volatile. - ThreadsExec *const th = ((ThreadsExec * volatile *)s_threads_exec)[ith]; - if (th) { - wait_yield(th->m_pool_state, ThreadsExec::Active); - } else { - ++thread_spawn_failed; - } - } - - s_current_function = nullptr; - s_current_function_arg = nullptr; - s_threads_process.m_pool_state = ThreadsExec::Inactive; - - memory_fence(); - - if (!thread_spawn_failed) { - // Bind process to the core on which it was located before spawning - // occurred - if (hwloc_can_bind) { - Kokkos::hwloc::bind_this_thread(proc_coord); - } - - if (thread_spawn_begin) { // Include process in pool. - const std::pair coord = - Kokkos::hwloc::get_this_thread_coordinate(); - - s_threads_exec[0] = &s_threads_process; - s_threads_process.m_numa_rank = coord.first; - s_threads_process.m_numa_core_rank = coord.second; - s_threads_process.m_pool_base = s_threads_exec; - s_threads_process.m_pool_rank = - thread_count - 1; // Reversed for scan-compatible reductions - s_threads_process.m_pool_size = thread_count; - s_threads_process.m_pool_fan_size = fan_size( - s_threads_process.m_pool_rank, s_threads_process.m_pool_size); - s_threads_pid[s_threads_process.m_pool_rank] = - std::this_thread::get_id(); - } else { - s_threads_process.m_pool_base = nullptr; - s_threads_process.m_pool_rank = 0; - s_threads_process.m_pool_size = 0; - s_threads_process.m_pool_fan_size = 0; - } - - // Initial allocations: - ThreadsExec::resize_scratch(1024, 1024); - } else { - s_thread_pool_size[0] = 0; - s_thread_pool_size[1] = 0; - s_thread_pool_size[2] = 0; - } - } - - if (is_initialized || thread_spawn_failed) { - std::ostringstream msg; - - msg << "Kokkos::Threads::initialize ERROR"; - - if (is_initialized) { - msg << " : already initialized"; - } - if (thread_spawn_failed) { - msg << " : failed to spawn " << thread_spawn_failed << " threads"; - } - - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - - // Check for over-subscription - if (Kokkos::show_warnings() && - (Impl::mpi_ranks_per_node() * long(thread_count) > - Impl::processors_per_node())) { - std::cerr << "Kokkos::Threads::initialize WARNING: You are likely " - "oversubscribing your CPU cores." - << std::endl; - std::cerr << " Detected: " - << Impl::processors_per_node() << " cores per node." << std::endl; - std::cerr << " Detected: " - << Impl::mpi_ranks_per_node() << " MPI_ranks per node." - << std::endl; - std::cerr << " Requested: " - << thread_count << " threads per process." << std::endl; - } - - // Init the array for used for arbitrarily sized atomics - Impl::init_lock_array_host_space(); - - Impl::SharedAllocationRecord::tracking_enable(); -} - -//---------------------------------------------------------------------------- - -void ThreadsExec::finalize() { - verify_is_process("ThreadsExec::finalize", false); - - fence(); - - resize_scratch(0, 0); - - const unsigned begin = s_threads_process.m_pool_base ? 1 : 0; - - for (unsigned i = s_thread_pool_size[0]; begin < i--;) { - if (s_threads_exec[i]) { - s_threads_exec[i]->m_pool_state = ThreadsExec::Terminating; - - wait_yield(s_threads_process.m_pool_state, ThreadsExec::Inactive); - - s_threads_process.m_pool_state = ThreadsExec::Inactive; - } - - s_threads_pid[i] = std::thread::id(); - } - - if (s_threads_process.m_pool_base) { - (&s_threads_process)->~ThreadsExec(); - s_threads_exec[0] = nullptr; - } - - if (Kokkos::hwloc::can_bind_threads()) { - Kokkos::hwloc::unbind_this_thread(); - } - - s_thread_pool_size[0] = 0; - s_thread_pool_size[1] = 0; - s_thread_pool_size[2] = 0; - - // Reset master thread to run solo. - s_threads_process.m_numa_rank = 0; - s_threads_process.m_numa_core_rank = 0; - s_threads_process.m_pool_base = nullptr; - s_threads_process.m_pool_rank = 0; - s_threads_process.m_pool_size = 1; - s_threads_process.m_pool_fan_size = 0; - s_threads_process.m_pool_state = ThreadsExec::Inactive; - - Kokkos::Profiling::finalize(); -} - -//---------------------------------------------------------------------------- - -} /* namespace Impl */ -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -int Threads::concurrency() { return impl_thread_pool_size(0); } -void Threads::fence(const std::string &name) const { - Impl::ThreadsExec::internal_fence(name, Impl::fence_is_static::no); -} - -Threads &Threads::impl_instance(int) { - static Threads t; - return t; -} - -int Threads::impl_thread_pool_rank_host() { - const std::thread::id pid = std::this_thread::get_id(); - int i = 0; - while ((i < Impl::s_thread_pool_size[0]) && (pid != Impl::s_threads_pid[i])) { - ++i; - } - return i; -} - -int Threads::impl_thread_pool_size(int depth) { - return Impl::s_thread_pool_size[depth]; -} - -const char *Threads::name() { return "Threads"; } - -namespace Impl { - -int g_threads_space_factory_initialized = - initialize_space_factory("050_Threads"); - -} // namespace Impl - -#ifdef KOKKOS_ENABLE_CXX14 -namespace Tools { -namespace Experimental { -constexpr DeviceType DeviceTypeTraits::id; -} -} // namespace Tools -#endif - -} /* namespace Kokkos */ diff --git a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_MDRange.hpp b/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_MDRange.hpp deleted file mode 100644 index 6d1a38da1d..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_MDRange.hpp +++ /dev/null @@ -1,322 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_THREADS_PARALLEL_MDRANGE_HPP -#define KOKKOS_THREADS_PARALLEL_MDRANGE_HPP - -#include - -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Threads> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - - using WorkTag = typename MDRangePolicy::work_tag; - - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using iterate_type = typename Kokkos::Impl::HostIterateTile< - MDRangePolicy, FunctorType, typename MDRangePolicy::work_tag, void>; - - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; // construct as RangePolicy( 0, num_tiles - // ).set_chunk_size(1) in ctor - - inline static void exec_range(const MDRangePolicy &mdr_policy, - const FunctorType &functor, const Member ibeg, - const Member iend) { -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - iterate_type(mdr_policy, functor)(i); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - exec_schedule(exec, arg); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelFor &self = *((const ParallelFor *)arg); - - WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - ParallelFor::exec_range(self.m_mdr_policy, self.m_functor, range.begin(), - range.end()); - - exec.fan_in(); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelFor &self = *((const ParallelFor *)arg); - - WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - exec.set_work_range(range.begin(), range.end(), self.m_policy.chunk_size()); - exec.reset_steal_target(); - exec.barrier(); - - long work_index = exec.get_work_index(); - - while (work_index != -1) { - const Member begin = - static_cast(work_index) * self.m_policy.chunk_size(); - const Member end = - begin + self.m_policy.chunk_size() < self.m_policy.end() - ? begin + self.m_policy.chunk_size() - : self.m_policy.end(); - - ParallelFor::exec_range(self.m_mdr_policy, self.m_functor, begin, end); - work_index = exec.get_work_index(); - } - - exec.fan_in(); - } - - public: - inline void execute() const { - ThreadsExec::start(&ParallelFor::exec, this); - ThreadsExec::fence(); - } - - ParallelFor(const FunctorType &arg_functor, const MDRangePolicy &arg_policy) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)) {} - - template - static int max_tile_size_product(const Policy &, const Functor &) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } -}; - -template -class ParallelReduce, ReducerType, - Kokkos::Threads> { - private: - using MDRangePolicy = Kokkos::MDRangePolicy; - using Policy = typename MDRangePolicy::impl_range_policy; - - using WorkTag = typename MDRangePolicy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - typename Kokkos::Impl::if_c::value, - WorkTag, void>::type; - - using Analysis = Impl::FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using value_type = typename Analysis::value_type; - using reference_type = typename Analysis::reference_type; - - using iterate_type = - typename Kokkos::Impl::HostIterateTile; - - const FunctorType m_functor; - const MDRangePolicy m_mdr_policy; - const Policy m_policy; // construct as RangePolicy( 0, num_tiles - // ).set_chunk_size(1) in ctor - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - inline static void exec_range(const MDRangePolicy &mdr_policy, - const FunctorType &functor, const Member &ibeg, - const Member &iend, reference_type update) { -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - iterate_type(mdr_policy, functor, update)(i); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - exec_schedule(exec, arg); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelReduce &self = *((const ParallelReduce *)arg); - const WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - typename Analysis::Reducer reducer( - &ReducerConditional::select(self.m_functor, self.m_reducer)); - - ParallelReduce::exec_range( - self.m_mdr_policy, self.m_functor, range.begin(), range.end(), - reducer.init(static_cast(exec.reduce_memory()))); - - exec.fan_in_reduce(reducer); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelReduce &self = *((const ParallelReduce *)arg); - const WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - exec.set_work_range(range.begin(), range.end(), self.m_policy.chunk_size()); - exec.reset_steal_target(); - exec.barrier(); - - long work_index = exec.get_work_index(); - typename Analysis::Reducer reducer( - &ReducerConditional::select(self.m_functor, self.m_reducer)); - - reference_type update = - reducer.init(static_cast(exec.reduce_memory())); - while (work_index != -1) { - const Member begin = - static_cast(work_index) * self.m_policy.chunk_size(); - const Member end = - begin + self.m_policy.chunk_size() < self.m_policy.end() - ? begin + self.m_policy.chunk_size() - : self.m_policy.end(); - ParallelReduce::exec_range(self.m_mdr_policy, self.m_functor, begin, end, - update); - work_index = exec.get_work_index(); - } - - exec.fan_in_reduce(reducer); - } - - public: - inline void execute() const { - ThreadsExec::resize_scratch( - Analysis::value_size(ReducerConditional::select(m_functor, m_reducer)), - 0); - - ThreadsExec::start(&ParallelReduce::exec, this); - - ThreadsExec::fence(); - - if (m_result_ptr) { - const pointer_type data = - (pointer_type)ThreadsExec::root_reduce_scratch(); - - const unsigned n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - for (unsigned i = 0; i < n; ++i) { - m_result_ptr[i] = data[i]; - } - } - } - - template - ParallelReduce(const FunctorType &arg_functor, - const MDRangePolicy &arg_policy, - const HostViewType &arg_result_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void *> = nullptr) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(InvalidType()), - m_result_ptr(arg_result_view.data()) { - static_assert(Kokkos::is_view::value, - "Kokkos::Threads reduce result must be a View"); - - static_assert( - std::is_same::value, - "Kokkos::Threads reduce result must be a View in HostSpace"); - } - - inline ParallelReduce(const FunctorType &arg_functor, - MDRangePolicy arg_policy, const ReducerType &reducer) - : m_functor(arg_functor), - m_mdr_policy(arg_policy), - m_policy(Policy(0, m_mdr_policy.m_num_tiles).set_chunk_size(1)), - m_reducer(reducer), - m_result_ptr(reducer.view().data()) { - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } - - template - static int max_tile_size_product(const Policy &, const Functor &) { - /** - * 1024 here is just our guess for a reasonable max tile size, - * it isn't a hardware constraint. If people see a use for larger - * tile size products, we're happy to change this. - */ - return 1024; - } -}; - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_Range.hpp b/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_Range.hpp deleted file mode 100644 index 971a0bb9c2..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_Range.hpp +++ /dev/null @@ -1,485 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_THREADS_PARALLEL_RANGE_HPP -#define KOKKOS_THREADS_PARALLEL_RANGE_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Threads> { - private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - const FunctorType m_functor; - const Policy m_policy; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member ibeg, const Member iend) { -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(i); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member ibeg, const Member iend) { - const TagType t{}; -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(t, i); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - exec_schedule(exec, arg); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelFor &self = *((const ParallelFor *)arg); - - WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - ParallelFor::template exec_range(self.m_functor, range.begin(), - range.end()); - - exec.fan_in(); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelFor &self = *((const ParallelFor *)arg); - - WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - exec.set_work_range(range.begin() - self.m_policy.begin(), - range.end() - self.m_policy.begin(), - self.m_policy.chunk_size()); - exec.reset_steal_target(); - exec.barrier(); - - long work_index = exec.get_work_index(); - - while (work_index != -1) { - const Member begin = - static_cast(work_index) * self.m_policy.chunk_size() + - self.m_policy.begin(); - const Member end = - begin + self.m_policy.chunk_size() < self.m_policy.end() - ? begin + self.m_policy.chunk_size() - : self.m_policy.end(); - ParallelFor::template exec_range(self.m_functor, begin, end); - work_index = exec.get_work_index(); - } - - exec.fan_in(); - } - - public: - inline void execute() const { - ThreadsExec::start(&ParallelFor::exec, this); - ThreadsExec::fence(); - } - - ParallelFor(const FunctorType &arg_functor, const Policy &arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} -}; - -template -class ParallelReduce, ReducerType, - Kokkos::Threads> { - private: - using Policy = Kokkos::RangePolicy; - - using WorkTag = typename Policy::work_tag; - using WorkRange = typename Policy::WorkRange; - using Member = typename Policy::member_type; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - typename Kokkos::Impl::if_c::value, - WorkTag, void>::type; - - using Analysis = Impl::FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member &ibeg, const Member &iend, - reference_type update) { -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(i, update); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member &ibeg, const Member &iend, - reference_type update) { - const TagType t{}; -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(t, i, update); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - exec_schedule(exec, arg); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelReduce &self = *((const ParallelReduce *)arg); - const WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - typename Analysis::Reducer reducer( - &ReducerConditional::select(self.m_functor, self.m_reducer)); - - ParallelReduce::template exec_range( - self.m_functor, range.begin(), range.end(), - reducer.init(static_cast(exec.reduce_memory()))); - - exec.fan_in_reduce(reducer); - } - - template - static std::enable_if_t::value> - exec_schedule(ThreadsExec &exec, const void *arg) { - const ParallelReduce &self = *((const ParallelReduce *)arg); - const WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - exec.set_work_range(range.begin() - self.m_policy.begin(), - range.end() - self.m_policy.begin(), - self.m_policy.chunk_size()); - exec.reset_steal_target(); - exec.barrier(); - - long work_index = exec.get_work_index(); - typename Analysis::Reducer reducer( - &ReducerConditional::select(self.m_functor, self.m_reducer)); - - reference_type update = - reducer.init(static_cast(exec.reduce_memory())); - while (work_index != -1) { - const Member begin = - static_cast(work_index) * self.m_policy.chunk_size() + - self.m_policy.begin(); - const Member end = - begin + self.m_policy.chunk_size() < self.m_policy.end() - ? begin + self.m_policy.chunk_size() - : self.m_policy.end(); - ParallelReduce::template exec_range(self.m_functor, begin, end, - update); - work_index = exec.get_work_index(); - } - - exec.fan_in_reduce(reducer); - } - - public: - inline void execute() const { - if (m_policy.end() <= m_policy.begin()) { - if (m_result_ptr) { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - final_reducer.init(m_result_ptr); - final_reducer.final(m_result_ptr); - } - } else { - ThreadsExec::resize_scratch( - Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)), - 0); - - ThreadsExec::start(&ParallelReduce::exec, this); - - ThreadsExec::fence(); - - if (m_result_ptr) { - const pointer_type data = - (pointer_type)ThreadsExec::root_reduce_scratch(); - - const unsigned n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - for (unsigned i = 0; i < n; ++i) { - m_result_ptr[i] = data[i]; - } - } - } - } - - template - ParallelReduce(const FunctorType &arg_functor, const Policy &arg_policy, - const HostViewType &arg_result_view, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void *> = nullptr) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(InvalidType()), - m_result_ptr(arg_result_view.data()) { - static_assert(Kokkos::is_view::value, - "Kokkos::Threads reduce result must be a View"); - - static_assert( - std::is_same::value, - "Kokkos::Threads reduce result must be a View in HostSpace"); - } - - inline ParallelReduce(const FunctorType &arg_functor, Policy arg_policy, - const ReducerType &reducer) - : m_functor(arg_functor), - m_policy(arg_policy), - m_reducer(reducer), - m_result_ptr(reducer.view().data()) { - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } -}; - -template -class ParallelScan, - Kokkos::Threads> { - private: - using Policy = Kokkos::RangePolicy; - using WorkRange = typename Policy::WorkRange; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using Analysis = Impl::FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - const FunctorType m_functor; - const Policy m_policy; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member &ibeg, const Member &iend, - reference_type update, const bool final) { -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(i, update, final); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member &ibeg, const Member &iend, - reference_type update, const bool final) { - const TagType t{}; -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(t, i, update, final); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - const ParallelScan &self = *((const ParallelScan *)arg); - - const WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - typename Analysis::Reducer final_reducer(&self.m_functor); - - reference_type update = - final_reducer.init(static_cast(exec.reduce_memory())); - - ParallelScan::template exec_range(self.m_functor, range.begin(), - range.end(), update, false); - - // exec.template scan_large( final_reducer ); - exec.scan_small(final_reducer); - - ParallelScan::template exec_range(self.m_functor, range.begin(), - range.end(), update, true); - - exec.fan_in(); - } - - public: - inline void execute() const { - ThreadsExec::resize_scratch(2 * Analysis::value_size(m_functor), 0); - ThreadsExec::start(&ParallelScan::exec, this); - ThreadsExec::fence(); - } - - ParallelScan(const FunctorType &arg_functor, const Policy &arg_policy) - : m_functor(arg_functor), m_policy(arg_policy) {} -}; - -template -class ParallelScanWithTotal, - ReturnType, Kokkos::Threads> { - private: - using Policy = Kokkos::RangePolicy; - using WorkRange = typename Policy::WorkRange; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - - using Analysis = Impl::FunctorAnalysis; - - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - const FunctorType m_functor; - const Policy m_policy; - ReturnType &m_returnvalue; - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member &ibeg, const Member &iend, - reference_type update, const bool final) { -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(i, update, final); - } - } - - template - inline static std::enable_if_t::value> exec_range( - const FunctorType &functor, const Member &ibeg, const Member &iend, - reference_type update, const bool final) { - const TagType t{}; -#if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ - defined(KOKKOS_ENABLE_PRAGMA_IVDEP) -#pragma ivdep -#endif - for (Member i = ibeg; i < iend; ++i) { - functor(t, i, update, final); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - const ParallelScanWithTotal &self = *((const ParallelScanWithTotal *)arg); - - const WorkRange range(self.m_policy, exec.pool_rank(), exec.pool_size()); - - typename Analysis::Reducer final_reducer(&self.m_functor); - - reference_type update = - final_reducer.init(static_cast(exec.reduce_memory())); - - ParallelScanWithTotal::template exec_range( - self.m_functor, range.begin(), range.end(), update, false); - - // exec.template scan_large(final_reducer); - exec.scan_small(final_reducer); - - ParallelScanWithTotal::template exec_range( - self.m_functor, range.begin(), range.end(), update, true); - - exec.fan_in(); - - if (exec.pool_rank() == exec.pool_size() - 1) { - self.m_returnvalue = update; - } - } - - public: - inline void execute() const { - ThreadsExec::resize_scratch(2 * Analysis::value_size(m_functor), 0); - ThreadsExec::start(&ParallelScanWithTotal::exec, this); - ThreadsExec::fence(); - } - - ParallelScanWithTotal(const FunctorType &arg_functor, - const Policy &arg_policy, ReturnType &arg_returnvalue) - : m_functor(arg_functor), - m_policy(arg_policy), - m_returnvalue(arg_returnvalue) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_Team.hpp b/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_Team.hpp deleted file mode 100644 index bdda110d37..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_Parallel_Team.hpp +++ /dev/null @@ -1,279 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_THREADS_PARALLEL_TEAM_HPP -#define KOKKOS_THREADS_PARALLEL_TEAM_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Threads> { - private: - using Policy = - Kokkos::Impl::TeamPolicyInternal; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - - const FunctorType m_functor; - const Policy m_policy; - const size_t m_shared; - - template - inline static std::enable_if_t::value && - std::is_same::value> - exec_team(const FunctorType &functor, Member member) { - for (; member.valid_static(); member.next_static()) { - functor(member); - } - } - - template - inline static std::enable_if_t::value && - std::is_same::value> - exec_team(const FunctorType &functor, Member member) { - const TagType t{}; - for (; member.valid_static(); member.next_static()) { - functor(t, member); - } - } - - template - inline static std::enable_if_t::value && - std::is_same::value> - exec_team(const FunctorType &functor, Member member) { - for (; member.valid_dynamic(); member.next_dynamic()) { - functor(member); - } - } - - template - inline static std::enable_if_t::value && - std::is_same::value> - exec_team(const FunctorType &functor, Member member) { - const TagType t{}; - for (; member.valid_dynamic(); member.next_dynamic()) { - functor(t, member); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - const ParallelFor &self = *((const ParallelFor *)arg); - - ParallelFor::exec_team( - self.m_functor, Member(&exec, self.m_policy, self.m_shared)); - - exec.barrier(); - exec.fan_in(); - } - template - Policy fix_policy(Policy policy) { - if (policy.impl_vector_length() < 0) { - policy.impl_set_vector_length(1); - } - if (policy.team_size() < 0) { - policy.impl_set_team_size( - policy.team_size_recommended(m_functor, ParallelForTag{})); - } - return policy; - } - - public: - inline void execute() const { - ThreadsExec::resize_scratch( - 0, Policy::member_type::team_reduce_size() + m_shared); - - ThreadsExec::start(&ParallelFor::exec, this); - - ThreadsExec::fence(); - } - - ParallelFor(const FunctorType &arg_functor, const Policy &arg_policy) - : m_functor(arg_functor), - m_policy(fix_policy(arg_policy)), - m_shared(m_policy.scratch_size(0) + m_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, m_policy.team_size())) {} -}; - -template -class ParallelReduce, - ReducerType, Kokkos::Threads> { - private: - using Policy = - Kokkos::Impl::TeamPolicyInternal; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - - using ReducerConditional = - Kokkos::Impl::if_c::value, - FunctorType, ReducerType>; - using ReducerTypeFwd = typename ReducerConditional::type; - using WorkTagFwd = - typename Kokkos::Impl::if_c::value, - WorkTag, void>::type; - - using Analysis = Impl::FunctorAnalysis; - using pointer_type = typename Analysis::pointer_type; - using reference_type = typename Analysis::reference_type; - - const FunctorType m_functor; - const Policy m_policy; - const ReducerType m_reducer; - const pointer_type m_result_ptr; - const size_t m_shared; - - template - inline static std::enable_if_t::value> exec_team( - const FunctorType &functor, Member member, reference_type update) { - for (; member.valid_static(); member.next_static()) { - functor(member, update); - } - } - - template - inline static std::enable_if_t::value> exec_team( - const FunctorType &functor, Member member, reference_type update) { - const TagType t{}; - for (; member.valid_static(); member.next_static()) { - functor(t, member, update); - } - } - - static void exec(ThreadsExec &exec, const void *arg) { - const ParallelReduce &self = *((const ParallelReduce *)arg); - - typename Analysis::Reducer reducer( - &ReducerConditional::select(self.m_functor, self.m_reducer)); - - ParallelReduce::template exec_team( - self.m_functor, Member(&exec, self.m_policy, self.m_shared), - reducer.init(static_cast(exec.reduce_memory()))); - - exec.fan_in_reduce(reducer); - } - - public: - inline void execute() const { - if (m_policy.league_size() * m_policy.team_size() == 0) { - if (m_result_ptr) { - typename Analysis::Reducer final_reducer( - &ReducerConditional::select(m_functor, m_reducer)); - final_reducer.init(m_result_ptr); - final_reducer.final(m_result_ptr); - } - } else { - ThreadsExec::resize_scratch( - Analysis::value_size( - ReducerConditional::select(m_functor, m_reducer)), - Policy::member_type::team_reduce_size() + m_shared); - - ThreadsExec::start(&ParallelReduce::exec, this); - - ThreadsExec::fence(); - - if (m_result_ptr) { - const pointer_type data = - (pointer_type)ThreadsExec::root_reduce_scratch(); - - const unsigned n = Analysis::value_count( - ReducerConditional::select(m_functor, m_reducer)); - for (unsigned i = 0; i < n; ++i) { - m_result_ptr[i] = data[i]; - } - } - } - } - - template - Policy fix_policy(Policy policy) { - if (policy.impl_vector_length() < 0) { - policy.impl_set_vector_length(1); - } - if (policy.team_size() < 0) { - policy.impl_set_team_size(policy.team_size_recommended( - m_functor, m_reducer, ParallelReduceTag{})); - } - return policy; - } - - template - inline ParallelReduce( - const FunctorType &arg_functor, const Policy &arg_policy, - const ViewType &arg_result, - std::enable_if_t::value && - !Kokkos::is_reducer::value, - void *> = nullptr) - : m_functor(arg_functor), - m_policy(fix_policy(arg_policy)), - m_reducer(InvalidType()), - m_result_ptr(arg_result.data()), - m_shared(m_policy.scratch_size(0) + m_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, m_policy.team_size())) {} - - inline ParallelReduce(const FunctorType &arg_functor, Policy arg_policy, - const ReducerType &reducer) - : m_functor(arg_functor), - m_policy(fix_policy(arg_policy)), - m_reducer(reducer), - m_result_ptr(reducer.view().data()), - m_shared(m_policy.scratch_size(0) + m_policy.scratch_size(1) + - FunctorTeamShmemSize::value( - arg_functor, m_policy.team_size())) { - /*static_assert( std::is_same< typename ViewType::memory_space - , Kokkos::HostSpace >::value - , "Reduction result on Kokkos::OpenMP must be a Kokkos::View in HostSpace" - );*/ - } -}; - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp b/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp deleted file mode 100644 index 5e8ac4604c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_THREADS_WORKGRAPHPOLICY_HPP -#define KOKKOS_THREADS_WORKGRAPHPOLICY_HPP - -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class ParallelFor, - Kokkos::Threads> { - private: - using Policy = Kokkos::WorkGraphPolicy; - - using Self = ParallelFor, - Kokkos::Threads>; - - Policy m_policy; - FunctorType m_functor; - - template - std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - m_functor(w); - } - - template - std::enable_if_t::value> exec_one( - const std::int32_t w) const noexcept { - const TagType t{}; - m_functor(t, w); - } - - inline void exec_one_thread() const noexcept { - // Spin until COMPLETED_TOKEN. - // END_TOKEN indicates no work is currently available. - - for (std::int32_t w = Policy::END_TOKEN; - Policy::COMPLETED_TOKEN != (w = m_policy.pop_work());) { - if (Policy::END_TOKEN != w) { - exec_one(w); - m_policy.completed_work(w); - } - } - } - - static inline void thread_main(ThreadsExec& exec, const void* arg) noexcept { - const Self& self = *(static_cast(arg)); - self.exec_one_thread(); - exec.fan_in(); - } - - public: - inline void execute() { - ThreadsExec::start(&Self::thread_main, this); - ThreadsExec::fence(); - } - - inline ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy) - : m_policy(arg_policy), m_functor(arg_functor) {} -}; - -} // namespace Impl -} // namespace Kokkos - -#endif /* #define KOKKOS_THREADS_WORKGRAPHPOLICY_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_CUDA.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_CUDA.hpp deleted file mode 100644 index aedb8d035c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_CUDA.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_CUDA_HPP -#define KOKKOS_DECLARE_CUDA_HPP - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HBWSpace.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HBWSpace.hpp deleted file mode 100644 index ff5133ed28..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HBWSpace.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_HBWSPACE_HPP -#define KOKKOS_DECLARE_HBWSPACE_HPP - -#ifdef KOKKOS_ENABLE_HBWSPACE -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HIP.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HIP.hpp deleted file mode 100644 index b3bf14dbf2..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HIP.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_HIP_HPP -#define KOKKOS_DECLARE_HIP_HPP - -#if defined(KOKKOS_ENABLE_HIP) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HPX.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HPX.hpp deleted file mode 100644 index 2cbecc9e76..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_HPX.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_HPX_HPP -#define KOKKOS_DECLARE_HPX_HPP - -#if defined(KOKKOS_ENABLE_HPX) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENACC.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENACC.hpp deleted file mode 100644 index 5c09b7a3b6..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENACC.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_OPENACC_HPP -#define KOKKOS_DECLARE_OPENACC_HPP - -#if defined(KOKKOS_ENABLE_OPENACC) -#include -#include -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENMP.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENMP.hpp deleted file mode 100644 index 069dd5c160..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENMP.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_OPENMP_HPP -#define KOKKOS_DECLARE_OPENMP_HPP - -#if defined(KOKKOS_ENABLE_OPENMP) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENMPTARGET.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENMPTARGET.hpp deleted file mode 100644 index b193d1e741..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_OPENMPTARGET.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_OPENMPTARGET_HPP -#define KOKKOS_DECLARE_OPENMPTARGET_HPP - -#if defined(KOKKOS_ENABLE_OPENMPTARGET) -#include -#include -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_SERIAL.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_SERIAL.hpp deleted file mode 100644 index 45661b5af2..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_SERIAL.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_SERIAL_HPP -#define KOKKOS_DECLARE_SERIAL_HPP - -#if defined(KOKKOS_ENABLE_SERIAL) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_SYCL.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_SYCL.hpp deleted file mode 100644 index 72a26b27f4..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_SYCL.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_SYCL_HPP -#define KOKKOS_DECLARE_SYCL_HPP - -#if defined(KOKKOS_ENABLE_SYCL) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_THREADS.hpp b/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_THREADS.hpp deleted file mode 100644 index adb8f12a9c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/decl/Kokkos_Declare_THREADS.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DECLARE_THREADS_HPP -#define KOKKOS_DECLARE_THREADS_HPP - -#if defined(KOKKOS_ENABLE_THREADS) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/dummy.cpp b/bundled/kokkos-3.7.00/core/src/dummy.cpp deleted file mode 100644 index 4f5f14e7d3..0000000000 --- a/bundled/kokkos-3.7.00/core/src/dummy.cpp +++ /dev/null @@ -1,10 +0,0 @@ - - -namespace Kokkos { -namespace AvoidCompilerWarnings { -int dontComplain() { - // keep the compiler from complaining about emptiness - return 0; -} -} // namespace AvoidCompilerWarnings -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_CUDA.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_CUDA.hpp deleted file mode 100644 index 4bda5e9411..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_CUDA.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_FWD_HPP_ -#define KOKKOS_CUDA_FWD_HPP_ -#if defined(KOKKOS_ENABLE_CUDA) -namespace Kokkos { - -class CudaSpace; ///< Memory space on Cuda GPU -class CudaUVMSpace; ///< Memory space on Cuda GPU with UVM -class CudaHostPinnedSpace; ///< Memory space on Host accessible to Cuda GPU -class Cuda; ///< Execution space for Cuda GPU - -namespace Impl { - -template -void cuda_prefetch_pointer(const ExecSpace& /*space*/, const void* /*ptr*/, - size_t /*bytes*/, bool /*to_device*/) {} - -void cuda_prefetch_pointer(const Cuda& space, const void* ptr, size_t bytes, - bool to_device); - -} // namespace Impl -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HBWSpace.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HBWSpace.hpp deleted file mode 100644 index d9dada27a0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HBWSpace.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HBWSPACE_FWD_HPP_ -#define KOKKOS_HBWSPACE_FWD_HPP_ - -#ifdef KOKKOS_ENABLE_HBWSPACE -namespace Kokkos { - -namespace Experimental { -class HBWSpace; /// Memory space for hbw_malloc from memkind (e.g. for KNL - /// processor) -} // namespace Experimental -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HIP.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HIP.hpp deleted file mode 100644 index 27540865a5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HIP.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HIP_FWD_HPP_ -#define KOKKOS_HIP_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_HIP) -namespace Kokkos { -namespace Experimental { -class HIPSpace; ///< Memory space on HIP GPU -class HIPHostPinnedSpace; ///< Memory space on Host accessible to HIP GPU -class HIPManagedSpace; ///< Memory migratable between Host and HIP GPU -class HIP; ///< Execution space for HIP GPU -} // namespace Experimental -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HPX.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HPX.hpp deleted file mode 100644 index 8949c527fc..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_HPX.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_HPX_FWD_HPP_ -#define KOKKOS_HPX_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_HPX) -namespace Kokkos { -namespace Experimental { -class HPX; ///< Execution space with HPX back-end. -} // namespace Experimental -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENACC.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENACC.hpp deleted file mode 100644 index d733f993d0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENACC.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENACC_FWD_HPP_ -#define KOKKOS_OPENACC_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_OPENACC) -namespace Kokkos { -namespace Experimental { -class OpenACC; ///< OpenACC execution space. -class OpenACCSpace; -} // namespace Experimental -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENMP.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENMP.hpp deleted file mode 100644 index fc2223d3e2..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENMP.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMP_FWD_HPP_ -#define KOKKOS_OPENMP_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_OPENMP) -namespace Kokkos { -class OpenMP; ///< OpenMP execution space. -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENMPTARGET.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENMPTARGET.hpp deleted file mode 100644 index 8d12b8b701..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_OPENMPTARGET.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_OPENMPTARGET_FWD_HPP_ -#define KOKKOS_OPENMPTARGET_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_OPENMPTARGET) -namespace Kokkos { -namespace Experimental { -class OpenMPTarget; ///< OpenMPTarget execution space. -class OpenMPTargetSpace; -} // namespace Experimental -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_SERIAL.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_SERIAL.hpp deleted file mode 100644 index 8f253d0a75..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_SERIAL.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SERIAL_FWD_HPP_ -#define KOKKOS_SERIAL_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_SERIAL) -namespace Kokkos { -class Serial; ///< Execution space main process on CPU. -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_SYCL.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_SYCL.hpp deleted file mode 100644 index 0ce680cd69..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_SYCL.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SYCL_FWD_HPP_ -#define KOKKOS_SYCL_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_SYCL) -namespace Kokkos { -namespace Experimental { -class SYCLDeviceUSMSpace; ///< Memory space on SYCL device, not accessible from - ///< the host -class SYCLSharedUSMSpace; ///< Memory space accessible from both the SYCL - ///< device and the host -class SYCLHostUSMSpace; ///< Memory space accessible from both the SYCL - ///< device and the host (host pinned) -class SYCL; ///< Execution space for SYCL -} // namespace Experimental -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_THREADS.hpp b/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_THREADS.hpp deleted file mode 100644 index af53777dde..0000000000 --- a/bundled/kokkos-3.7.00/core/src/fwd/Kokkos_Fwd_THREADS.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_THREADS_FWD_HPP_ -#define KOKKOS_THREADS_FWD_HPP_ - -#if defined(KOKKOS_ENABLE_THREADS) -namespace Kokkos { -class Threads; ///< Execution space with C++11 threads back-end. -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/KokkosExp_ViewMapping.hpp b/bundled/kokkos-3.7.00/core/src/impl/KokkosExp_ViewMapping.hpp deleted file mode 100644 index a41ffb1e8a..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/KokkosExp_ViewMapping.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -// Deprecated file for backward compatibility - -#include diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Assembly.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Assembly.hpp deleted file mode 100644 index a31dd1cf49..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Assembly.hpp +++ /dev/null @@ -1,107 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_ASSEMBLY_HPP) -#define KOKKOS_ATOMIC_ASSEMBLY_HPP -namespace Kokkos { - -namespace Impl { - -#if !defined(_WIN32) -struct cas128_t { - uint64_t lower; - uint64_t upper; - - KOKKOS_INLINE_FUNCTION - cas128_t() { - lower = 0; - upper = 0; - } - - KOKKOS_INLINE_FUNCTION - cas128_t(const cas128_t& a) { - lower = a.lower; - upper = a.upper; - } - KOKKOS_INLINE_FUNCTION - cas128_t(volatile cas128_t* a) { - lower = a->lower; - upper = a->upper; - } - - KOKKOS_INLINE_FUNCTION - bool operator!=(const cas128_t& a) const { - return (lower != a.lower) || upper != a.upper; - } - - KOKKOS_INLINE_FUNCTION - void operator=(const cas128_t& a) { - lower = a.lower; - upper = a.upper; - } - KOKKOS_INLINE_FUNCTION - void operator=(const cas128_t& a) volatile { - lower = a.lower; - upper = a.upper; - } -} __attribute__((__aligned__(16))); -#endif - -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) -inline cas128_t cas128(volatile cas128_t* ptr, cas128_t cmp, cas128_t swap) { - bool swapped = false; - __asm__ __volatile__( - "lock cmpxchg16b %1\n\t" - "setz %0" - : "=q"(swapped), "+m"(*ptr), "+d"(cmp.upper), "+a"(cmp.lower) - : "c"(swap.upper), "b"(swap.lower), "q"(swapped)); - return cmp; -} -#endif - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp deleted file mode 100644 index e203c0a2bd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp +++ /dev/null @@ -1,437 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && \ - !defined(KOKKOS_ATOMIC_COMPARE_EXCHANGE_STRONG_HPP) -#define KOKKOS_ATOMIC_COMPARE_EXCHANGE_STRONG_HPP - -#include -#include - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#endif - -namespace Kokkos { - -//---------------------------------------------------------------------------- -// Cuda native CAS supports int, unsigned int, and unsigned long long int -// (non-standard type). Must cast-away 'volatile' for the CAS call. - -#if defined(KOKKOS_ENABLE_CUDA) - -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -__inline__ __device__ int atomic_compare_exchange(volatile int* const dest, - const int compare, - const int val) { - return atomicCAS((int*)dest, compare, val); -} - -__inline__ __device__ unsigned int atomic_compare_exchange( - volatile unsigned int* const dest, const unsigned int compare, - const unsigned int val) { - return atomicCAS((unsigned int*)dest, compare, val); -} - -__inline__ __device__ unsigned long long int atomic_compare_exchange( - volatile unsigned long long int* const dest, - const unsigned long long int compare, const unsigned long long int val) { - return atomicCAS((unsigned long long int*)dest, compare, val); -} - -template -__inline__ __device__ T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - const int tmp = atomicCAS((int*)dest, *((int*)&compare), *((int*)&val)); - return *((T*)&tmp); -} - -template -__inline__ __device__ T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t - val) { - using type = unsigned long long int; - const type tmp = atomicCAS((type*)dest, *((type*)&compare), *((type*)&val)); - return *((T*)&tmp); -} - -template -__inline__ __device__ T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8), const T>& val) { - T return_val; - // This is a way to (hopefully) avoid dead lock in a warp - int done = 0; - unsigned int mask = __activemask(); - unsigned int active = __ballot_sync(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda_space((void*)dest)) { - Kokkos::memory_fence(); - return_val = *dest; - if (return_val == compare) *dest = val; - Kokkos::memory_fence(); - Impl::unlock_address_cuda_space((void*)dest); - done = 1; - } - } - done_active = __ballot_sync(mask, done); - } - return return_val; -} -#endif -#endif - -//---------------------------------------------------------------------------- -// GCC native CAS supports int, long, unsigned int, unsigned long. -// Intel native CAS support int and long with the same interface as GCC. -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_WINDOWS_ATOMICS) -// atomic_compare_exchange are already defined in Kokkos_Atomic_Windows.hpp -#elif defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -inline int atomic_compare_exchange(volatile int* const dest, const int compare, - const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_val_compare_and_swap(dest, compare, val); -} - -inline long atomic_compare_exchange(volatile long* const dest, - const long compare, const long val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_val_compare_and_swap(dest, compare, val); -} - -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) - -// GCC supports unsigned - -inline unsigned int atomic_compare_exchange(volatile unsigned int* const dest, - const unsigned int compare, - const unsigned int val) { - return __sync_val_compare_and_swap(dest, compare, val); -} - -inline unsigned long atomic_compare_exchange(volatile unsigned long* const dest, - const unsigned long compare, - const unsigned long val) { - return __sync_val_compare_and_swap(dest, compare, val); -} - -inline unsigned long long atomic_compare_exchange( - volatile unsigned long long* const dest, const unsigned long long compare, - const unsigned long long val) { - return __sync_val_compare_and_swap(dest, compare, val); -} - -#endif - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - union U { - int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - tmp.i = - __sync_val_compare_and_swap((int*)dest, *((int*)&compare), *((int*)&val)); - return tmp.t; -} - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t - val) { - union U { - long i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - tmp.i = __sync_val_compare_and_swap((long*)dest, *((long*)&compare), - *((long*)&val)); - return tmp.t; -} - -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t - val) { - union U { - Impl::cas128_t i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - tmp.i = Impl::cas128((Impl::cas128_t*)dest, *((Impl::cas128_t*)&compare), - *((Impl::cas128_t*)&val)); - return tmp.t; -} -#endif - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T compare, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8) -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - && (sizeof(T) != 16) -#endif - , - const T>& val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = *dest; - if (return_val == compare) { - // Don't use the following line of code here: - // - // const T tmp = *dest = val; - // - // Instead, put each assignment in its own statement. This is - // because the overload of T::operator= for volatile *this should - // return void, not volatile T&. See Kokkos #177: - // - // https://github.com/kokkos/kokkos/issues/177 - *dest = val; - const T tmp = *dest; -#ifndef KOKKOS_COMPILER_CLANG - (void)tmp; -#endif - Kokkos::memory_fence(); - } - Impl::unlock_address_host_space((void*)dest); - return return_val; -} -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -KOKKOS_INLINE_FUNCTION T atomic_compare_exchange(volatile T* const dest, - const T compare, const T val) { - T retval; -#pragma omp critical - { - retval = dest[0]; - if (retval == compare) dest[0] = val; - } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -KOKKOS_INLINE_FUNCTION T atomic_compare_exchange(volatile T* const dest_v, - const T compare, const T val) { - T* dest = const_cast(dest_v); - T retval = *dest; - if (retval == compare) *dest = val; - return retval; -} - -#endif -#endif - -// dummy for non-CUDA Kokkos headers being processed by NVCC -#if defined(__CUDA_ARCH__) && !defined(KOKKOS_ENABLE_CUDA) -template -__inline__ __device__ T -atomic_compare_exchange(volatile T* const, const Kokkos::Impl::identity_t, - const Kokkos::Impl::identity_t) { - return T(); -} -#endif - -template -KOKKOS_INLINE_FUNCTION bool atomic_compare_exchange_strong( - volatile T* const dest, const T compare, const T val) { - return compare == atomic_compare_exchange(dest, compare, val); -} -//---------------------------------------------------------------------------- - -namespace Impl { -// memory-ordered versions are in the Impl namespace - -template -KOKKOS_INLINE_FUNCTION bool _atomic_compare_exchange_strong_fallback( - T* dest, T compare, T val, memory_order_seq_cst_t, MemoryOrderFailure) { - Kokkos::memory_fence(); - auto rv = Kokkos::atomic_compare_exchange_strong(dest, compare, val); - Kokkos::memory_fence(); - return rv; -} - -template -KOKKOS_INLINE_FUNCTION bool _atomic_compare_exchange_strong_fallback( - T* dest, T compare, T val, memory_order_acquire_t, MemoryOrderFailure) { - auto rv = Kokkos::atomic_compare_exchange_strong(dest, compare, val); - Kokkos::memory_fence(); - return rv; -} - -template -KOKKOS_INLINE_FUNCTION bool _atomic_compare_exchange_strong_fallback( - T* dest, T compare, T val, memory_order_release_t, MemoryOrderFailure) { - Kokkos::memory_fence(); - return Kokkos::atomic_compare_exchange_strong(dest, compare, val); -} - -template -KOKKOS_INLINE_FUNCTION bool _atomic_compare_exchange_strong_fallback( - T* dest, T compare, T val, memory_order_relaxed_t, MemoryOrderFailure) { - return Kokkos::atomic_compare_exchange_strong(dest, compare, val); -} - -#if (defined(KOKKOS_ENABLE_GNU_ATOMICS) && !defined(__CUDA_ARCH__)) || \ - (defined(KOKKOS_ENABLE_INTEL_ATOMICS) && !defined(__CUDA_ARCH__)) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - -#if defined(__CUDA_ARCH__) -#define KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH __inline__ __device__ -#else -#define KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH inline -#endif - -template -KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH bool _atomic_compare_exchange_strong( - T* dest, T compare, T val, MemoryOrderSuccess, MemoryOrderFailure, - std::enable_if_t< - (sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8 || - sizeof(T) == 16) && - std::is_same>::value && - std::is_same>::value, - void const**> = nullptr) { - return __atomic_compare_exchange_n(dest, &compare, val, /* weak = */ false, - MemoryOrderSuccess::gnu_constant, - MemoryOrderFailure::gnu_constant); -} - -template -KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH bool _atomic_compare_exchange_strong( - T* dest, T compare, T val, MemoryOrderSuccess order_success, - MemoryOrderFailure order_failure, - std::enable_if_t< - !(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || - sizeof(T) == 8 || sizeof(T) == 16) && - std::is_same>::value && - std::is_same>::value, - void const**> = nullptr) { - return _atomic_compare_exchange_fallback(dest, compare, val, order_success, - order_failure); -} - -#else - -template -KOKKOS_INLINE_FUNCTION bool _atomic_compare_exchange_strong( - T* dest, T compare, T val, MemoryOrderSuccess order_success, - MemoryOrderFailure order_failure) { - return _atomic_compare_exchange_strong_fallback(dest, compare, val, - order_success, order_failure); -} - -#endif - -// TODO static asserts in overloads that don't make sense (as listed in -// https://gcc.gnu.org/onlinedocs/gcc-5.2.0/gcc/_005f_005fatomic-Builtins.html) -template -KOKKOS_FORCEINLINE_FUNCTION bool atomic_compare_exchange_strong( - T* dest, T compare, T val, MemoryOrderSuccess order_success, - MemoryOrderFailure order_failure) { - return _atomic_compare_exchange_strong(dest, compare, val, order_success, - order_failure); -} - -} // end namespace Impl - -} // namespace Kokkos - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Compare_Exchange_Weak.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Compare_Exchange_Weak.hpp deleted file mode 100644 index ad5b010558..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Compare_Exchange_Weak.hpp +++ /dev/null @@ -1,408 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#include -#ifndef KOKKOS_ATOMIC_COMPARE_EXCHANGE_WEAK_HPP -#define KOKKOS_ATOMIC_COMPARE_EXCHANGE_WEAK_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -// Cuda sm_70 or greater supports C++-like semantics directly - -#if defined(KOKKOS_ENABLE_CUDA) - -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -#if __CUDA_ARCH__ >= 700 -// See: https://github.com/ogiroux/freestanding -#define kokkos_cuda_internal_cas_release_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.release.sys.b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define kokkos_cuda_internal_cas_acquire_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.acquire.sys.b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define kokkos_cuda_internal_cas_acq_rel_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.acq_rel.sys.b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define kokkos_cuda_internal_cas_relaxed_32(ptr, old, expected, desired) \ - asm volatile("atom.cas.relaxed.sys.b32 %0, [%1], %2, %3;" \ - : "=r"(old) \ - : "l"(ptr), "r"(expected), "r"(desired) \ - : "memory") -#define kokkos_cuda_internal_fence_seq_cst() \ - asm volatile("fence.sc.sys;" : : : "memory") -#define kokkos_cuda_internal_fence_acq_rel() \ - asm volatile("fence.acq_rel.sys;" : : : "memory") -#else -#define kokkos_cuda_internal_fence_acq_rel() \ - asm volatile("membar.sys;" : : : "memory") -#define kokkos_cuda_internal_fence_seq_cst() \ - asm volatile("membar.sys;" : : : "memory") -#endif - -// 32-bit version -template = 0> -__inline__ __device__ bool atomic_compare_exchange_weak( - T volatile* const dest, T* const expected, T const desired, - std::memory_order success_order = std::memory_order_seq_cst, - std::memory_order failure_order = std::memory_order_seq_cst) { - // TODO assert that success_order >= failure_order - // See: https://github.com/ogiroux/freestanding - int32_t tmp = 0; - int32_t old = 0; - memcpy(&tmp, &desired, sizeof(T)); - memcpy(&old, expected, sizeof(T)); - int32_t old_tmp = old; -#if __CUDA_ARCH__ >= 700 - switch (success_order) { - case std::memory_order_seq_cst: - // sequentially consistent is just an acquire with a seq_cst fence - kokkos_cuda_internal_fence_seq_cst(); - kokkos_cuda_internal_cas_acquire_32((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_acquire: - kokkos_cuda_internal_cas_acquire_32((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_consume: - // same as acquire on PTX compatible platforms - kokkos_cuda_internal_cas_acquire_32((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_acq_rel: - kokkos_cuda_internal_cas_acq_rel_32((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_release: - kokkos_cuda_internal_cas_release_32((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_relaxed: - kokkos_cuda_internal_cas_relaxed_32((T*)dest, old, old_tmp, tmp); - break; - }; -#else - // All of the orders that require a fence before the relaxed atomic operation: - if (success_order == std::memory_order_release || - success_order == std::memory_order_acq_rel) { - kokkos_cuda_internal_fence_acq_rel(); - } else if (success_order == std::memory_order_seq_cst) { - kokkos_cuda_internal_fence_seq_cst(); - } - // This is relaxed: - // Cuda API requires casting away volatile - atomicCAS((T*)dest, old_tmp, tmp); -#endif - bool const rv = (old == old_tmp); -#if __CUDA_ARCH__ < 700 - if (rv) { - if (success_order == std::memory_order_acquire || - success_order == std::memory_order_consume || - success_order == std::memory_order_acq_rel) { - kokkos_cuda_internal_fence_acq_rel(); - } else if (success_order == std::memory_order_seq_cst) { - kokkos_cuda_internal_fence_seq_cst(); - } - } else { - if (failure_order == std::memory_order_acquire || - failure_order == std::memory_order_consume || - failure_order == std::memory_order_acq_rel) { - kokkos_cuda_internal_fence_acq_rel(); - } else if (failure_order == std::memory_order_seq_cst) { - kokkos_cuda_internal_fence_seq_cst(); - } - } -#endif - memcpy(expected, &old, sizeof(T)); - return rv; -} - -// 64-bit version -template = 0> -bool atomic_compare_exchange_weak( - T volatile* const dest, T* const expected, T const desired, - std::memory_order success_order = std::memory_order_seq_cst, - std::memory_order failure_order = std::memory_order_seq_cst) { - // TODO assert that success_order >= failure_order - // See: https://github.com/ogiroux/freestanding - int64_t tmp = 0; - int64_t old = 0; - memcpy(&tmp, &desired, sizeof(T)); - memcpy(&old, expected, sizeof(T)); - int64_t old_tmp = old; -#if __CUDA_ARCH__ >= 700 - switch (success_order) { - case std::memory_order_seq_cst: - // sequentially consistent is just an acquire with a seq_cst fence - kokkos_cuda_internal_fence_seq_cst(); - kokkos_cuda_internal_cas_acquire_64((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_acquire: - kokkos_cuda_internal_cas_acquire_64((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_consume: - // same as acquire on PTX compatible platforms - kokkos_cuda_internal_cas_acquire_64((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_acq_rel: - kokkos_cuda_internal_cas_acq_rel_64((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_release: - kokkos_cuda_internal_cas_release_64((T*)dest, old, old_tmp, tmp); - break; - case std::memory_order_relaxed: - kokkos_cuda_internal_cas_relaxed_64((T*)dest, old, old_tmp, tmp); - break; - }; -#else - // Cuda API requires casting away volatile - atomicCAS((T*)dest, old_tmp, tmp); -#endif - bool const rv = (old == old_tmp); - memcpy(expected, &old, sizeof(T)); - return rv; -} - -#endif // defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -#endif // defined( KOKKOS_ENABLE_CUDA ) - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -// GCC native CAS supports int, long, unsigned int, unsigned long. -// Intel native CAS support int and long with the same interface as GCC. -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -inline int atomic_compare_exchange(volatile int* const dest, const int compare, - const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_val_compare_and_swap(dest, compare, val); -} - -inline long atomic_compare_exchange(volatile long* const dest, - const long compare, const long val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_val_compare_and_swap(dest, compare, val); -} - -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) - -// GCC supports unsigned - -inline unsigned int atomic_compare_exchange(volatile unsigned int* const dest, - const unsigned int compare, - const unsigned int val) { - return __sync_val_compare_and_swap(dest, compare, val); -} - -inline unsigned long atomic_compare_exchange(volatile unsigned long* const dest, - const unsigned long compare, - const unsigned long val) { - return __sync_val_compare_and_swap(dest, compare, val); -} - -inline unsigned long long atomic_compare_exchange( - volatile unsigned long long* const dest, const unsigned long long compare, - const unsigned long long val) { - return __sync_val_compare_and_swap(dest, compare, val); -} - -#endif - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - union U { - int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - tmp.i = - __sync_val_compare_and_swap((int*)dest, *((int*)&compare), *((int*)&val)); - return tmp.t; -} - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t - val) { - union U { - long i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - tmp.i = __sync_val_compare_and_swap((long*)dest, *((long*)&compare), - *((long*)&val)); - return tmp.t; -} - -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t - val) { - union U { - Impl::cas128_t i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - tmp.i = Impl::cas128((Impl::cas128_t*)dest, *((Impl::cas128_t*)&compare), - *((Impl::cas128_t*)&val)); - return tmp.t; -} -#endif - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T compare, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8) -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - && (sizeof(T) != 16) -#endif - , - const T>& val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = *dest; - if (return_val == compare) { - // Don't use the following line of code here: - // - // const T tmp = *dest = val; - // - // Instead, put each assignment in its own statement. This is - // because the overload of T::operator= for volatile *this should - // return void, not volatile T&. See Kokkos #177: - // - // https://github.com/kokkos/kokkos/issues/177 - *dest = val; - const T tmp = *dest; -#ifndef KOKKOS_COMPILER_CLANG - (void)tmp; -#endif - Kokkos::memory_fence(); - } - Impl::unlock_address_host_space((void*)dest); - return return_val; -} -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -KOKKOS_INLINE_FUNCTION T atomic_compare_exchange(volatile T* const dest, - const T compare, const T val) { - T retval; -#pragma omp critical - { - retval = dest[0]; - if (retval == compare) dest[0] = val; - } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -KOKKOS_INLINE_FUNCTION T atomic_compare_exchange(volatile T* const dest_v, - const T compare, const T val) { - T* dest = const_cast(dest_v); - T retval = *dest; - if (retval == compare) *dest = val; - return retval; -} - -#endif -#endif - -template -KOKKOS_INLINE_FUNCTION bool atomic_compare_exchange_strong( - volatile T* const dest, const T compare, const T val) { - return compare == atomic_compare_exchange(dest, compare, val); -} -//---------------------------------------------------------------------------- - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Decrement.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Decrement.hpp deleted file mode 100644 index 47961b5c71..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Decrement.hpp +++ /dev/null @@ -1,147 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_DECREMENT_HPP) -#define KOKKOS_ATOMIC_DECREMENT_HPP - -#include "impl/Kokkos_Atomic_Fetch_Sub.hpp" - -namespace Kokkos { - -// Atomic decrement -template <> -KOKKOS_INLINE_FUNCTION void atomic_decrement(volatile char* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock decb %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - char* a_nv = const_cast(a); - --(*a_nv); -#else - Kokkos::atomic_fetch_sub(a, char(1)); -#endif -} - -template <> -KOKKOS_INLINE_FUNCTION void atomic_decrement(volatile short* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock decw %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - short* a_nv = const_cast(a); - --(*a_nv); -#else - Kokkos::atomic_fetch_sub(a, short(1)); -#endif -} - -template <> -KOKKOS_INLINE_FUNCTION void atomic_decrement(volatile int* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock decl %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - int* a_nv = const_cast(a); - --(*a_nv); -#else - Kokkos::atomic_fetch_sub(a, int(1)); -#endif -} - -template <> -KOKKOS_INLINE_FUNCTION void atomic_decrement( - volatile long long int* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock decq %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - long long int* a_nv = const_cast(a); - --(*a_nv); -#else - using T = long long int; - Kokkos::atomic_fetch_sub(a, T(1)); -#endif -} - -template -KOKKOS_INLINE_FUNCTION void atomic_decrement(volatile T* a) { -#if defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - T* a_nv = const_cast(a); - --(*a_nv); -#else - Kokkos::atomic_fetch_sub(a, T(1)); -#endif -} - -} // End of namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Exchange.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Exchange.hpp deleted file mode 100644 index a8f77d8353..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Exchange.hpp +++ /dev/null @@ -1,404 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_EXCHANGE_HPP) -#define KOKKOS_ATOMIC_EXCHANGE_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- - -#if defined(KOKKOS_ENABLE_CUDA) -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -__inline__ __device__ int atomic_exchange(volatile int* const dest, - const int val) { - // return __iAtomicExch( (int*) dest , val ); - return atomicExch((int*)dest, val); -} - -__inline__ __device__ unsigned int atomic_exchange( - volatile unsigned int* const dest, const unsigned int val) { - // return __uAtomicExch( (unsigned int*) dest , val ); - return atomicExch((unsigned int*)dest, val); -} - -__inline__ __device__ unsigned long long int atomic_exchange( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - // return __ullAtomicExch( (unsigned long long*) dest , val ); - return atomicExch((unsigned long long*)dest, val); -} - -/** \brief Atomic exchange for any type with compatible size */ -template -__inline__ __device__ T -atomic_exchange(volatile T* const dest, - std::enable_if_t val) { - // int tmp = __ullAtomicExch( (int*) dest , *((int*)&val) ); -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - int tmp = atomicExch(((int*)dest), *((int*)&val)); - return *((T*)&tmp); -} - -template -__inline__ __device__ T atomic_exchange( - volatile T* const dest, - std::enable_if_t - val) { - using type = unsigned long long int; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - // type tmp = __ullAtomicExch( (type*) dest , *((type*)&val) ); - type tmp = atomicExch(((type*)dest), *((type*)&val)); - return *((T*)&tmp); -} - -template -__inline__ __device__ T atomic_exchange( - volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8), const T>& val) { - T return_val; - // This is a way to (hopefully) avoid dead lock in a warp -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - int done = 0; - unsigned int mask = __activemask(); - unsigned int active = __ballot_sync(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda_space((void*)dest)) { - Kokkos::memory_fence(); - return_val = *dest; - *dest = val; - Kokkos::memory_fence(); - Impl::unlock_address_cuda_space((void*)dest); - done = 1; - } - } - done_active = __ballot_sync(mask, done); - } - return return_val; -} -/** \brief Atomic exchange for any type with compatible size */ -template -__inline__ __device__ void atomic_assign( - volatile T* const dest, - std::enable_if_t val) { - // (void) __ullAtomicExch( (int*) dest , *((int*)&val) ); - (void)atomicExch(((int*)dest), *((int*)&val)); -} - -template -__inline__ __device__ void atomic_assign( - volatile T* const dest, - std::enable_if_t - val) { - using type = unsigned long long int; - // (void) __ullAtomicExch( (type*) dest , *((type*)&val) ); - (void)atomicExch(((type*)dest), *((type*)&val)); -} - -template -__inline__ __device__ void atomic_assign( - volatile T* const dest, - std::enable_if_t - val) { - (void)atomic_exchange(dest, val); -} - -#endif -#endif - -//---------------------------------------------------------------------------- - -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -template -inline T atomic_exchange( - volatile T* const dest, - std::enable_if_t - val) { - using type = std::conditional_t; -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - const type v = *((type*)&val); // Extract to be sure the value doesn't change - - type assumed; - - union U { - T val_T; - type val_type; - inline U() {} - } old; - - old.val_T = *dest; - - do { - assumed = old.val_type; - old.val_type = - __sync_val_compare_and_swap((volatile type*)dest, assumed, v); - } while (assumed != old.val_type); - - return old.val_T; -} - -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) -template -inline T atomic_exchange( - volatile T* const dest, - std::enable_if_t val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - union U { - Impl::cas128_t i; - T t; - inline U() {} - } assume, oldval, newval; - - oldval.t = *dest; - newval.t = val; - - do { - assume.i = oldval.i; - oldval.i = Impl::cas128((volatile Impl::cas128_t*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} -#endif - -//---------------------------------------------------------------------------- - -template -inline T atomic_exchange(volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8) -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - && (sizeof(T) != 16) -#endif - , - const T>& val) { - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = *dest; - // Don't use the following line of code here: - // - // const T tmp = *dest = val; - // - // Instead, put each assignment in its own statement. This is - // because the overload of T::operator= for volatile *this should - // return void, not volatile T&. See Kokkos #177: - // - // https://github.com/kokkos/kokkos/issues/177 - *dest = val; - const T tmp = *dest; -#ifndef KOKKOS_COMPILER_CLANG - (void)tmp; -#endif - Kokkos::memory_fence(); - Impl::unlock_address_host_space((void*)dest); - return return_val; -} - -template -inline void atomic_assign( - volatile T* const dest, - std::enable_if_t - val) { - using type = std::conditional_t; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - const type v = *((type*)&val); // Extract to be sure the value doesn't change - - type assumed; - - union U { - T val_T; - type val_type; - inline U() {} - } old; - - old.val_T = *dest; - - do { - assumed = old.val_type; - old.val_type = - __sync_val_compare_and_swap((volatile type*)dest, assumed, v); - } while (assumed != old.val_type); -} - -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) -template -inline void atomic_assign( - volatile T* const dest, - std::enable_if_t val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - union U { - Impl::cas128_t i; - T t; - inline U() {} - } assume, oldval, newval; - - oldval.t = *dest; - newval.t = val; - do { - assume.i = oldval.i; - oldval.i = Impl::cas128((volatile Impl::cas128_t*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); -} -#endif - -template -inline void atomic_assign(volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8) -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - && (sizeof(T) != 16) -#endif - , - const T>& val) { - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - // This is likely an aggregate type with a defined - // 'volatile T & operator = ( const T & ) volatile' - // member. The volatile return value implicitly defines a - // dereference that some compilers (gcc 4.7.2) warn is being ignored. - // Suppress warning by casting return to void. - //(void)( *dest = val ); - *dest = val; - Kokkos::memory_fence(); - Impl::unlock_address_host_space((void*)dest); -} -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -inline T atomic_exchange(volatile T* const dest, const T val) { - T retval; - //#pragma omp atomic capture -#pragma omp critical - { - retval = dest[0]; - dest[0] = val; - } - return retval; -} - -template -inline void atomic_assign(volatile T* const dest, const T val) { - //#pragma omp atomic -#pragma omp critical - { dest[0] = val; } -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -inline T atomic_exchange(volatile T* const dest_v, const T val) { - T* dest = const_cast(dest_v); - T retval = *dest; - *dest = val; - return retval; -} - -template -inline void atomic_assign(volatile T* const dest_v, const T val) { - T* dest = const_cast(dest_v); - *dest = val; -} - -#endif -#endif - -// dummy for non-CUDA Kokkos headers being processed by NVCC -#if defined(__CUDA_ARCH__) && !defined(KOKKOS_ENABLE_CUDA) -template -__inline__ __device__ T atomic_exchange(volatile T* const, - const Kokkos::Impl::identity_t) { - return T(); -} - -template -__inline__ __device__ void atomic_assign(volatile T* const, - const Kokkos::Impl::identity_t) {} -#endif - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Add.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Add.hpp deleted file mode 100644 index c188f45427..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Add.hpp +++ /dev/null @@ -1,389 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_FETCH_ADD_HPP) -#define KOKKOS_ATOMIC_FETCH_ADD_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- - -#if defined(KOKKOS_ENABLE_CUDA) -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -// Support for int, unsigned int, unsigned long long int, and float - -__inline__ __device__ int atomic_fetch_add(volatile int* const dest, - const int val) { - return atomicAdd((int*)dest, val); -} - -__inline__ __device__ unsigned int atomic_fetch_add( - volatile unsigned int* const dest, const unsigned int val) { - return atomicAdd((unsigned int*)dest, val); -} - -__inline__ __device__ unsigned long long int atomic_fetch_add( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return atomicAdd((unsigned long long int*)dest, val); -} - -__inline__ __device__ float atomic_fetch_add(volatile float* const dest, - const float val) { - return atomicAdd((float*)dest, val); -} - -#if (600 <= __CUDA_ARCH__) -__inline__ __device__ double atomic_fetch_add(volatile double* const dest, - const double val) { - return atomicAdd((double*)dest, val); -} -#endif - -template -__inline__ __device__ T -atomic_fetch_add(volatile T* const dest, - std::enable_if_t val) { - // to work around a bug in the clang cuda compiler, the name here needs to be - // different from the one internal to the other overloads - union U1 { - int i; - T t; - KOKKOS_INLINE_FUNCTION U1() {} - } assume, oldval, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = atomicCAS((int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -__inline__ __device__ T atomic_fetch_add( - volatile T* const dest, - std::enable_if_t - val) { - // to work around a bug in the clang cuda compiler, the name here needs to be - // different from the one internal to the other overloads - union U2 { - unsigned long long int i; - T t; - KOKKOS_INLINE_FUNCTION U2() {} - } assume, oldval, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = atomicCAS((unsigned long long int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -//---------------------------------------------------------------------------- - -template -__inline__ __device__ T atomic_fetch_add( - volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8), const T>& val) { - T return_val; - // This is a way to (hopefully) avoid dead lock in a warp - int done = 0; - unsigned int mask = __activemask(); - unsigned int active = __ballot_sync(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - bool locked = Impl::lock_address_cuda_space((void*)dest); - if (locked) { - Kokkos::memory_fence(); - return_val = *dest; - *dest = return_val + val; - Kokkos::memory_fence(); - Impl::unlock_address_cuda_space((void*)dest); - done = 1; - } - } - - done_active = __ballot_sync(mask, done); - } - return return_val; -} -#endif -#endif -//---------------------------------------------------------------------------- -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -#if defined(KOKKOS_ENABLE_ASM) && (defined(KOKKOS_ENABLE_ISA_X86_64) || \ - defined(KOKKOS_KNL_USE_ASM_WORKAROUND)) -inline int atomic_fetch_add(volatile int* dest, const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - int original = val; - - __asm__ __volatile__("lock xadd %1, %0" - : "+m"(*dest), "+r"(original) - : "m"(*dest), "r"(original) - : "memory"); - - return original; -} -#else -inline int atomic_fetch_add(volatile int* const dest, const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_add(dest, val); -} -#endif - -inline long int atomic_fetch_add(volatile long int* const dest, - const long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_add(dest, val); -} - -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) - -inline unsigned int atomic_fetch_add(volatile unsigned int* const dest, - const unsigned int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_add(dest, val); -} - -inline unsigned long int atomic_fetch_add( - volatile unsigned long int* const dest, const unsigned long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_add(dest, val); -} - -inline unsigned long long int atomic_fetch_add( - volatile unsigned long long int* const dest, - const unsigned long long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_add(dest, val); -} - -#endif - -template -inline T atomic_fetch_add( - volatile T* const dest, - std::enable_if_t val) { - union U { - int i; - T t; - inline U() {} - } assume, oldval, newval; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = __sync_val_compare_and_swap((int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -inline T atomic_fetch_add( - volatile T* const dest, - std::enable_if_t - val) { - union U { - long i; - T t; - inline U() {} - } assume, oldval, newval; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = __sync_val_compare_and_swap((long*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) -template -inline T atomic_fetch_add( - volatile T* const dest, - std::enable_if_t - val) { - union U { - Impl::cas128_t i; - T t; - inline U() {} - } assume, oldval, newval; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t + val; - oldval.i = Impl::cas128((volatile Impl::cas128_t*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} -#endif - -//---------------------------------------------------------------------------- - -template -inline T atomic_fetch_add(volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8) -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - && (sizeof(T) != 16) -#endif - , - const T>& val) { - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = *dest; - - // Don't use the following line of code here: - // - // const T tmp = *dest = return_val + val; - // - // Instead, put each assignment in its own statement. This is - // because the overload of T::operator= for volatile *this should - // return void, not volatile T&. See Kokkos #177: - // - // https://github.com/kokkos/kokkos/issues/177 - *dest = return_val + val; - const T tmp = *dest; - (void)tmp; - Kokkos::memory_fence(); - Impl::unlock_address_host_space((void*)dest); - - return return_val; -} -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -T atomic_fetch_add(volatile T* const dest, const T val) { - T retval; -#pragma omp atomic capture - { - retval = dest[0]; - dest[0] += val; - } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -T atomic_fetch_add(volatile T* const dest_v, std::add_const_t val) { - T* dest = const_cast(dest_v); - T retval = *dest; - *dest += val; - return retval; -} - -#endif -#endif -//---------------------------------------------------------------------------- - -// dummy for non-CUDA Kokkos headers being processed by NVCC -#if defined(__CUDA_ARCH__) && !defined(KOKKOS_ENABLE_CUDA) -template -__inline__ __device__ T atomic_fetch_add(volatile T* const, - Kokkos::Impl::identity_t) { - return T(); -} -#endif - -} // namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_And.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_And.hpp deleted file mode 100644 index 50bd2b0fa1..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_And.hpp +++ /dev/null @@ -1,192 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_FETCH_AND_HPP) -#define KOKKOS_ATOMIC_FETCH_AND_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- - -#if defined(KOKKOS_ENABLE_CUDA) -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -// Support for int, unsigned int, unsigned long long int, and float - -__inline__ __device__ int atomic_fetch_and(volatile int* const dest, - const int val) { - return atomicAnd((int*)dest, val); -} - -__inline__ __device__ unsigned int atomic_fetch_and( - volatile unsigned int* const dest, const unsigned int val) { - return atomicAnd((unsigned int*)dest, val); -} - -#if defined(__CUDA_ARCH__) && (350 <= __CUDA_ARCH__) -__inline__ __device__ unsigned long long int atomic_fetch_and( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return atomicAnd((unsigned long long int*)dest, val); -} -#endif -#endif -#endif - -// 08/05/20 Overload to work around https://bugs.llvm.org/show_bug.cgi?id=46922 - -#if (defined(KOKKOS_ENABLE_CUDA) && \ - (defined(__CUDA_ARCH__) || \ - defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND))) || \ - (defined(KOKKOS_ENABLE_HIP)) -__inline__ __device__ unsigned long atomic_fetch_and( - volatile unsigned long* const dest, const unsigned long val) { - return atomic_fetch_and(dest, val); -} -__inline__ __device__ long atomic_fetch_and(volatile long* const dest, - long val) { - return atomic_fetch_and(dest, val); -} -#endif - -//---------------------------------------------------------------------------- -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -inline int atomic_fetch_and(volatile int* const dest, const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_and(dest, val); -} - -inline long int atomic_fetch_and(volatile long int* const dest, - const long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_and(dest, val); -} - -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) - -inline unsigned int atomic_fetch_and(volatile unsigned int* const dest, - const unsigned int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_and(dest, val); -} - -inline unsigned long int atomic_fetch_and( - volatile unsigned long int* const dest, const unsigned long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_and(dest, val); -} - -inline unsigned long long int atomic_fetch_and( - volatile unsigned long long int* const dest, - const unsigned long long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_and(dest, val); -} - -#endif - -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -T atomic_fetch_and(volatile T* const dest, const T val) { - T retval; -#pragma omp atomic capture - { - retval = dest[0]; - dest[0] &= val; - } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -T atomic_fetch_and(volatile T* const dest_v, const T val) { - T* dest = const_cast(dest_v); - T retval = *dest; - *dest &= val; - return retval; -} - -#endif -#endif -//---------------------------------------------------------------------------- - -// dummy for non-CUDA Kokkos headers being processed by NVCC -#if defined(__CUDA_ARCH__) && !defined(KOKKOS_ENABLE_CUDA) -template -__inline__ __device__ T atomic_fetch_and(volatile T* const, - Kokkos::Impl::identity_t) { - return T(); -} -#endif - -// Simpler version of atomic_fetch_and without the fetch -template -KOKKOS_INLINE_FUNCTION void atomic_and(volatile T* const dest, const T src) { - (void)atomic_fetch_and(dest, src); -} - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Or.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Or.hpp deleted file mode 100644 index 7a04a8c7ca..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Or.hpp +++ /dev/null @@ -1,193 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_FETCH_OR_HPP) -#define KOKKOS_ATOMIC_FETCH_OR_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- - -#if defined(KOKKOS_ENABLE_CUDA) -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -// Support for int, unsigned int, unsigned long long int, and float - -__inline__ __device__ int atomic_fetch_or(volatile int* const dest, - const int val) { - return atomicOr((int*)dest, val); -} - -__inline__ __device__ unsigned int atomic_fetch_or( - volatile unsigned int* const dest, const unsigned int val) { - return atomicOr((unsigned int*)dest, val); -} - -#if defined(__CUDA_ARCH__) && (350 <= __CUDA_ARCH__) -__inline__ __device__ unsigned long long int atomic_fetch_or( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return atomicOr((unsigned long long int*)dest, val); -} -#endif -#endif -#endif - -// 08/05/20 Overload to work around https://bugs.llvm.org/show_bug.cgi?id=46922 - -#if (defined(KOKKOS_ENABLE_CUDA) && \ - (defined(__CUDA_ARCH__) || \ - defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND))) || \ - (defined(KOKKOS_ENABLE_HIP)) -__inline__ __device__ unsigned long atomic_fetch_or( - volatile unsigned long* const dest, const unsigned long val) { - return atomic_fetch_or(dest, val); -} - -__inline__ __device__ long atomic_fetch_or(volatile long* const dest, - long val) { - return atomic_fetch_or(dest, val); -} -#endif - -//---------------------------------------------------------------------------- -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -inline int atomic_fetch_or(volatile int* const dest, const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_or(dest, val); -} - -inline long int atomic_fetch_or(volatile long int* const dest, - const long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_or(dest, val); -} - -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) - -inline unsigned int atomic_fetch_or(volatile unsigned int* const dest, - const unsigned int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_or(dest, val); -} - -inline unsigned long int atomic_fetch_or(volatile unsigned long int* const dest, - const unsigned long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_or(dest, val); -} - -inline unsigned long long int atomic_fetch_or( - volatile unsigned long long int* const dest, - const unsigned long long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_or(dest, val); -} - -#endif - -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -T atomic_fetch_or(volatile T* const dest, const T val) { - T retval; -#pragma omp atomic capture - { - retval = dest[0]; - dest[0] |= val; - } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -T atomic_fetch_or(volatile T* const dest_v, const T val) { - T* dest = const_cast(dest_v); - T retval = *dest; - *dest |= val; - return retval; -} - -#endif -#endif -//---------------------------------------------------------------------------- - -// dummy for non-CUDA Kokkos headers being processed by NVCC -#if defined(__CUDA_ARCH__) && !defined(KOKKOS_ENABLE_CUDA) -template -__inline__ __device__ T atomic_fetch_or(volatile T* const, - Kokkos::Impl::identity_t) { - return T(); -} -#endif - -// Simpler version of atomic_fetch_or without the fetch -template -KOKKOS_INLINE_FUNCTION void atomic_or(volatile T* const dest, const T src) { - (void)atomic_fetch_or(dest, src); -} - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Sub.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Sub.hpp deleted file mode 100644 index 6aaf36970c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Fetch_Sub.hpp +++ /dev/null @@ -1,323 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_FETCH_SUB_HPP) -#define KOKKOS_ATOMIC_FETCH_SUB_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- - -#if defined(KOKKOS_ENABLE_CUDA) -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -// Support for int, unsigned int, unsigned long long int, and float - -__inline__ __device__ int atomic_fetch_sub(volatile int* const dest, - const int val) { - return atomicSub((int*)dest, val); -} - -__inline__ __device__ unsigned int atomic_fetch_sub( - volatile unsigned int* const dest, const unsigned int val) { - return atomicSub((unsigned int*)dest, val); -} - -__inline__ __device__ unsigned int atomic_fetch_sub( - volatile int64_t* const dest, const int64_t val) { - return atomic_fetch_add(dest, -val); -} - -__inline__ __device__ unsigned int atomic_fetch_sub(volatile float* const dest, - const float val) { - return atomicAdd((float*)dest, -val); -} - -#if (600 <= __CUDA_ARCH__) -__inline__ __device__ unsigned int atomic_fetch_sub(volatile double* const dest, - const double val) { - return atomicAdd((double*)dest, -val); -} -#endif - -template -__inline__ __device__ T -atomic_fetch_sub(volatile T* const dest, - std::enable_if_t val) { - union U { - int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t - val; - oldval.i = atomicCAS((int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -__inline__ __device__ T atomic_fetch_sub( - volatile T* const dest, - std::enable_if_t - val) { - union U { - unsigned long long int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t - val; - oldval.i = atomicCAS((unsigned long long int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -//---------------------------------------------------------------------------- - -template -__inline__ __device__ T atomic_fetch_sub( - volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8), const T>& val) { - T return_val; - // This is a way to (hopefully) avoid dead lock in a warp - int done = 0; - unsigned int mask = __activemask(); - unsigned int active = __ballot_sync(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda_space((void*)dest)) { - Kokkos::memory_fence(); - return_val = *dest; - *dest = return_val - val; - Kokkos::memory_fence(); - Impl::unlock_address_cuda_space((void*)dest); - done = 1; - } - } - done_active = __ballot_sync(mask, done); - } - return return_val; -} -#endif -#endif -//---------------------------------------------------------------------------- -#if !defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || defined(KOKKOS_ENABLE_INTEL_ATOMICS) - -inline int atomic_fetch_sub(volatile int* const dest, const int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_sub(dest, val); -} - -inline long int atomic_fetch_sub(volatile long int* const dest, - const long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_sub(dest, val); -} - -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) - -inline unsigned int atomic_fetch_sub(volatile unsigned int* const dest, - const unsigned int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_sub(dest, val); -} - -inline unsigned long int atomic_fetch_sub( - volatile unsigned long int* const dest, const unsigned long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_sub(dest, val); -} - -inline unsigned long long int atomic_fetch_sub( - volatile unsigned long long int* const dest, - const unsigned long long int val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - return __sync_fetch_and_sub(dest, val); -} - -#endif - -template -inline T atomic_fetch_sub( - volatile T* const dest, - std::enable_if_t val) { - union U { - int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t - val; - oldval.i = __sync_val_compare_and_swap((int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -inline T atomic_fetch_sub( - volatile T* const dest, - std::enable_if_t - val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - union U { - long i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - assume.i = oldval.i; - newval.t = assume.t - val; - oldval.i = __sync_val_compare_and_swap((long*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -//---------------------------------------------------------------------------- - -template -inline T atomic_fetch_sub( - volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8), const T>& val) { -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)dest, _MM_HINT_ET0); -#endif - - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = *dest; - *dest = return_val - val; - Kokkos::memory_fence(); - Impl::unlock_address_host_space((void*)dest); - return return_val; -} - -//---------------------------------------------------------------------------- - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -T atomic_fetch_sub(volatile T* const dest, const T val) { - T retval; -#pragma omp atomic capture - { - retval = dest[0]; - dest[0] -= val; - } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -T atomic_fetch_sub(volatile T* const dest_v, const T val) { - T* dest = const_cast(dest_v); - T retval = *dest; - *dest -= val; - return retval; -} - -#endif -#endif - -// dummy for non-CUDA Kokkos headers being processed by NVCC -#if defined(__CUDA_ARCH__) && !defined(KOKKOS_ENABLE_CUDA) -template -__inline__ __device__ T atomic_fetch_sub(volatile T* const, - Kokkos::Impl::identity_t) { - return T(); -} -#endif - -} // namespace Kokkos - -#include -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Generic.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Generic.hpp deleted file mode 100644 index aac0d12c81..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Generic.hpp +++ /dev/null @@ -1,555 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_GENERIC_HPP) -#define KOKKOS_ATOMIC_GENERIC_HPP -#include - -// Combination operands to be used in an Compare and Exchange based atomic -// operation -namespace Kokkos { -namespace Impl { - -template -struct _check_early_exit_impl { - KOKKOS_FORCEINLINE_FUNCTION - static constexpr bool check(Op const&, Scalar1 const&, - Scalar2 const&) noexcept { - return false; - } -}; - -template -struct _check_early_exit_impl< - Op, Scalar1, Scalar2, - decltype(std::declval().check_early_exit( - std::declval(), std::declval()))> { - KOKKOS_FORCEINLINE_FUNCTION - static constexpr bool check(Op const& op, Scalar1 const& v1, - Scalar2 const& v2) { - return op.check_early_exit(v1, v2); - } -}; - -template -KOKKOS_FORCEINLINE_FUNCTION constexpr bool check_early_exit( - Op const& op, Scalar1 const& v1, Scalar2 const& v2) noexcept { - return _check_early_exit_impl::check(op, v1, v2); -} - -template -struct MaxOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return (val1 > val2 ? val1 : val2); - } - KOKKOS_FORCEINLINE_FUNCTION - static constexpr bool check_early_exit(Scalar1 const& val1, - Scalar2 const& val2) noexcept { - return (val1 > val2); - } -}; - -template -struct MinOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return (val1 < val2 ? val1 : val2); - } - KOKKOS_FORCEINLINE_FUNCTION - static constexpr bool check_early_exit(Scalar1 const& val1, - Scalar2 const& val2) noexcept { - return (val1 < val2); - } -}; - -template -struct AddOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 + val2; - } -}; - -template -struct SubOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 - val2; - } -}; - -template -struct MulOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 * val2; - } -}; - -template -struct DivOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 / val2; - } -}; - -template -struct ModOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 % val2; - } -}; - -template -struct AndOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 & val2; - } -}; - -template -struct OrOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 | val2; - } -}; - -template -struct XorOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 ^ val2; - } -}; - -template -struct LShiftOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 << val2; - } -}; - -template -struct RShiftOper { - KOKKOS_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 >> val2; - } -}; - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_oper( - const Oper& op, volatile T* const dest, - std::enable_if_t - val) { - union U { - unsigned long long int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - if (check_early_exit(op, oldval.t, val)) return oldval.t; - assume.i = oldval.i; - newval.t = op.apply(assume.t, val); - oldval.i = Kokkos::atomic_compare_exchange((unsigned long long int*)dest, - assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -KOKKOS_INLINE_FUNCTION T atomic_oper_fetch( - const Oper& op, volatile T* const dest, - std::enable_if_t - val) { - union U { - unsigned long long int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - if (check_early_exit(op, oldval.t, val)) return oldval.t; - assume.i = oldval.i; - newval.t = op.apply(assume.t, val); - oldval.i = Kokkos::atomic_compare_exchange((unsigned long long int*)dest, - assume.i, newval.i); - } while (assume.i != oldval.i); - - return newval.t; -} - -template -KOKKOS_INLINE_FUNCTION T -atomic_fetch_oper(const Oper& op, volatile T* const dest, - std::enable_if_t val) { - union U { - int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - if (check_early_exit(op, oldval.t, val)) return oldval.t; - assume.i = oldval.i; - newval.t = op.apply(assume.t, val); - oldval.i = Kokkos::atomic_compare_exchange((int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return oldval.t; -} - -template -KOKKOS_INLINE_FUNCTION T -atomic_oper_fetch(const Oper& op, volatile T* const dest, - std::enable_if_t val) { - union U { - int i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } oldval, assume, newval; - - oldval.t = *dest; - - do { - if (check_early_exit(op, oldval.t, val)) return oldval.t; - assume.i = oldval.i; - newval.t = op.apply(assume.t, val); - oldval.i = Kokkos::atomic_compare_exchange((int*)dest, assume.i, newval.i); - } while (assume.i != oldval.i); - - return newval.t; -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_oper( - const Oper& op, volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8), const T> val) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = *dest; - *dest = op.apply(return_val, val); - Kokkos::memory_fence(); - Impl::unlock_address_host_space((void*)dest); - return return_val; -#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA) - // This is a way to (hopefully) avoid dead lock in a warp - T return_val; - int done = 0; - unsigned int mask = __activemask(); - unsigned int active = __ballot_sync(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda_space((void*)dest)) { - Kokkos::memory_fence(); - return_val = *dest; - *dest = op.apply(return_val, val); - Kokkos::memory_fence(); - Impl::unlock_address_cuda_space((void*)dest); - done = 1; - } - } - done_active = __ballot_sync(mask, done); - } - return return_val; -#elif defined(__HIP_DEVICE_COMPILE__) - T return_val = *dest; - int done = 0; - unsigned int active = __ballot(1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip_space((void*)dest)) { - return_val = *dest; - *dest = op.apply(return_val, val); - Impl::unlock_address_hip_space((void*)dest); - done = 1; - } - } - done_active = __ballot(done); - } - return return_val; -#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL) - // FIXME_SYCL - Kokkos::abort("Not implemented!"); - (void)op; - (void)dest; - (void)val; - return 0; -#endif -} - -template -KOKKOS_INLINE_FUNCTION T -atomic_oper_fetch(const Oper& op, volatile T* const dest, - std::enable_if_t<(sizeof(T) != 4) && (sizeof(T) != 8) -#if defined(KOKKOS_ENABLE_ASM) && \ - defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST) - && (sizeof(T) != 16) -#endif - , - const T>& val) { - -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST - while (!Impl::lock_address_host_space((void*)dest)) - ; - Kokkos::memory_fence(); - T return_val = op.apply(*dest, val); - *dest = return_val; - Kokkos::memory_fence(); - Impl::unlock_address_host_space((void*)dest); - return return_val; -#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA) - T return_val; - // This is a way to (hopefully) avoid dead lock in a warp - int done = 0; - unsigned int mask = __activemask(); - unsigned int active = __ballot_sync(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda_space((void*)dest)) { - Kokkos::memory_fence(); - return_val = op.apply(*dest, val); - *dest = return_val; - Kokkos::memory_fence(); - Impl::unlock_address_cuda_space((void*)dest); - done = 1; - } - } - done_active = __ballot_sync(mask, done); - } - return return_val; -#elif defined(__HIP_DEVICE_COMPILE__) - T return_val; - int done = 0; - unsigned int active = __ballot(1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip_space((void*)dest)) { - return_val = op.apply(*dest, val); - *dest = return_val; - Impl::unlock_address_hip_space((void*)dest); - done = 1; - } - } - done_active = __ballot(done); - } - return return_val; -#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL) - // FIXME_SYCL - std::abort(); - (void)op; - (void)dest; - (void)val; - return 0; -#endif -} - -} // namespace Impl -} // namespace Kokkos - -namespace Kokkos { - -// Fetch_Oper atomics: return value before operation -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_max(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::MaxOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_min(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::MinOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_mul(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::MulOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_div(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::DivOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_mod(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::ModOper(), dest, val); -} - -#if !defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_and(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::AndOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_or(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::OrOper(), dest, val); -} - -#endif - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_xor(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::XorOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_lshift(volatile T* const dest, - const unsigned int val) { - return Impl::atomic_fetch_oper(Impl::LShiftOper(), - dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_rshift(volatile T* const dest, - const unsigned int val) { - return Impl::atomic_fetch_oper(Impl::RShiftOper(), - dest, val); -} - -// Oper Fetch atomics: return value after operation -template -KOKKOS_INLINE_FUNCTION T atomic_max_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::MaxOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_min_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::MinOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_mul_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::MulOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_div_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::DivOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_mod_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::ModOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_and_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::AndOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_or_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::OrOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_xor_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::XorOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_lshift_fetch(volatile T* const dest, - const unsigned int val) { - return Impl::atomic_oper_fetch(Impl::LShiftOper(), - dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_rshift_fetch(volatile T* const dest, - const unsigned int val) { - return Impl::atomic_oper_fetch(Impl::RShiftOper(), - dest, val); -} - -#ifdef _WIN32 -template -KOKKOS_INLINE_FUNCTION T atomic_add_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::AddOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_sub_fetch(volatile T* const dest, const T val) { - return Impl::atomic_oper_fetch(Impl::SubOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_add(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::AddOper(), dest, val); -} - -template -KOKKOS_INLINE_FUNCTION T atomic_fetch_sub(volatile T* const dest, const T val) { - return Impl::atomic_fetch_oper(Impl::SubOper(), dest, val); -} -#endif - -} // namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Generic_Secondary.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Generic_Secondary.hpp deleted file mode 100644 index 7ab6358434..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Generic_Secondary.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_GENERIC_SECONDARY_HPP) -#define KOKKOS_ATOMIC_GENERIC_SECONDARY_HPP -#include - -namespace Kokkos { - -#ifndef KOKKOS_ENABLE_SERIAL_ATOMICS -template -KOKKOS_INLINE_FUNCTION T atomic_exchange(volatile T* const dest, const T val) { - T oldval = *dest; - T assume; - do { - assume = oldval; - oldval = atomic_compare_exchange(dest, assume, val); - } while (assume != oldval); - - return oldval; -} -#endif - -template -KOKKOS_INLINE_FUNCTION void atomic_add(volatile T* const dest, const T val) { - (void)atomic_fetch_add(dest, val); -} - -template -KOKKOS_INLINE_FUNCTION void atomic_sub(volatile T* const dest, const T val) { - (void)atomic_fetch_sub(dest, val); -} - -template -KOKKOS_INLINE_FUNCTION void atomic_mul(volatile T* const dest, const T val) { - (void)atomic_fetch_mul(dest, val); -} - -template -KOKKOS_INLINE_FUNCTION void atomic_div(volatile T* const dest, const T val) { - (void)atomic_fetch_div(dest, val); -} - -} // namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Increment.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Increment.hpp deleted file mode 100644 index 65630aa84c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Increment.hpp +++ /dev/null @@ -1,147 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) -#include -#endif - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_INCREMENT_HPP) -#define KOKKOS_ATOMIC_INCREMENT_HPP - -namespace Kokkos { - -// Atomic increment -template <> -KOKKOS_INLINE_FUNCTION void atomic_increment(volatile char* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock incb %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - char* a_nv = const_cast(a); - ++(*a_nv); -#else - Kokkos::atomic_fetch_add(a, char(1)); -#endif -} - -template <> -KOKKOS_INLINE_FUNCTION void atomic_increment(volatile short* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock incw %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - short* a_nv = const_cast(a); - ++(*a_nv); -#else - Kokkos::atomic_fetch_add(a, short(1)); -#endif -} - -#ifndef _WIN32 -template <> -KOKKOS_INLINE_FUNCTION void atomic_increment(volatile int* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock incl %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - int* a_nv = const_cast(a); - ++(*a_nv); -#else - Kokkos::atomic_fetch_add(a, int(1)); -#endif -} -#endif - -template <> -KOKKOS_INLINE_FUNCTION void atomic_increment( - volatile long long int* a) { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) && \ - !defined(_WIN32) && !defined(__CUDA_ARCH__) -#if defined(KOKKOS_ENABLE_RFO_PREFETCH) - _mm_prefetch((const char*)a, _MM_HINT_ET0); -#endif - __asm__ __volatile__("lock incq %0" - : /* no output registers */ - : "m"(a[0]) - : "memory"); -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - long long int* a_nv = const_cast(a); - ++(*a_nv); -#else - using T = long long int; - Kokkos::atomic_fetch_add(a, T(1)); -#endif -} - -template -KOKKOS_INLINE_FUNCTION void atomic_increment(volatile T* a) { -#if defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - T* a_nv = const_cast(a); - ++(*a_nv); -#else - Kokkos::atomic_fetch_add(a, T(1)); -#endif -} - -} // End of namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Load.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Load.hpp deleted file mode 100644 index f4437326dd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Load.hpp +++ /dev/null @@ -1,228 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_KOKKOS_ATOMIC_LOAD_HPP -#define KOKKOS_IMPL_KOKKOS_ATOMIC_LOAD_HPP - -#include -#if defined(KOKKOS_ATOMIC_HPP) - -#include -#include - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#endif - -namespace Kokkos { -namespace Impl { - -// Olivier's implementation helpfully binds to the same builtins as GNU, so -// we make this code common across multiple options -#if (defined(KOKKOS_ENABLE_GNU_ATOMICS) && !defined(__CUDA_ARCH__)) || \ - (defined(KOKKOS_ENABLE_INTEL_ATOMICS) && !defined(__CUDA_ARCH__)) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - -#if defined(__CUDA_ARCH__) && defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) -#define KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH __inline__ __device__ -#else -#define KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH inline -#endif - -template -KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH T _atomic_load( - T* ptr, MemoryOrder, - std::enable_if_t<(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || - sizeof(T) == 8) && - std::is_same>::value, - void const**> = nullptr) { - return __atomic_load_n(ptr, MemoryOrder::gnu_constant); -} - -template -KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH T _atomic_load( - T* ptr, MemoryOrder, - std::enable_if_t::value && - std::is_same>::value, - void const**> = nullptr) { - T rv{}; - __atomic_load(ptr, &rv, MemoryOrder::gnu_constant); - return rv; -} - -#undef KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH - -#elif defined(__CUDA_ARCH__) - -// Not compiling for Volta or later, or Cuda ASM atomics were manually disabled - -template -__device__ __inline__ T _relaxed_atomic_load_impl( - T* ptr, std::enable_if_t<(sizeof(T) == 1 || sizeof(T) == 2 || - sizeof(T) == 4 || sizeof(T) == 8), - void const**> = nullptr) { - return *ptr; -} - -template -struct NoOpOper { - __device__ __inline__ static constexpr T apply(T const& t, - T const&) noexcept { - return t; - } -}; - -template -__device__ __inline__ T _relaxed_atomic_load_impl( - T* ptr, std::enable_if_t = nullptr) { - T rv{}; - // TODO remove a copy operation here? - return Kokkos::Impl::atomic_oper_fetch(NoOpOper{}, ptr, rv); -} - -template -__device__ __inline__ T _atomic_load(T* ptr, memory_order_seq_cst_t) { - Kokkos::memory_fence(); - T rv = Impl::_relaxed_atomic_load_impl(ptr); - Kokkos::memory_fence(); - return rv; -} - -template -__device__ __inline__ T _atomic_load(T* ptr, memory_order_acquire_t) { - T rv = Impl::_relaxed_atomic_load_impl(ptr); - Kokkos::memory_fence(); - return rv; -} - -template -__device__ __inline__ T _atomic_load(T* ptr, memory_order_relaxed_t) { - return _relaxed_atomic_load_impl(ptr); -} - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -inline T _atomic_load(T* ptr, MemoryOrder) { - // AFAICT, all OpenMP atomics are sequentially consistent, so memory order - // doesn't matter - T retval{}; -#pragma omp atomic read - { retval = *ptr; } - return retval; -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -inline T _atomic_load(T* ptr, MemoryOrder) { - return *ptr; -} - -#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS) - -template -inline T _atomic_load(T* ptr, MemoryOrder) { - atomic_compare_exchange(ptr, 0, 0); - return *ptr; -} - -#endif // end of all atomic implementations - -template -KOKKOS_FORCEINLINE_FUNCTION T atomic_load(T* ptr, - Impl::memory_order_seq_cst_t) { - return _atomic_load(ptr, Impl::memory_order_seq_cst); -} - -template -KOKKOS_FORCEINLINE_FUNCTION T atomic_load(T* ptr, - Impl::memory_order_acquire_t) { - return _atomic_load(ptr, Impl::memory_order_acquire); -} - -template -KOKKOS_FORCEINLINE_FUNCTION T atomic_load(T* ptr, - Impl::memory_order_relaxed_t) { - return _atomic_load(ptr, Impl::memory_order_relaxed); -} - -template -KOKKOS_FORCEINLINE_FUNCTION T atomic_load(T* /*ptr*/, - Impl::memory_order_release_t) { - static_assert( - sizeof(T) == 0, // just something that will always be false, but only on - // instantiation - "atomic_load with memory order release doesn't make any sense!"); -} - -template -KOKKOS_FORCEINLINE_FUNCTION T atomic_load(T* /*ptr*/, - Impl::memory_order_acq_rel_t) { - static_assert( - sizeof(T) == 0, // just something that will always be false, but only on - // instantiation - "atomic_load with memory order acq_rel doesn't make any sense!"); -} - -template -KOKKOS_FORCEINLINE_FUNCTION T atomic_load(T* ptr) { - // relaxed by default! - return _atomic_load(ptr, Impl::memory_order_relaxed); -} - -} // end namespace Impl -} // end namespace Kokkos - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#endif - -#endif // defined(KOKKOS_ATOMIC_HPP) -#endif // KOKKOS_IMPL_KOKKOS_ATOMIC_LOAD_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Memory_Order.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Memory_Order.hpp deleted file mode 100644 index 72a6dfa9a4..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Memory_Order.hpp +++ /dev/null @@ -1,121 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_KOKKOS_ATOMIC_MEMORY_ORDER_HPP -#define KOKKOS_KOKKOS_ATOMIC_MEMORY_ORDER_HPP - -#include - -#include - -namespace Kokkos { -namespace Impl { - -/** @file - * Provides strongly-typed analogs of the standard memory order enumerators. - * In addition to (very slightly) reducing the constant propagation burden on - * the compiler, this allows us to give compile-time errors for things that - * don't make sense, like atomic_load with memory order release. - */ - -struct memory_order_seq_cst_t { - using memory_order = memory_order_seq_cst_t; -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || \ - defined(KOKKOS_ENABLE_INTEL_ATOMICS) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - static constexpr auto gnu_constant = __ATOMIC_SEQ_CST; -#endif - static constexpr auto std_constant = std::memory_order_seq_cst; -}; -constexpr memory_order_seq_cst_t memory_order_seq_cst = {}; - -struct memory_order_relaxed_t { - using memory_order = memory_order_relaxed_t; -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || \ - defined(KOKKOS_ENABLE_INTEL_ATOMICS) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - static constexpr auto gnu_constant = __ATOMIC_RELAXED; -#endif - static constexpr auto std_constant = std::memory_order_relaxed; -}; -constexpr memory_order_relaxed_t memory_order_relaxed = {}; - -struct memory_order_acquire_t { - using memory_order = memory_order_acquire_t; -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || \ - defined(KOKKOS_ENABLE_INTEL_ATOMICS) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - static constexpr auto gnu_constant = __ATOMIC_ACQUIRE; -#endif - static constexpr auto std_constant = std::memory_order_acquire; -}; -constexpr memory_order_acquire_t memory_order_acquire = {}; - -struct memory_order_release_t { - using memory_order = memory_order_release_t; -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || \ - defined(KOKKOS_ENABLE_INTEL_ATOMICS) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - static constexpr auto gnu_constant = __ATOMIC_RELEASE; -#endif - static constexpr auto std_constant = std::memory_order_release; -}; -constexpr memory_order_release_t memory_order_release = {}; - -struct memory_order_acq_rel_t { - using memory_order = memory_order_acq_rel_t; -#if defined(KOKKOS_ENABLE_GNU_ATOMICS) || \ - defined(KOKKOS_ENABLE_INTEL_ATOMICS) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - static constexpr auto gnu_constant = __ATOMIC_ACQ_REL; -#endif - static constexpr auto std_constant = std::memory_order_acq_rel; -}; -constexpr memory_order_acq_rel_t memory_order_acq_rel = {}; - -// Intentionally omit consume (for now) - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_KOKKOS_ATOMIC_MEMORY_ORDER_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_MinMax.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_MinMax.hpp deleted file mode 100644 index 7338a5c545..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_MinMax.hpp +++ /dev/null @@ -1,319 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_ATOMIC_MINMAX_HPP) -#define KOKKOS_ATOMIC_MINMAX_HPP - -namespace Kokkos { - -//---------------------------------------------------------------------------- - -#if defined(KOKKOS_ENABLE_CUDA) -#if defined(__CUDA_ARCH__) || defined(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND) - -// Support for int, unsigned int, unsigned long long int, and float - -// Atomic_fetch_{min,max} - -#ifdef KOKKOS_IMPL_CUDA_CLANG_WORKAROUND - -// Host implementations for CLANG compiler - -inline __host__ int atomic_fetch_min(volatile int* const dest, const int val) { - return Impl::atomic_fetch_oper(Impl::MinOper(), dest, - val); -} - -inline __host__ unsigned int atomic_fetch_min(volatile unsigned int* const dest, - const unsigned int val) { - return Impl::atomic_fetch_oper( - Impl::MinOper(), dest, val); -} - -inline __host__ unsigned long long int atomic_fetch_min( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_fetch_oper(Impl::MinOper(), - dest, val); -} - -inline __host__ int atomic_fetch_max(volatile int* const dest, const int val) { - return Impl::atomic_fetch_oper(Impl::MaxOper(), dest, - val); -} - -inline __host__ unsigned int atomic_fetch_max(volatile unsigned int* const dest, - const unsigned int val) { - return Impl::atomic_fetch_oper( - Impl::MaxOper(), dest, val); -} - -inline __host__ unsigned long long int atomic_fetch_max( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_fetch_oper(Impl::MaxOper(), - dest, val); -} - -#endif - -#if (350 > __CUDA_ARCH__) - -// Fallback for atomic{Min,Max} for Kepler - -inline __device__ int atomic_fetch_min(volatile int* const dest, - const int val) { - return Impl::atomic_fetch_oper(Impl::MinOper(), dest, - val); -} - -inline __device__ unsigned int atomic_fetch_min( - volatile unsigned int* const dest, const unsigned int val) { - return Impl::atomic_fetch_oper( - Impl::MinOper(), dest, val); -} - -inline __device__ unsigned long long int atomic_fetch_min( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_fetch_oper(Impl::MinOper(), - dest, val); -} - -inline __device__ int atomic_fetch_max(volatile int* const dest, - const int val) { - return Impl::atomic_fetch_oper(Impl::MaxOper(), dest, - val); -} - -inline __device__ unsigned int atomic_fetch_max( - volatile unsigned int* const dest, const unsigned int val) { - return Impl::atomic_fetch_oper( - Impl::MaxOper(), dest, val); -} - -inline __device__ unsigned long long int atomic_fetch_max( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_fetch_oper(Impl::MaxOper(), - dest, val); -} - -#else // Supported by devices of compute capability 3.5 and higher - -inline __device__ int atomic_fetch_min(volatile int* const dest, - const int val) { - return atomicMin((int*)dest, val); -} - -inline __device__ unsigned int atomic_fetch_min( - volatile unsigned int* const dest, const unsigned int val) { - return atomicMin((unsigned int*)dest, val); -} - -inline __device__ unsigned long long int atomic_fetch_min( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return atomicMin((unsigned long long int*)dest, val); -} - -inline __device__ int atomic_fetch_max(volatile int* const dest, - const int val) { - return atomicMax((int*)dest, val); -} - -inline __device__ unsigned int atomic_fetch_max( - volatile unsigned int* const dest, const unsigned int val) { - return atomicMax((unsigned int*)dest, val); -} - -inline __device__ unsigned long long int atomic_fetch_max( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return atomicMax((unsigned long long int*)dest, val); -} - -#endif - -// Atomic_{min,max}_fetch - -#ifdef KOKKOS_IMPL_CUDA_CLANG_WORKAROUND - -// Host implementations for CLANG compiler - -inline __host__ int atomic_min_fetch(volatile int* const dest, const int val) { - return Impl::atomic_oper_fetch(Impl::MinOper(), dest, - val); -} - -inline __host__ unsigned int atomic_min_fetch(volatile unsigned int* const dest, - const unsigned int val) { - return Impl::atomic_oper_fetch( - Impl::MinOper(), dest, val); -} - -inline __host__ unsigned long long int atomic_min_fetch( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_oper_fetch(Impl::MinOper(), - dest, val); -} - -inline __host__ int atomic_max_fetch(volatile int* const dest, const int val) { - return Impl::atomic_oper_fetch(Impl::MaxOper(), dest, - val); -} - -inline __host__ unsigned int atomic_max_fetch(volatile unsigned int* const dest, - const unsigned int val) { - return Impl::atomic_oper_fetch( - Impl::MaxOper(), dest, val); -} - -inline __host__ unsigned long long int atomic_max_fetch( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_oper_fetch(Impl::MaxOper(), - dest, val); -} -#endif - -#if (350 > __CUDA_ARCH__) - -// Fallback for atomic{Min,Max} for Kepler - -inline __device__ int atomic_min_fetch(volatile int* const dest, - const int val) { - return Impl::atomic_oper_fetch(Impl::MinOper(), dest, - val); -} - -inline __device__ unsigned int atomic_min_fetch( - volatile unsigned int* const dest, const unsigned int val) { - return Impl::atomic_oper_fetch( - Impl::MinOper(), dest, val); -} - -inline __device__ unsigned long long int atomic_min_fetch( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_oper_fetch(Impl::MinOper(), - dest, val); -} - -inline __device__ int atomic_max_fetch(volatile int* const dest, - const int val) { - return Impl::atomic_oper_fetch(Impl::MaxOper(), dest, - val); -} - -inline __device__ unsigned int atomic_max_fetch( - volatile unsigned int* const dest, const unsigned int val) { - return Impl::atomic_oper_fetch( - Impl::MaxOper(), dest, val); -} - -inline __device__ unsigned long long int atomic_max_fetch( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - return Impl::atomic_oper_fetch(Impl::MaxOper(), - dest, val); -} - -#else // Supported by devices of compute capability 3.5 and higher - -inline __device__ int atomic_min_fetch(volatile int* const dest, - const int val) { - const int old = atomicMin((int*)dest, val); - return old < val ? old : val; -} - -inline __device__ unsigned int atomic_min_fetch( - volatile unsigned int* const dest, const unsigned int val) { - const unsigned int old = atomicMin((unsigned int*)dest, val); - return old < val ? old : val; -} - -inline __device__ unsigned long long int atomic_min_fetch( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - const unsigned long long old = atomicMin((unsigned long long*)dest, val); - return old < val ? old : val; -} - -inline __device__ int atomic_max_fetch(volatile int* const dest, - const int val) { - const int old = atomicMax((int*)dest, val); - return old >= val ? old : val; -} - -inline __device__ unsigned int atomic_max_fetch( - volatile unsigned int* const dest, const unsigned int val) { - const unsigned int old = atomicMax((unsigned int*)dest, val); - return old >= val ? old : val; -} - -inline __device__ unsigned long long int atomic_max_fetch( - volatile unsigned long long int* const dest, - const unsigned long long int val) { - const unsigned long long old = atomicMax((unsigned long long*)dest, val); - return old >= val ? old : val; -} - -#endif - -#endif -#endif -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Store.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Store.hpp deleted file mode 100644 index ffe018b4d6..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Store.hpp +++ /dev/null @@ -1,224 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_KOKKOS_ATOMIC_STORE_HPP -#define KOKKOS_IMPL_KOKKOS_ATOMIC_STORE_HPP - -#include -#if defined(KOKKOS_ATOMIC_HPP) - -#include -#include - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#endif - -namespace Kokkos { -namespace Impl { - -// Olivier's implementation helpfully binds to the same builtins as GNU, so -// we make this code common across multiple options -#if (defined(KOKKOS_ENABLE_GNU_ATOMICS) && !defined(__CUDA_ARCH__)) || \ - (defined(KOKKOS_ENABLE_INTEL_ATOMICS) && !defined(__CUDA_ARCH__)) || \ - defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) - -#if defined(__CUDA_ARCH__) && defined(KOKKOS_ENABLE_CUDA_ASM_ATOMICS) -#define KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH __inline__ __device__ -#else -#define KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH inline -#endif - -template -KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH void _atomic_store( - T* ptr, T val, MemoryOrder, - std::enable_if_t<(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || - sizeof(T) == 8) && - std::is_same>::value, - void const**> = nullptr) { - __atomic_store_n(ptr, val, MemoryOrder::gnu_constant); -} - -template -KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH void _atomic_store( - T* ptr, T val, MemoryOrder, - std::enable_if_t::value && - std::is_same>::value, - void const**> = nullptr) { - __atomic_store(ptr, &val, MemoryOrder::gnu_constant); -} - -#undef KOKKOS_INTERNAL_INLINE_DEVICE_IF_CUDA_ARCH - -#elif defined(__CUDA_ARCH__) - -// Not compiling for Volta or later, or Cuda ASM atomics were manually disabled - -template -__device__ __inline__ void _relaxed_atomic_store_impl( - T* ptr, T val, - std::enable_if_t<(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || - sizeof(T) == 8), - void const**> = nullptr) { - *ptr = val; -} - -template -struct StoreOper { - __device__ __inline__ static constexpr T apply(T const&, - T const& val) noexcept { - return val; - } -}; - -template -__device__ __inline__ void _relaxed_atomic_store_impl( - T* ptr, T val, - std::enable_if_t = nullptr) { - Kokkos::Impl::atomic_oper_fetch(StoreOper{}, ptr, (T &&) val); -} - -template -__device__ __inline__ void _atomic_store(T* ptr, T val, - memory_order_seq_cst_t) { - Kokkos::memory_fence(); - Impl::_relaxed_atomic_store_impl(ptr, val); - Kokkos::memory_fence(); -} - -template -__device__ __inline__ void _atomic_store(T* ptr, T val, - memory_order_release_t) { - Kokkos::memory_fence(); - _relaxed_atomic_store_impl(ptr, val); -} - -template -__device__ __inline__ void _atomic_store(T* ptr, T val, - memory_order_relaxed_t) { - _relaxed_atomic_store_impl(ptr, val); -} - -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) - -template -inline void _atomic_store(T* ptr, T val, MemoryOrder) { - // AFAICT, all OpenMP atomics are sequentially consistent, so memory order - // doesn't matter -#pragma omp atomic write - { *ptr = val; } -} - -#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS) - -template -inline void _atomic_store(T* ptr, T val, MemoryOrder) { - *ptr = val; -} - -#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS) - -template -inline void _atomic_store(T* ptr, T val, MemoryOrder) { - atomic_exchange(ptr, val); -} - -#endif // end of all atomic implementations - -template -KOKKOS_FORCEINLINE_FUNCTION void atomic_store(T* ptr, T val, - Impl::memory_order_seq_cst_t) { - _atomic_store(ptr, val, Impl::memory_order_seq_cst); -} - -template -KOKKOS_FORCEINLINE_FUNCTION void atomic_store(T* ptr, T val, - Impl::memory_order_release_t) { - _atomic_store(ptr, val, Impl::memory_order_release); -} - -template -KOKKOS_FORCEINLINE_FUNCTION void atomic_store(T* ptr, T val, - Impl::memory_order_relaxed_t) { - _atomic_store(ptr, val, Impl::memory_order_relaxed); -} - -template -KOKKOS_FORCEINLINE_FUNCTION void atomic_store(T* /*ptr*/, T /*val*/, - Impl::memory_order_acquire_t) { - static_assert( - sizeof(T) == 0, // just something that will always be false, but only on - // instantiation - "atomic_store with memory order acquire doesn't make any sense!"); -} - -template -KOKKOS_FORCEINLINE_FUNCTION void atomic_store(T* /*ptr*/, T /*val*/, - Impl::memory_order_acq_rel_t) { - static_assert( - sizeof(T) == 0, // just something that will always be false, but only on - // instantiation - "atomic_store with memory order acq_rel doesn't make any sense!"); -} - -template -KOKKOS_FORCEINLINE_FUNCTION void atomic_store(T* ptr, T val) { - // relaxed by default! - _atomic_store(ptr, val, Impl::memory_order_relaxed); -} - -} // end namespace Impl -} // end namespace Kokkos - -#if defined(KOKKOS_ENABLE_CUDA) -#include -#endif - -#endif // defined(KOKKOS_ATOMIC_HPP) -#endif // KOKKOS_IMPL_KOKKOS_ATOMIC_STORE_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_View.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_View.hpp deleted file mode 100644 index f763f8c791..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_View.hpp +++ /dev/null @@ -1,384 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_ATOMIC_VIEW_HPP -#define KOKKOS_ATOMIC_VIEW_HPP - -#include -#include - -namespace Kokkos { -namespace Impl { - -// The following tag is used to prevent an implicit call of the constructor when -// trying to assign a literal 0 int ( = 0 ); -struct AtomicViewConstTag {}; - -template -class AtomicDataElement { - public: - using value_type = typename ViewTraits::value_type; - using const_value_type = typename ViewTraits::const_value_type; - using non_const_value_type = typename ViewTraits::non_const_value_type; - volatile value_type* const ptr; - - KOKKOS_INLINE_FUNCTION - AtomicDataElement(value_type* ptr_, AtomicViewConstTag) : ptr(ptr_) {} - - KOKKOS_INLINE_FUNCTION - const_value_type operator=(const_value_type& val) const { - *ptr = val; - return val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator=(volatile const_value_type& val) const { - *ptr = val; - return val; - } - - KOKKOS_INLINE_FUNCTION - void inc() const { Kokkos::atomic_increment(ptr); } - - KOKKOS_INLINE_FUNCTION - void dec() const { Kokkos::atomic_decrement(ptr); } - - KOKKOS_INLINE_FUNCTION - const_value_type operator++() const { - const_value_type tmp = - Kokkos::atomic_fetch_add(ptr, non_const_value_type(1)); - return tmp + 1; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator--() const { - const_value_type tmp = - Kokkos::atomic_fetch_sub(ptr, non_const_value_type(1)); - return tmp - 1; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator++(int) const { - return Kokkos::atomic_fetch_add(ptr, non_const_value_type(1)); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator--(int) const { - return Kokkos::atomic_fetch_sub(ptr, non_const_value_type(1)); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator+=(const_value_type& val) const { - const_value_type tmp = Kokkos::atomic_fetch_add(ptr, val); - return tmp + val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator+=(volatile const_value_type& val) const { - const_value_type tmp = Kokkos::atomic_fetch_add(ptr, val); - return tmp + val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator-=(const_value_type& val) const { - const_value_type tmp = Kokkos::atomic_fetch_sub(ptr, val); - return tmp - val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator-=(volatile const_value_type& val) const { - const_value_type tmp = Kokkos::atomic_fetch_sub(ptr, val); - return tmp - val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator*=(const_value_type& val) const { - return Kokkos::atomic_mul_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator*=(volatile const_value_type& val) const { - return Kokkos::atomic_mul_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator/=(const_value_type& val) const { - return Kokkos::atomic_div_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator/=(volatile const_value_type& val) const { - return Kokkos::atomic_div_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator%=(const_value_type& val) const { - return Kokkos::atomic_mod_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator%=(volatile const_value_type& val) const { - return Kokkos::atomic_mod_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator&=(const_value_type& val) const { - return Kokkos::atomic_and_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator&=(volatile const_value_type& val) const { - return Kokkos::atomic_and_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator^=(const_value_type& val) const { - return Kokkos::atomic_xor_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator^=(volatile const_value_type& val) const { - return Kokkos::atomic_xor_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator|=(const_value_type& val) const { - return Kokkos::atomic_or_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator|=(volatile const_value_type& val) const { - return Kokkos::atomic_or_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator<<=(const_value_type& val) const { - return Kokkos::atomic_lshift_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator<<=(volatile const_value_type& val) const { - return Kokkos::atomic_lshift_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator>>=(const_value_type& val) const { - return Kokkos::atomic_rshift_fetch(ptr, val); - } - KOKKOS_INLINE_FUNCTION - const_value_type operator>>=(volatile const_value_type& val) const { - return Kokkos::atomic_rshift_fetch(ptr, val); - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator+(const_value_type& val) const { return *ptr + val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator+(volatile const_value_type& val) const { - return *ptr + val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator-(const_value_type& val) const { return *ptr - val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator-(volatile const_value_type& val) const { - return *ptr - val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator*(const_value_type& val) const { return *ptr * val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator*(volatile const_value_type& val) const { - return *ptr * val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator/(const_value_type& val) const { return *ptr / val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator/(volatile const_value_type& val) const { - return *ptr / val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator%(const_value_type& val) const { return *ptr ^ val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator%(volatile const_value_type& val) const { - return *ptr ^ val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator!() const { return !*ptr; } - - KOKKOS_INLINE_FUNCTION - const_value_type operator&&(const_value_type& val) const { - return *ptr && val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator&&(volatile const_value_type& val) const { - return *ptr && val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator||(const_value_type& val) const { - return *ptr | val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator||(volatile const_value_type& val) const { - return *ptr | val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator&(const_value_type& val) const { return *ptr & val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator&(volatile const_value_type& val) const { - return *ptr & val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator|(const_value_type& val) const { return *ptr | val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator|(volatile const_value_type& val) const { - return *ptr | val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator^(const_value_type& val) const { return *ptr ^ val; } - KOKKOS_INLINE_FUNCTION - const_value_type operator^(volatile const_value_type& val) const { - return *ptr ^ val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator~() const { return ~*ptr; } - - KOKKOS_INLINE_FUNCTION - const_value_type operator<<(const unsigned int& val) const { - return *ptr << val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator<<(volatile const unsigned int& val) const { - return *ptr << val; - } - - KOKKOS_INLINE_FUNCTION - const_value_type operator>>(const unsigned int& val) const { - return *ptr >> val; - } - KOKKOS_INLINE_FUNCTION - const_value_type operator>>(volatile const unsigned int& val) const { - return *ptr >> val; - } - - KOKKOS_INLINE_FUNCTION - bool operator==(const AtomicDataElement& val) const { return *ptr == val; } - KOKKOS_INLINE_FUNCTION - bool operator==(volatile const AtomicDataElement& val) const { - return *ptr == val; - } - - KOKKOS_INLINE_FUNCTION - bool operator!=(const AtomicDataElement& val) const { return *ptr != val; } - KOKKOS_INLINE_FUNCTION - bool operator!=(volatile const AtomicDataElement& val) const { - return *ptr != val; - } - - KOKKOS_INLINE_FUNCTION - bool operator>=(const_value_type& val) const { return *ptr >= val; } - KOKKOS_INLINE_FUNCTION - bool operator>=(volatile const_value_type& val) const { return *ptr >= val; } - - KOKKOS_INLINE_FUNCTION - bool operator<=(const_value_type& val) const { return *ptr <= val; } - KOKKOS_INLINE_FUNCTION - bool operator<=(volatile const_value_type& val) const { return *ptr <= val; } - - KOKKOS_INLINE_FUNCTION - bool operator<(const_value_type& val) const { return *ptr < val; } - KOKKOS_INLINE_FUNCTION - bool operator<(volatile const_value_type& val) const { return *ptr < val; } - - KOKKOS_INLINE_FUNCTION - bool operator>(const_value_type& val) const { return *ptr > val; } - KOKKOS_INLINE_FUNCTION - bool operator>(volatile const_value_type& val) const { return *ptr > val; } - - KOKKOS_INLINE_FUNCTION - operator const_value_type() const { - // return Kokkos::atomic_load(ptr); - return *ptr; - } - - KOKKOS_INLINE_FUNCTION - operator non_const_value_type() volatile const { - return Kokkos::Impl::atomic_load(ptr); - } -}; - -template -class AtomicViewDataHandle { - public: - typename ViewTraits::value_type* ptr; - - KOKKOS_INLINE_FUNCTION - AtomicViewDataHandle() : ptr(nullptr) {} - - KOKKOS_INLINE_FUNCTION - AtomicViewDataHandle(typename ViewTraits::value_type* ptr_) : ptr(ptr_) {} - - template - KOKKOS_INLINE_FUNCTION AtomicDataElement operator[]( - const iType& i) const { - return AtomicDataElement(ptr + i, AtomicViewConstTag()); - } - - KOKKOS_INLINE_FUNCTION - operator typename ViewTraits::value_type*() const { return ptr; } -}; - -template -struct Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars; - -template <> -struct Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars<4> { - using type = int; -}; - -template <> -struct Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars<8> { - using type = int64_t; -}; - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Windows.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Windows.hpp deleted file mode 100644 index c5207b51e9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Atomic_Windows.hpp +++ /dev/null @@ -1,155 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -#ifndef KOKKOS_ATOMIC_WINDOWS_HPP -#define KOKKOS_ATOMIC_WINDOWS_HPP - -#ifdef _WIN32 - -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include -#include - -namespace Kokkos { -namespace Impl { -#ifdef _MSC_VER -_declspec(align(16)) -#endif - struct cas128_t { - LONGLONG lower; - LONGLONG upper; - KOKKOS_INLINE_FUNCTION - bool operator!=(const cas128_t& a) const { - return (lower != a.lower) || upper != a.upper; - } -} -#if defined(__GNUC__) || defined(__clang__) -__attribute__((aligned(16))) -#endif -; -} // namespace Impl - -#if !defined(__CUDA_ARCH__) || defined(__clang__) -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - union U { - CHAR i; - T t; - KOKKOS_INLINE_FUNCTION U(){}; - } tmp; - - tmp.i = _InterlockedCompareExchange8((CHAR*)dest, *((CHAR*)&val), - *((CHAR*)&compare)); - return tmp.t; -} - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - union U { - SHORT i; - T t; - KOKKOS_INLINE_FUNCTION U(){}; - } tmp; - - tmp.i = _InterlockedCompareExchange16((SHORT*)dest, *((SHORT*)&val), - *((SHORT*)&compare)); - return tmp.t; -} - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - union U { - LONG i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - - tmp.i = _InterlockedCompareExchange((LONG*)dest, *((LONG*)&val), - *((LONG*)&compare)); - return tmp.t; -} - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - union U { - LONGLONG i; - T t; - KOKKOS_INLINE_FUNCTION U() {} - } tmp; - - tmp.i = _InterlockedCompareExchange64((LONGLONG*)dest, *((LONGLONG*)&val), - *((LONGLONG*)&compare)); - return tmp.t; -} - -template -inline T atomic_compare_exchange( - volatile T* const dest, const T& compare, - std::enable_if_t val) { - T compare_and_result(compare); - union U { - Impl::cas128_t i; - T t; - KOKKOS_INLINE_FUNCTION U(){}; - } newval; - newval.t = val; - _InterlockedCompareExchange128((LONGLONG*)dest, newval.i.upper, - newval.i.lower, - ((LONGLONG*)&compare_and_result)); - return compare_and_result; -} -#endif - -} // namespace Kokkos -#endif -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_CPUDiscovery.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_CPUDiscovery.cpp deleted file mode 100644 index a8fc928d10..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_CPUDiscovery.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#elif defined(__APPLE__) -#include -#include -#else -#include -#endif -#include -#include -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -int processors_per_node() { -#ifdef _SC_NPROCESSORS_ONLN - int const num_procs = sysconf(_SC_NPROCESSORS_ONLN); - int const num_procs_max = sysconf(_SC_NPROCESSORS_CONF); - if ((num_procs < 1) || (num_procs_max < 1)) { - return -1; - } - return num_procs; -#elif defined(__APPLE__) - int ncpu; - int activecpu; - size_t size = sizeof(int); - sysctlbyname("hw.ncpu", &ncpu, &size, nullptr, 0); - sysctlbyname("hw.activecpu", &activecpu, &size, nullptr, 0); - if (ncpu < 1 || activecpu < 1) - return -1; - else - return activecpu; -#else - return -1; -#endif -} - -int mpi_ranks_per_node() { - char *str; - int ppn = 1; - // if ((str = getenv("SLURM_TASKS_PER_NODE"))) { - // ppn = std::stoi(str); - // if(ppn<=0) ppn = 1; - //} - if ((str = getenv("MV2_COMM_WORLD_LOCAL_SIZE"))) { - ppn = std::stoi(str); - if (ppn <= 0) ppn = 1; - } - if ((str = getenv("OMPI_COMM_WORLD_LOCAL_SIZE"))) { - ppn = std::stoi(str); - if (ppn <= 0) ppn = 1; - } - return ppn; -} - -int mpi_local_rank_on_node() { - char *str; - int local_rank = 0; - // if ((str = getenv("SLURM_LOCALID"))) { - // local_rank = std::stoi(str); - //} - if ((str = getenv("MV2_COMM_WORLD_LOCAL_RANK"))) { - local_rank = std::stoi(str); - } - if ((str = getenv("OMPI_COMM_WORLD_LOCAL_RANK"))) { - local_rank = std::stoi(str); - } - return local_rank; -} - -} // namespace Impl -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_CPUDiscovery.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_CPUDiscovery.hpp deleted file mode 100644 index 6ba14c85b1..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_CPUDiscovery.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ -namespace Kokkos { -namespace Impl { - -int processors_per_node(); -int mpi_ranks_per_node(); -int mpi_local_rank_on_node(); - -} // namespace Impl -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Command_Line_Parsing.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Command_Line_Parsing.hpp deleted file mode 100644 index b22bc3e342..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Command_Line_Parsing.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_COMMAND_LINE_PARSING_HPP -#define KOKKOS_COMMAND_LINE_PARSING_HPP - -#include -#include - -namespace Kokkos { -namespace Impl { -bool is_unsigned_int(const char* str); -bool check_arg(char const* arg, char const* expected); -bool check_arg_bool(char const* arg, char const* name, bool& val); -bool check_arg_int(char const* arg, char const* name, int& val); -bool check_arg_str(char const* arg, char const* name, std::string& val); -bool check_env_bool(char const* name, bool& val); -bool check_env_int(char const* name, int& val); -void warn_deprecated_environment_variable(std::string deprecated); -void warn_deprecated_environment_variable(std::string deprecated, - std::string use_instead); -void warn_deprecated_command_line_argument(std::string deprecated); -void warn_deprecated_command_line_argument(std::string deprecated, - std::string use_instead); -void warn_not_recognized_command_line_argument(std::string not_recognized); -void do_not_warn_not_recognized_command_line_argument(std::regex ignore); -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Default_Graph_fwd.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Default_Graph_fwd.hpp deleted file mode 100644 index cdbed72888..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Default_Graph_fwd.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_KOKKOS_HOST_GRAPH_FWD_HPP -#define KOKKOS_KOKKOS_HOST_GRAPH_FWD_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -struct GraphNodeKernelDefaultImpl; - -template -struct GraphNodeAggregateKernelDefaultImpl; - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_KOKKOS_HOST_GRAPH_FWD_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_DeviceManagement.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_DeviceManagement.hpp deleted file mode 100644 index 34421f0fe5..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_DeviceManagement.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_DEVICE_MANAGEMENT_HPP -#define KOKKOS_DEVICE_MANAGEMENT_HPP - -#include - -namespace Kokkos { -class InitializationSettings; -namespace Impl { -int get_gpu(const Kokkos::InitializationSettings& settings); -// This declaration is provided for testing purposes only -int get_ctest_gpu(const char* local_rank_str); -// ditto -std::vector get_visible_devices( - Kokkos::InitializationSettings const& settings, int device_count); -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Error.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Error.cpp deleted file mode 100644 index 7502283315..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Error.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { -void traceback_callstack(std::ostream &msg) { -#ifdef KOKKOS_IMPL_ENABLE_STACKTRACE - msg << "\nBacktrace:\n"; - save_stacktrace(); - print_demangled_saved_stacktrace(msg); -#else - msg << "\nTraceback functionality not available\n"; -#endif -} - -void throw_runtime_exception(const std::string &msg) { - throw std::runtime_error(msg); -} - -void host_abort(const char *const message) { - std::cerr << message; - traceback_callstack(std::cerr); - ::abort(); -} - -std::string human_memory_size(size_t arg_bytes) { - double bytes = arg_bytes; - const double K = 1024; - const double M = K * 1024; - const double G = M * 1024; - - std::ostringstream out; - if (bytes < K) { - out << std::setprecision(4) << bytes << " B"; - } else if (bytes < M) { - bytes /= K; - out << std::setprecision(4) << bytes << " K"; - } else if (bytes < G) { - bytes /= M; - out << std::setprecision(4) << bytes << " M"; - } else { - bytes /= G; - out << std::setprecision(4) << bytes << " G"; - } - return out.str(); -} - -} // namespace Impl - -void Experimental::RawMemoryAllocationFailure::print_error_message( - std::ostream &o) const { - o << "Allocation of size " << Impl::human_memory_size(m_attempted_size); - o << " failed"; - switch (m_failure_mode) { - case FailureMode::OutOfMemoryError: - o << ", likely due to insufficient memory."; - break; - case FailureMode::AllocationNotAligned: - o << " because the allocation was improperly aligned."; - break; - case FailureMode::InvalidAllocationSize: - o << " because the requested allocation size is not a valid size for the" - " requested allocation mechanism (it's probably too large)."; - break; - // TODO move this to the subclass for Cuda-related things - case FailureMode::MaximumCudaUVMAllocationsExceeded: - o << " because the maximum Cuda UVM allocations was exceeded."; - break; - case FailureMode::Unknown: o << " because of an unknown error."; break; - } - o << " (The allocation mechanism was "; - switch (m_mechanism) { - case AllocationMechanism::StdMalloc: o << "standard malloc()."; break; - case AllocationMechanism::PosixMemAlign: o << "posix_memalign()."; break; - case AllocationMechanism::PosixMMap: o << "POSIX mmap()."; break; - case AllocationMechanism::IntelMMAlloc: - o << "the Intel _mm_malloc() intrinsic."; - break; - case AllocationMechanism::CudaMalloc: o << "cudaMalloc()."; break; - case AllocationMechanism::CudaMallocManaged: - o << "cudaMallocManaged()."; - break; - case AllocationMechanism::CudaHostAlloc: o << "cudaHostAlloc()."; break; - case AllocationMechanism::HIPMalloc: o << "hipMalloc()."; break; - case AllocationMechanism::HIPHostMalloc: o << "hipHostMalloc()."; break; - case AllocationMechanism::HIPMallocManaged: - o << "hipMallocManaged()."; - break; - case AllocationMechanism::SYCLMallocDevice: - o << "sycl::malloc_device()."; - break; - case AllocationMechanism::SYCLMallocShared: - o << "sycl::malloc_shared()."; - break; - case AllocationMechanism::SYCLMallocHost: - o << "sycl::malloc_host()."; - break; - } - append_additional_error_information(o); - o << ")" << std::endl; -} - -std::string Experimental::RawMemoryAllocationFailure::get_error_message() - const { - std::ostringstream out; - print_error_message(out); - return out.str(); -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -#ifdef KOKKOS_ENABLE_CUDA -namespace Experimental { - -void CudaRawMemoryAllocationFailure::append_additional_error_information( - std::ostream &o) const { - if (m_error_code != cudaSuccess) { - o << " The Cuda allocation returned the error code \"\"" - << cudaGetErrorName(m_error_code) << "\"."; - } -} - -} // end namespace Experimental -#endif - -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Error.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Error.hpp deleted file mode 100644 index 63b40f297e..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Error.hpp +++ /dev/null @@ -1,293 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_ERROR_HPP -#define KOKKOS_IMPL_ERROR_HPP - -#include -#include -#include -#ifdef KOKKOS_ENABLE_CUDA -#include -#endif -#ifdef KOKKOS_ENABLE_HIP -#include -#endif -#ifdef KOKKOS_ENABLE_SYCL -#include -#endif - -#ifndef KOKKOS_ABORT_MESSAGE_BUFFER_SIZE -#define KOKKOS_ABORT_MESSAGE_BUFFER_SIZE 2048 -#endif // ifndef KOKKOS_ABORT_MESSAGE_BUFFER_SIZE - -namespace Kokkos { -namespace Impl { - -[[noreturn]] void host_abort(const char *const); - -#if defined(KOKKOS_ENABLE_CUDA) && defined(__CUDA_ARCH__) - -#if defined(__APPLE__) || defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) -// cuda_abort does not abort when building for macOS. -// required to workaround failures in random number generator unit tests with -// pre-volta architectures -#define KOKKOS_IMPL_ABORT_NORETURN -#else -// cuda_abort aborts when building for other platforms than macOS -#define KOKKOS_IMPL_ABORT_NORETURN [[noreturn]] -#endif - -#elif defined(KOKKOS_COMPILER_NVHPC) - -#define KOKKOS_IMPL_ABORT_NORETURN - -#elif defined(KOKKOS_ENABLE_HIP) && defined(__HIP_DEVICE_COMPILE__) -// HIP aborts -#define KOKKOS_IMPL_ABORT_NORETURN [[noreturn]] -#elif defined(KOKKOS_ENABLE_SYCL) && defined(__SYCL_DEVICE_ONLY__) -// FIXME_SYCL SYCL doesn't abort -#define KOKKOS_IMPL_ABORT_NORETURN -#elif !defined(KOKKOS_ENABLE_OPENMPTARGET) -// Host aborts -#define KOKKOS_IMPL_ABORT_NORETURN [[noreturn]] -#else -// Everything else does not abort -#define KOKKOS_IMPL_ABORT_NORETURN -#endif - -#ifdef KOKKOS_ENABLE_SYCL // FIXME_SYCL -#define KOKKOS_IMPL_ABORT_NORETURN_DEVICE -#else -#define KOKKOS_IMPL_ABORT_NORETURN_DEVICE KOKKOS_IMPL_ABORT_NORETURN -#endif - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) || \ - defined(KOKKOS_ENABLE_SYCL) || defined(KOKKOS_ENABLE_OPENMPTARGET) -KOKKOS_IMPL_ABORT_NORETURN_DEVICE inline KOKKOS_IMPL_DEVICE_FUNCTION void -device_abort(const char *const msg) { -#if defined(KOKKOS_ENABLE_CUDA) - ::Kokkos::Impl::cuda_abort(msg); -#elif defined(KOKKOS_ENABLE_HIP) - ::Kokkos::Impl::hip_abort(msg); -#elif defined(KOKKOS_ENABLE_SYCL) - ::Kokkos::Impl::sycl_abort(msg); -#elif defined(KOKKOS_ENABLE_OPENMPTARGET) - printf("%s", msg); // FIXME_OPENMPTARGET -#else -#error faulty logic -#endif -} -#endif - -[[noreturn]] void throw_runtime_exception(const std::string &msg); - -void traceback_callstack(std::ostream &); - -std::string human_memory_size(size_t arg_bytes); - -} // namespace Impl - -namespace Experimental { - -class RawMemoryAllocationFailure : public std::bad_alloc { - public: - enum class FailureMode { - OutOfMemoryError, - AllocationNotAligned, - InvalidAllocationSize, - MaximumCudaUVMAllocationsExceeded, - Unknown - }; - enum class AllocationMechanism { - StdMalloc, - PosixMemAlign, - PosixMMap, - IntelMMAlloc, - CudaMalloc, - CudaMallocManaged, - CudaHostAlloc, - HIPMalloc, - HIPHostMalloc, - HIPMallocManaged, - SYCLMallocDevice, - SYCLMallocShared, - SYCLMallocHost - }; - - private: - size_t m_attempted_size; - size_t m_attempted_alignment; - FailureMode m_failure_mode; - AllocationMechanism m_mechanism; - - public: - RawMemoryAllocationFailure( - size_t arg_attempted_size, size_t arg_attempted_alignment, - FailureMode arg_failure_mode = FailureMode::OutOfMemoryError, - AllocationMechanism arg_mechanism = - AllocationMechanism::StdMalloc) noexcept - : m_attempted_size(arg_attempted_size), - m_attempted_alignment(arg_attempted_alignment), - m_failure_mode(arg_failure_mode), - m_mechanism(arg_mechanism) {} - - RawMemoryAllocationFailure() noexcept = delete; - - RawMemoryAllocationFailure(RawMemoryAllocationFailure const &) noexcept = - default; - RawMemoryAllocationFailure(RawMemoryAllocationFailure &&) noexcept = default; - - RawMemoryAllocationFailure &operator =( - RawMemoryAllocationFailure const &) noexcept = default; - RawMemoryAllocationFailure &operator =( - RawMemoryAllocationFailure &&) noexcept = default; - - ~RawMemoryAllocationFailure() noexcept override = default; - - KOKKOS_ATTRIBUTE_NODISCARD - const char *what() const noexcept override { - if (m_failure_mode == FailureMode::OutOfMemoryError) { - return "Memory allocation error: out of memory"; - } else if (m_failure_mode == FailureMode::AllocationNotAligned) { - return "Memory allocation error: allocation result was under-aligned"; - } - - return nullptr; // unreachable - } - - KOKKOS_ATTRIBUTE_NODISCARD - size_t attempted_size() const noexcept { return m_attempted_size; } - - KOKKOS_ATTRIBUTE_NODISCARD - size_t attempted_alignment() const noexcept { return m_attempted_alignment; } - - KOKKOS_ATTRIBUTE_NODISCARD - AllocationMechanism allocation_mechanism() const noexcept { - return m_mechanism; - } - - KOKKOS_ATTRIBUTE_NODISCARD - FailureMode failure_mode() const noexcept { return m_failure_mode; } - - void print_error_message(std::ostream &o) const; - KOKKOS_ATTRIBUTE_NODISCARD - std::string get_error_message() const; - - virtual void append_additional_error_information(std::ostream &) const {} -}; - -} // end namespace Experimental - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -KOKKOS_IMPL_ABORT_NORETURN KOKKOS_INLINE_FUNCTION void abort( - const char *const message) { - KOKKOS_IF_ON_HOST(::Kokkos::Impl::host_abort(message);) - KOKKOS_IF_ON_DEVICE(::Kokkos::Impl::device_abort(message);) -} - -#undef KOKKOS_IMPL_ABORT_NORETURN - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#if !defined(NDEBUG) || defined(KOKKOS_ENFORCE_CONTRACTS) || \ - defined(KOKKOS_ENABLE_DEBUG) -#define KOKKOS_EXPECTS(...) \ - { \ - if (!bool(__VA_ARGS__)) { \ - ::Kokkos::abort( \ - "Kokkos contract violation:\n " \ - " Expected precondition `" #__VA_ARGS__ \ - "` evaluated false.\n" \ - "Error at " KOKKOS_IMPL_TOSTRING(__FILE__) ":" KOKKOS_IMPL_TOSTRING( \ - __LINE__) " \n"); \ - } \ - } -#define KOKKOS_ENSURES(...) \ - { \ - if (!bool(__VA_ARGS__)) { \ - ::Kokkos::abort( \ - "Kokkos contract violation:\n " \ - " Ensured postcondition `" #__VA_ARGS__ \ - "` evaluated false.\n" \ - "Error at " KOKKOS_IMPL_TOSTRING(__FILE__) ":" KOKKOS_IMPL_TOSTRING( \ - __LINE__) " \n"); \ - } \ - } -// some projects already define this for themselves, so don't mess -// them up -#ifndef KOKKOS_ASSERT -#define KOKKOS_ASSERT(...) \ - { \ - if (!bool(__VA_ARGS__)) { \ - ::Kokkos::abort( \ - "Kokkos contract violation:\n " \ - " Asserted condition `" #__VA_ARGS__ \ - "` evaluated false.\n" \ - "Error at " KOKKOS_IMPL_TOSTRING(__FILE__) ":" KOKKOS_IMPL_TOSTRING( \ - __LINE__) " \n"); \ - } \ - } -#endif // ifndef KOKKOS_ASSERT -#else // not debug mode -#define KOKKOS_EXPECTS(...) -#define KOKKOS_ENSURES(...) -#ifndef KOKKOS_ASSERT -#define KOKKOS_ASSERT(...) -#endif // ifndef KOKKOS_ASSERT -#endif // end debug mode ifdefs - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #ifndef KOKKOS_IMPL_ERROR_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ExecPolicy.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ExecPolicy.cpp deleted file mode 100644 index 75b89c73a9..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ExecPolicy.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#include - -namespace Kokkos { -namespace Impl { -PerTeamValue::PerTeamValue(size_t arg) : value(arg) {} - -PerThreadValue::PerThreadValue(size_t arg) : value(arg) {} -} // namespace Impl - -Impl::PerTeamValue PerTeam(const size_t& arg) { - return Impl::PerTeamValue(arg); -} - -Impl::PerThreadValue PerThread(const size_t& arg) { - return Impl::PerThreadValue(arg); -} - -void team_policy_check_valid_storage_level_argument(int level) { - if (!(level == 0 || level == 1)) { - std::stringstream ss; - ss << "TeamPolicy::set_scratch_size(/*level*/ " << level - << ", ...) storage level argument must be 0 or 1 to be valid\n"; - Impl::throw_runtime_exception(ss.str()); - } -} - -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ExecSpaceManager.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ExecSpaceManager.hpp deleted file mode 100644 index 354bdde9bf..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ExecSpaceManager.hpp +++ /dev/null @@ -1,162 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_EXEC_SPACE_MANAGER_HPP -#define KOKKOS_EXEC_SPACE_MANAGER_HPP - -#include -#include -#include - -#include -#include -#include - -namespace { - -template -using public_member_types_t = std::enable_if_t< - Kokkos::is_execution_space::value && - Kokkos::is_memory_space::value && - Kokkos::is_device::value && - Kokkos::is_array_layout::value && - std::is_integral::value && - Kokkos::is_memory_space::value>; - -template -using print_configuration_t = std::enable_if_t< - std::is_void().print_configuration( - std::declval()))>::value && - std::is_void().print_configuration( - std::declval(), false))>::value>; - -template -using initialize_finalize_t = std::enable_if_t< - std::is_void()))>::value && - std::is_void::value>; - -template -using fence_t = std::enable_if_t< - std::is_void().fence())>::value && - std::is_void().fence("name"))>::value && - std::is_void::value>; - -#define STATIC_ASSERT(...) static_assert(__VA_ARGS__, "") // FIXME C++17 - -template -constexpr bool check_valid_execution_space() { - using Kokkos::is_detected; - STATIC_ASSERT(std::is_default_constructible::value); - STATIC_ASSERT(is_detected::value); - STATIC_ASSERT(is_detected::value); - STATIC_ASSERT(is_detected::value); - STATIC_ASSERT(is_detected::value); -#ifndef KOKKOS_ENABLE_HPX // FIXME_HPX - STATIC_ASSERT(sizeof(ExecutionSpace) <= 2 * sizeof(void*)); -#endif - return true; -} - -#undef STATIC_ASSERT - -} // namespace - -namespace Kokkos { -namespace Impl { - -struct ExecSpaceBase { - virtual void initialize(InitializationSettings const&) = 0; - virtual void finalize() = 0; - virtual void static_fence(std::string const&) = 0; - virtual void print_configuration(std::ostream& os, bool verbose) = 0; - virtual ~ExecSpaceBase() = default; -}; - -template -struct ExecSpaceDerived : ExecSpaceBase { - static_assert(check_valid_execution_space(), ""); - void initialize(InitializationSettings const& settings) final { - ExecutionSpace::impl_initialize(settings); - } - void finalize() final { ExecutionSpace::impl_finalize(); } - void static_fence(std::string const& label) final { - ExecutionSpace::impl_static_fence(label); - } - void print_configuration(std::ostream& os, bool verbose) final { - ExecutionSpace().print_configuration(os, verbose); - } -}; - -/* ExecSpaceManager - Responsible for initializing all the registered - * backends. Backends are registered using the register_space_initializer() - * function which should be called from a global context so that it is called - * prior to initialize_spaces() which is called from Kokkos::initialize() - */ -class ExecSpaceManager { - std::map> exec_space_factory_list; - ExecSpaceManager() = default; - - public: - void register_space_factory(std::string name, - std::unique_ptr ptr); - void initialize_spaces(const Kokkos::InitializationSettings& settings); - void finalize_spaces(); - void static_fence(const std::string&); - void print_configuration(std::ostream& os, bool verbose); - static ExecSpaceManager& get_instance(); -}; - -template -int initialize_space_factory(std::string name) { - auto space_ptr = std::make_unique>(); - ExecSpaceManager::get_instance().register_space_factory(name, - std::move(space_ptr)); - return 1; -} - -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_FixedBufferMemoryPool.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_FixedBufferMemoryPool.hpp deleted file mode 100644 index 3068ef3db0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_FixedBufferMemoryPool.hpp +++ /dev/null @@ -1,306 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_KOKKOS_FIXEDBUFFERMEMORYPOOL_HPP -#define KOKKOS_IMPL_KOKKOS_FIXEDBUFFERMEMORYPOOL_HPP - -#include -#include - -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class FixedBlockSizeMemoryPool - : private MemorySpaceInstanceStorage { - public: - using memory_space = typename DeviceType::memory_space; - using size_type = SizeType; - - private: - using memory_space_storage_base = - MemorySpaceInstanceStorage; - using tracker_type = Kokkos::Impl::SharedAllocationTracker; - using record_type = Kokkos::Impl::SharedAllocationRecord; - - struct alignas(Align) Block { - union { - char ignore; - char data[Size]; - }; - }; - - static constexpr auto actual_size = sizeof(Block); - - // TODO shared allocation tracker - // TODO @optimization put the index values on different cache lines (CPU) or - // pages (GPU)? - - tracker_type m_tracker = {}; - size_type m_num_blocks = 0; - size_type m_first_free_idx = 0; - size_type m_last_free_idx = 0; - Kokkos::OwningRawPtr m_first_block = nullptr; - Kokkos::OwningRawPtr m_free_indices = nullptr; - - enum : size_type { IndexInUse = ~size_type(0) }; - - public: - FixedBlockSizeMemoryPool(memory_space const& mem_space, size_type num_blocks) - : memory_space_storage_base(mem_space), - m_tracker(), - m_num_blocks(num_blocks), - m_first_free_idx(0), - m_last_free_idx(num_blocks) { - // TODO alignment? - auto block_record = record_type::allocate( - mem_space, "FixedBlockSizeMemPool_blocks", num_blocks * sizeof(Block)); - KOKKOS_ASSERT(intptr_t(block_record->data()) % Align == 0); - m_tracker.assign_allocated_record_to_uninitialized(block_record); - m_first_block = (Block*)block_record->data(); - - auto idx_record = - record_type::allocate(mem_space, "Kokkos::FixedBlockSizeMemPool_blocks", - num_blocks * sizeof(size_type)); - KOKKOS_ASSERT(intptr_t(idx_record->data()) % alignof(size_type) == 0); - m_tracker.assign_allocated_record_to_uninitialized(idx_record); - m_free_indices = (size_type*)idx_record->data(); - - for (size_type i = 0; i < num_blocks; ++i) { - m_free_indices[i] = i; - } - - Kokkos::memory_fence(); - } - - // For compatibility with MemoryPool<> - FixedBlockSizeMemoryPool(memory_space const& mem_space, - size_t mempool_capacity, unsigned, unsigned, - unsigned) - : FixedBlockSizeMemoryPool( - mem_space, mempool_capacity / - actual_size) { /* forwarding ctor, must be empty */ - } - - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool() = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool( - FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool( - FixedBlockSizeMemoryPool const&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=( - FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=( - FixedBlockSizeMemoryPool const&) = default; - - KOKKOS_INLINE_FUNCTION - void* allocate(size_type alloc_size) const noexcept { - (void)alloc_size; - KOKKOS_EXPECTS(alloc_size <= Size); - auto free_idx_counter = Kokkos::atomic_fetch_add( - (volatile size_type*)&m_first_free_idx, size_type(1)); - auto free_idx_idx = free_idx_counter % m_num_blocks; - - // We don't have exclusive access to m_free_indices[free_idx_idx] because - // the allocate counter might have lapped us since we incremented it - auto current_free_idx = m_free_indices[free_idx_idx]; - size_type free_idx = IndexInUse; - free_idx = Kokkos::atomic_compare_exchange(&m_free_indices[free_idx_idx], - current_free_idx, free_idx); - Kokkos::memory_fence(); - - // TODO figure out how to decrement here? - - if (free_idx == IndexInUse) { - return nullptr; - } else { - return (void*)&m_first_block[free_idx]; - } - } - - KOKKOS_INLINE_FUNCTION - void deallocate(void* ptr, size_type /*alloc_size*/) const noexcept { - // figure out which block we are - auto offset = intptr_t(ptr) - intptr_t(m_first_block); - - KOKKOS_EXPECTS(offset % actual_size == 0 && - offset / actual_size < m_num_blocks); - - Kokkos::memory_fence(); - auto last_idx_idx = Kokkos::atomic_fetch_add( - (volatile size_type*)&m_last_free_idx, size_type(1)); - last_idx_idx %= m_num_blocks; - m_free_indices[last_idx_idx] = offset / actual_size; - } -}; - -#if 0 -template < - class DeviceType, - size_t Size, - size_t Align=1, - class SizeType = typename DeviceType::execution_space::size_type -> -class FixedBlockSizeChaseLevMemoryPool - : private MemorySpaceInstanceStorage -{ -public: - - using memory_space = typename DeviceType::memory_space; - using size_type = SizeType; - -private: - - using memory_space_storage_base = MemorySpaceInstanceStorage; - using tracker_type = Kokkos::Impl::SharedAllocationTracker; - using record_type = Kokkos::Impl::SharedAllocationRecord; - - struct alignas(Align) Block { union { char ignore; char data[Size]; }; }; - - static constexpr auto actual_size = sizeof(Block); - - tracker_type m_tracker = { }; - size_type m_num_blocks = 0; - size_type m_first_free_idx = 0; - size_type m_last_free_idx = 0; - - - enum : size_type { IndexInUse = ~size_type(0) }; - -public: - - FixedBlockSizeMemoryPool( - memory_space const& mem_space, - size_type num_blocks - ) : memory_space_storage_base(mem_space), - m_tracker(), - m_num_blocks(num_blocks), - m_first_free_idx(0), - m_last_free_idx(num_blocks) - { - // TODO alignment? - auto block_record = record_type::allocate( - mem_space, "FixedBlockSizeMemPool_blocks", num_blocks * sizeof(Block) - ); - KOKKOS_ASSERT(intptr_t(block_record->data()) % Align == 0); - m_tracker.assign_allocated_record_to_uninitialized(block_record); - m_first_block = (Block*)block_record->data(); - - auto idx_record = record_type::allocate( - mem_space, "FixedBlockSizeMemPool_blocks", num_blocks * sizeof(size_type) - ); - KOKKOS_ASSERT(intptr_t(idx_record->data()) % alignof(size_type) == 0); - m_tracker.assign_allocated_record_to_uninitialized(idx_record); - m_free_indices = (size_type*)idx_record->data(); - - for(size_type i = 0; i < num_blocks; ++i) { - m_free_indices[i] = i; - } - - Kokkos::memory_fence(); - } - - // For compatibility with MemoryPool<> - FixedBlockSizeMemoryPool( - memory_space const& mem_space, - size_t mempool_capacity, - unsigned, unsigned, unsigned - ) : FixedBlockSizeMemoryPool(mem_space, mempool_capacity / actual_size) - { /* forwarding ctor, must be empty */ } - - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool() = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool(FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool(FixedBlockSizeMemoryPool const&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=(FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=(FixedBlockSizeMemoryPool const&) = default; - - - KOKKOS_INLINE_FUNCTION - void* allocate(size_type alloc_size) const noexcept - { - KOKKOS_EXPECTS(alloc_size <= Size); - auto free_idx_counter = Kokkos::atomic_fetch_add((volatile size_type*)&m_first_free_idx, size_type(1)); - auto free_idx_idx = free_idx_counter % m_num_blocks; - - // We don't have exclusive access to m_free_indices[free_idx_idx] because - // the allocate counter might have lapped us since we incremented it - auto current_free_idx = m_free_indices[free_idx_idx]; - size_type free_idx = IndexInUse; - free_idx = - Kokkos::atomic_compare_exchange(&m_free_indices[free_idx_idx], current_free_idx, free_idx); - Kokkos::memory_fence(); - - // TODO figure out how to decrement here? - - if(free_idx == IndexInUse) { - return nullptr; - } - else { - return (void*)&m_first_block[free_idx]; - } - } - - KOKKOS_INLINE_FUNCTION - void deallocate(void* ptr, size_type alloc_size) const noexcept - { - // figure out which block we are - auto offset = intptr_t(ptr) - intptr_t(m_first_block); - - KOKKOS_EXPECTS(offset % actual_size == 0 && offset/actual_size < m_num_blocks); - - Kokkos::memory_fence(); - auto last_idx_idx = Kokkos::atomic_fetch_add((volatile size_type*)&m_last_free_idx, size_type(1)); - last_idx_idx %= m_num_blocks; - m_free_indices[last_idx_idx] = offset / actual_size; - } - -}; -#endif - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_IMPL_KOKKOS_FIXEDBUFFERMEMORYPOOL_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_GraphImpl_fwd.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_GraphImpl_fwd.hpp deleted file mode 100644 index 37c53075d0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_GraphImpl_fwd.hpp +++ /dev/null @@ -1,87 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_KOKKOS_GRAPHIMPL_FWD_HPP -#define KOKKOS_IMPL_KOKKOS_GRAPHIMPL_FWD_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -struct GraphNodeImpl; - -template -struct GraphImpl; - -template -class GraphNodeKernelImpl; - -struct _graph_node_kernel_ctor_tag {}; -struct _graph_node_predecessor_ctor_tag {}; -struct _graph_node_is_root_ctor_tag {}; - -struct GraphAccess; - -// Customizable for backends -template -struct GraphNodeBackendSpecificDetails; - -// Customizable for backends -template -struct GraphNodeBackendDetailsBeforeTypeErasure; - -// TODO move this to a more appropriate place -struct DoNotExplicitlySpecifyThisTemplateParameter; - -struct KernelInGraphProperty {}; - -struct IsGraphKernelTag {}; - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_IMPL_KOKKOS_GRAPHIMPL_FWD_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HBWSpace.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HBWSpace.cpp deleted file mode 100644 index d533ec05cd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HBWSpace.cpp +++ /dev/null @@ -1,384 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#ifdef KOKKOS_ENABLE_HBWSPACE -#include -#endif - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -#ifdef KOKKOS_ENABLE_HBWSPACE -#define MEMKIND_TYPE MEMKIND_HBW // hbw_get_kind(HBW_PAGESIZE_4KB) - -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Experimental { - -/* Default allocation mechanism */ -HBWSpace::HBWSpace() : m_alloc_mech(HBWSpace::STD_MALLOC) { - printf("Init\n"); - setenv("MEMKIND_HBW_NODES", "1", 0); -} - -/* Default allocation mechanism */ -HBWSpace::HBWSpace(const HBWSpace::AllocationMechanism &arg_alloc_mech) - : m_alloc_mech(HBWSpace::STD_MALLOC) { - printf("Init2\n"); - setenv("MEMKIND_HBW_NODES", "1", 0); - if (arg_alloc_mech == STD_MALLOC) { - m_alloc_mech = HBWSpace::STD_MALLOC; - } -} - -void *HBWSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void *HBWSpace::allocate(const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void *HBWSpace::impl_allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - static_assert(sizeof(void *) == sizeof(uintptr_t), - "Error sizeof(void*) != sizeof(uintptr_t)"); - - static_assert( - Kokkos::Impl::power_of_two::value, - "Memory alignment must be power of two"); - - constexpr uintptr_t alignment = Kokkos::Impl::MEMORY_ALIGNMENT; - constexpr uintptr_t alignment_mask = alignment - 1; - - void *ptr = nullptr; - - if (arg_alloc_size) { - if (m_alloc_mech == STD_MALLOC) { - // Over-allocate to and round up to guarantee proper alignment. - size_t size_padded = arg_alloc_size + sizeof(void *) + alignment; - - void *alloc_ptr = memkind_malloc(MEMKIND_TYPE, size_padded); - - if (alloc_ptr) { - uintptr_t address = reinterpret_cast(alloc_ptr); - - // offset enough to record the alloc_ptr - address += sizeof(void *); - uintptr_t rem = address % alignment; - uintptr_t offset = rem ? (alignment - rem) : 0u; - address += offset; - ptr = reinterpret_cast(address); - // record the alloc'd pointer - address -= sizeof(void *); - *reinterpret_cast(address) = alloc_ptr; - } - } - } - - if ((ptr == nullptr) || (reinterpret_cast(ptr) == ~uintptr_t(0)) || - (reinterpret_cast(ptr) & alignment_mask)) { - std::ostringstream msg; - msg << "Kokkos::Experimental::HBWSpace::allocate[ "; - switch (m_alloc_mech) { - case STD_MALLOC: msg << "STD_MALLOC"; break; - case POSIX_MEMALIGN: msg << "POSIX_MEMALIGN"; break; - case POSIX_MMAP: msg << "POSIX_MMAP"; break; - case INTEL_MM_ALLOC: msg << "INTEL_MM_ALLOC"; break; - } - msg << " ]( " << arg_alloc_size << " ) FAILED"; - if (ptr == nullptr) { - msg << " nullptr"; - } else { - msg << " NOT ALIGNED " << ptr; - } - - std::cerr << msg.str() << std::endl; - std::cerr.flush(); - - Kokkos::Impl::throw_runtime_exception(msg.str()); - } - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - - return ptr; -} - -void HBWSpace::deallocate(void *const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} -void HBWSpace::deallocate(const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void HBWSpace::impl_deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (arg_alloc_ptr) { - if (Kokkos::Profiling::profileLibraryLoaded()) { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - - if (m_alloc_mech == STD_MALLOC) { - void *alloc_ptr = *(reinterpret_cast(arg_alloc_ptr) - 1); - memkind_free(MEMKIND_TYPE, alloc_ptr); - } - } -} - -} // namespace Experimental -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -#ifdef KOKKOS_ENABLE_DEBUG -SharedAllocationRecord - SharedAllocationRecord::s_root_record; -#endif - -void SharedAllocationRecord::deallocate( - SharedAllocationRecord *arg_rec) { - delete static_cast(arg_rec); -} - -SharedAllocationRecord::~SharedAllocationRecord() -#if defined( \ - KOKKOS_IMPL_INTEL_WORKAROUND_NOEXCEPT_SPECIFICATION_VIRTUAL_FUNCTION) - noexcept -#endif -{ - - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size, - (SharedAllocationRecord::m_alloc_size - - sizeof(SharedAllocationHeader))); -} - -SharedAllocationRecord:: - SharedAllocationRecord( - const Kokkos::Experimental::HBWSpace &arg_space, - const std::string &arg_label, const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : SharedAllocationRecord( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - // Fill in the Header information - RecordBase::m_alloc_ptr->m_record = - static_cast *>(this); - - strncpy(RecordBase::m_alloc_ptr->m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length - 1); - // Set last element zero, in case c_str is too long - RecordBase::m_alloc_ptr - ->m_label[SharedAllocationHeader::maximum_label_length - 1] = '\0'; -} - -//---------------------------------------------------------------------------- - -void * -SharedAllocationRecord::allocate_tracked( - const Kokkos::Experimental::HBWSpace &arg_space, - const std::string &arg_alloc_label, const size_t arg_alloc_size) { - if (!arg_alloc_size) return nullptr; - - SharedAllocationRecord *const r = - allocate(arg_space, arg_alloc_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); -} - -void SharedAllocationRecord::deallocate_tracked(void *const - arg_alloc_ptr) { - if (arg_alloc_ptr != nullptr) { - SharedAllocationRecord *const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } -} - -void *SharedAllocationRecord:: - reallocate_tracked(void *const arg_alloc_ptr, const size_t arg_alloc_size) { - SharedAllocationRecord *const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord *const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - Kokkos::fence( - "SharedAllocationRecord::reallocate_tracked(): fence after copying data"); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); -} - -SharedAllocationRecord - *SharedAllocationRecord::get_record( - void *alloc_ptr) { - using Header = SharedAllocationHeader; - using RecordHost = - SharedAllocationRecord; - - SharedAllocationHeader const *const head = - alloc_ptr ? Header::get_header(alloc_ptr) : nullptr; - RecordHost *const record = - head ? static_cast(head->m_record) : nullptr; - - if (!alloc_ptr || record->m_alloc_ptr != head) { - Kokkos::Impl::throw_runtime_exception(std::string( - "Kokkos::Impl::SharedAllocationRecord< Kokkos::Experimental::HBWSpace " - ", void >::get_record ERROR")); - } - - return record; -} - -// Iterate records to print orphaned memory ... -void SharedAllocationRecord:: - print_records(std::ostream &s, const Kokkos::Experimental::HBWSpace &space, - bool detail) { -#ifdef KOKKOS_ENABLE_DEBUG - SharedAllocationRecord::print_host_accessible_records( - s, "HBWSpace", &s_root_record, detail); -#else - throw_runtime_exception( - "SharedAllocationRecord::print_records" - " only works with KOKKOS_ENABLE_DEBUG enabled"); -#endif -} - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace Experimental { -namespace { -const unsigned HBW_SPACE_ATOMIC_MASK = 0xFFFF; -const unsigned HBW_SPACE_ATOMIC_XOR_MASK = 0x5A39; -static int HBW_SPACE_ATOMIC_LOCKS[HBW_SPACE_ATOMIC_MASK + 1]; -} // namespace - -namespace Impl { -void init_lock_array_hbw_space() { - static int is_initialized = 0; - if (!is_initialized) - for (int i = 0; i < static_cast(HBW_SPACE_ATOMIC_MASK + 1); i++) - HBW_SPACE_ATOMIC_LOCKS[i] = 0; -} - -bool lock_address_hbw_space(void *ptr) { - return 0 == atomic_compare_exchange( - &HBW_SPACE_ATOMIC_LOCKS[((size_t(ptr) >> 2) & - HBW_SPACE_ATOMIC_MASK) ^ - HBW_SPACE_ATOMIC_XOR_MASK], - 0, 1); -} - -void unlock_address_hbw_space(void *ptr) { - atomic_exchange( - &HBW_SPACE_ATOMIC_LOCKS[((size_t(ptr) >> 2) & HBW_SPACE_ATOMIC_MASK) ^ - HBW_SPACE_ATOMIC_XOR_MASK], - 0); -} - -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostBarrier.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostBarrier.cpp deleted file mode 100644 index 9ad2dae551..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostBarrier.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include - -#include - -#include -#if defined(_WIN32) -#include -#include -#include -#endif - -namespace Kokkos { -namespace Impl { - -void HostBarrier::impl_backoff_wait_until_equal( - int* ptr, const int v, const bool active_wait) noexcept { - unsigned count = 0u; - - while (!test_equal(ptr, v)) { - const int c = int_log2(++count); - if (!active_wait || c > log2_iterations_till_sleep) { - std::this_thread::sleep_for( - std::chrono::nanoseconds(c < 16 ? 256 * c : 4096)); - } else if (c > log2_iterations_till_yield) { - std::this_thread::yield(); - } -#if defined(KOKKOS_ENABLE_ASM) -#if defined(__PPC64__) - for (int j = 0; j < num_nops; ++j) { - asm volatile("nop\n"); - } - asm volatile("or 27, 27, 27" ::: "memory"); -#elif defined(__amd64) || defined(__amd64__) || defined(__x86_64) || \ - defined(__x86_64__) - for (int j = 0; j < num_nops; ++j) { - asm volatile("nop\n"); - } - asm volatile("pause\n" ::: "memory"); -#endif -#endif - } -} -} // namespace Impl -} // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostSpace.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostSpace.cpp deleted file mode 100644 index 4bf9048919..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostSpace.cpp +++ /dev/null @@ -1,373 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include - -/*--------------------------------------------------------------------------*/ - -#if defined(KOKKOS_COMPILER_INTEL) && !defined(KOKKOS_ENABLE_CUDA) - -// Intel specialized allocator does not interoperate with CUDA memory allocation - -#define KOKKOS_ENABLE_INTEL_MM_ALLOC - -#endif - -/*--------------------------------------------------------------------------*/ - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -/* Default allocation mechanism */ -HostSpace::HostSpace() - : m_alloc_mech( -#if defined(KOKKOS_ENABLE_INTEL_MM_ALLOC) - HostSpace::INTEL_MM_ALLOC -#else - HostSpace::STD_MALLOC -#endif - ) { -} - -/* Default allocation mechanism */ -HostSpace::HostSpace(const HostSpace::AllocationMechanism &arg_alloc_mech) - : m_alloc_mech(HostSpace::STD_MALLOC) { - if (arg_alloc_mech == STD_MALLOC) { - m_alloc_mech = HostSpace::STD_MALLOC; - } -#if defined(KOKKOS_ENABLE_INTEL_MM_ALLOC) - else if (arg_alloc_mech == HostSpace::INTEL_MM_ALLOC) { - m_alloc_mech = HostSpace::INTEL_MM_ALLOC; - } -#endif - else { - const char *const mech = - (arg_alloc_mech == HostSpace::INTEL_MM_ALLOC) - ? "INTEL_MM_ALLOC" - : ((arg_alloc_mech == HostSpace::POSIX_MMAP) ? "POSIX_MMAP" : ""); - - std::string msg; - msg.append("Kokkos::HostSpace "); - msg.append(mech); - msg.append(" is not available"); - Kokkos::Impl::throw_runtime_exception(msg); - } -} - -void *HostSpace::allocate(const size_t arg_alloc_size) const { - return allocate("[unlabeled]", arg_alloc_size); -} -void *HostSpace::allocate(const char *arg_label, const size_t arg_alloc_size, - const size_t - - arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); -} -void *HostSpace::impl_allocate( - const char *arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - const size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - static_assert(sizeof(void *) == sizeof(uintptr_t), - "Error sizeof(void*) != sizeof(uintptr_t)"); - - static_assert( - Kokkos::Impl::is_integral_power_of_two(Kokkos::Impl::MEMORY_ALIGNMENT), - "Memory alignment must be power of two"); - - constexpr uintptr_t alignment = Kokkos::Impl::MEMORY_ALIGNMENT; - constexpr uintptr_t alignment_mask = alignment - 1; - - void *ptr = nullptr; - - if (arg_alloc_size) { - if (m_alloc_mech == STD_MALLOC) { - // Over-allocate to and round up to guarantee proper alignment. - size_t size_padded = arg_alloc_size + sizeof(void *) + alignment; - - void *alloc_ptr = malloc(size_padded); - - if (alloc_ptr) { - auto address = reinterpret_cast(alloc_ptr); - - // offset enough to record the alloc_ptr - address += sizeof(void *); - uintptr_t rem = address % alignment; - uintptr_t offset = rem ? (alignment - rem) : 0u; - address += offset; - ptr = reinterpret_cast(address); - // record the alloc'd pointer - address -= sizeof(void *); - *reinterpret_cast(address) = alloc_ptr; - } - } -#if defined(KOKKOS_ENABLE_INTEL_MM_ALLOC) - else if (m_alloc_mech == INTEL_MM_ALLOC) { - ptr = _mm_malloc(arg_alloc_size, alignment); - } -#endif - } - - if ((ptr == nullptr) || (reinterpret_cast(ptr) == ~uintptr_t(0)) || - (reinterpret_cast(ptr) & alignment_mask)) { - Experimental::RawMemoryAllocationFailure::FailureMode failure_mode = - Experimental::RawMemoryAllocationFailure::FailureMode:: - AllocationNotAligned; - if (ptr == nullptr) { - failure_mode = Experimental::RawMemoryAllocationFailure::FailureMode:: - OutOfMemoryError; - } - - Experimental::RawMemoryAllocationFailure::AllocationMechanism alloc_mec = - Experimental::RawMemoryAllocationFailure::AllocationMechanism:: - StdMalloc; - switch (m_alloc_mech) { - case STD_MALLOC: break; // default - case POSIX_MEMALIGN: - alloc_mec = Experimental::RawMemoryAllocationFailure:: - AllocationMechanism::PosixMemAlign; - break; - case POSIX_MMAP: - alloc_mec = Experimental::RawMemoryAllocationFailure:: - AllocationMechanism::PosixMMap; - break; - case INTEL_MM_ALLOC: - alloc_mec = Experimental::RawMemoryAllocationFailure:: - AllocationMechanism::IntelMMAlloc; - break; - } - - throw Kokkos::Experimental::RawMemoryAllocationFailure( - arg_alloc_size, alignment, failure_mode, alloc_mec); - } - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); - } - return ptr; -} - -void HostSpace::deallocate(void *const arg_alloc_ptr, - const size_t arg_alloc_size) const { - deallocate("[unlabeled]", arg_alloc_ptr, arg_alloc_size); -} - -void HostSpace::deallocate(const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, - const size_t - - arg_logical_size) const { - impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size); -} -void HostSpace::impl_deallocate( - const char *arg_label, void *const arg_alloc_ptr, - const size_t arg_alloc_size, const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { - if (arg_alloc_ptr) { - Kokkos::fence("HostSpace::impl_deallocate before free"); - size_t reported_size = - (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, - reported_size); - } - if (m_alloc_mech == STD_MALLOC) { - void *alloc_ptr = *(reinterpret_cast(arg_alloc_ptr) - 1); - free(alloc_ptr); - } -#if defined(KOKKOS_ENABLE_INTEL_MM_ALLOC) - else if (m_alloc_mech == INTEL_MM_ALLOC) { - _mm_free(arg_alloc_ptr); - } -#endif - } -} - -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -#ifdef KOKKOS_ENABLE_DEBUG -SharedAllocationRecord - SharedAllocationRecord::s_root_record; -#endif - -SharedAllocationRecord::~SharedAllocationRecord() -#if defined( \ - KOKKOS_IMPL_INTEL_WORKAROUND_NOEXCEPT_SPECIFICATION_VIRTUAL_FUNCTION) - noexcept -#endif -{ - m_space.deallocate(m_label.c_str(), - SharedAllocationRecord::m_alloc_ptr, - SharedAllocationRecord::m_alloc_size, - (SharedAllocationRecord::m_alloc_size - - sizeof(SharedAllocationHeader))); -} - -SharedAllocationHeader *_do_allocation(Kokkos::HostSpace const &space, - std::string const &label, - size_t alloc_size) { - try { - return reinterpret_cast( - space.allocate(alloc_size)); - } catch (Experimental::RawMemoryAllocationFailure const &failure) { - if (failure.failure_mode() == Experimental::RawMemoryAllocationFailure:: - FailureMode::AllocationNotAligned) { - // TODO: delete the misaligned memory - } - - std::cerr << "Kokkos failed to allocate memory for label \"" << label - << "\". Allocation using MemorySpace named \"" << space.name() - << " failed with the following error: "; - failure.print_error_message(std::cerr); - std::cerr.flush(); - Kokkos::Impl::throw_runtime_exception("Memory allocation failure"); - } - return nullptr; // unreachable -} - -SharedAllocationRecord::SharedAllocationRecord( - const Kokkos::HostSpace &arg_space, const std::string &arg_label, - const size_t arg_alloc_size, - const SharedAllocationRecord::function_type arg_dealloc) - // Pass through allocated [ SharedAllocationHeader , user_memory ] - // Pass through deallocation function - : base_t( -#ifdef KOKKOS_ENABLE_DEBUG - &SharedAllocationRecord::s_root_record, -#endif - Impl::checked_allocation_with_header(arg_space, arg_label, - arg_alloc_size), - sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc, - arg_label), - m_space(arg_space) { - this->base_t::_fill_host_accessible_header_info(*RecordBase::m_alloc_ptr, - arg_label); -} - -} // namespace Impl -} // namespace Kokkos - -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -namespace Kokkos { -namespace { -const unsigned HOST_SPACE_ATOMIC_MASK = 0xFFFF; -const unsigned HOST_SPACE_ATOMIC_XOR_MASK = 0x5A39; -static int HOST_SPACE_ATOMIC_LOCKS[HOST_SPACE_ATOMIC_MASK + 1]; -} // namespace - -namespace Impl { -void init_lock_array_host_space() { - static int is_initialized = 0; - if (!is_initialized) - for (int i = 0; i < static_cast(HOST_SPACE_ATOMIC_MASK + 1); i++) - HOST_SPACE_ATOMIC_LOCKS[i] = 0; -} - -bool lock_address_host_space(void *ptr) { - return 0 == atomic_compare_exchange( - &HOST_SPACE_ATOMIC_LOCKS[((size_t(ptr) >> 2) & - HOST_SPACE_ATOMIC_MASK) ^ - HOST_SPACE_ATOMIC_XOR_MASK], - 0, 1); -} - -void unlock_address_host_space(void *ptr) { - atomic_exchange( - &HOST_SPACE_ATOMIC_LOCKS[((size_t(ptr) >> 2) & HOST_SPACE_ATOMIC_MASK) ^ - HOST_SPACE_ATOMIC_XOR_MASK], - 0); -} - -} // namespace Impl -} // namespace Kokkos - -//============================================================================== -// {{{1 - -#include - -namespace Kokkos { -namespace Impl { - -// To avoid additional compilation cost for something that's (mostly?) not -// performance sensitive, we explicity instantiate these CRTP base classes here, -// where we have access to the associated *_timpl.hpp header files. -template class SharedAllocationRecordCommon; - -} // end namespace Impl -} // end namespace Kokkos - -// end Explicit instantiations of CRTP Base classes }}}1 -//============================================================================== diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostSpace_deepcopy.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostSpace_deepcopy.hpp deleted file mode 100644 index 88d37672d2..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_HostSpace_deepcopy.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_HOSTSPACE_DEEPCOPY_HPP -#define KOKKOS_IMPL_HOSTSPACE_DEEPCOPY_HPP - -#include - -namespace Kokkos { - -namespace Impl { - -void hostspace_fence(const DefaultHostExecutionSpace& exec); - -void hostspace_parallel_deepcopy(void* dst, const void* src, ptrdiff_t n); -// DeepCopy called with an execution space that can't access HostSpace -void hostspace_parallel_deepcopy_async(void* dst, const void* src, ptrdiff_t n); -void hostspace_parallel_deepcopy_async(const DefaultHostExecutionSpace& exec, - void* dst, const void* src, ptrdiff_t n); - -} // namespace Impl - -} // namespace Kokkos - -#endif // KOKKOS_IMPL_HOSTSPACE_DEEPCOPY_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_InitializationSettings.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_InitializationSettings.hpp deleted file mode 100644 index ceb35f0247..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_InitializationSettings.hpp +++ /dev/null @@ -1,195 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_INITIALIZATION_SETTINGS_HPP -#define KOKKOS_INITIALIZATION_SETTINGS_HPP - -#include - -#include -#include - -namespace Kokkos { - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -struct InitArguments { - int num_threads; - int num_numa; - int device_id; - int ndevices; - int skip_device; - bool disable_warnings; - bool tune_internals; - bool tool_help = false; - std::string tool_lib = {}; - std::string tool_args = {}; - - KOKKOS_DEPRECATED_WITH_COMMENT("Use InitializationSettings instead!") - InitArguments(int nt = -1, int nn = -1, int dv = -1, bool dw = false, - bool ti = false) - : num_threads{nt}, - num_numa{nn}, - device_id{dv}, - ndevices{-1}, - skip_device{9999}, - disable_warnings{dw}, - tune_internals{ti} {} -}; -#endif - -namespace Impl { -// FIXME_CXX17 replace with std::optional -template -struct InitializationSettingsHelper; -template <> -struct InitializationSettingsHelper { - using value_type = int; - using storage_type = int; - - static constexpr storage_type unspecified = INT_MIN; -}; -template <> -struct InitializationSettingsHelper { - using value_type = bool; - using storage_type = char; - - static constexpr storage_type unspecified = CHAR_MAX; - static_assert(static_cast(true) != unspecified && - static_cast(false) != unspecified, - ""); -}; -template <> -struct InitializationSettingsHelper { - using value_type = std::string; - using storage_type = std::string; - - static storage_type const unspecified; -}; -} // namespace Impl - -class InitializationSettings { -#define KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER(NAME) \ - impl_do_not_use_i_really_mean_it_##NAME##_ - -#define KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER_TYPE(NAME) impl_##NAME##_type - -#define KOKKOS_IMPL_DECLARE(TYPE, NAME) \ - private: \ - using KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER_TYPE(NAME) = TYPE; \ - Impl::InitializationSettingsHelper::storage_type \ - KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER(NAME) = \ - Impl::InitializationSettingsHelper::unspecified; \ - \ - public: \ - InitializationSettings& set_##NAME( \ - Impl::InitializationSettingsHelper::value_type NAME) { \ - KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER(NAME) = NAME; \ - return *this; \ - } \ - bool has_##NAME() const noexcept { \ - return KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER(NAME) != \ - Impl::InitializationSettingsHelper< \ - KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER_TYPE(NAME)>::unspecified; \ - } \ - KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER_TYPE(NAME) get_##NAME() const noexcept { \ - return KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER(NAME); \ - } \ - static_assert(true, "no-op to require trailing semicolon") - - public: - KOKKOS_IMPL_DECLARE(int, num_threads); - KOKKOS_IMPL_DECLARE(int, device_id); - KOKKOS_IMPL_DECLARE(std::string, map_device_id_by); - KOKKOS_IMPL_DECLARE(int, num_devices); // deprecated - KOKKOS_IMPL_DECLARE(int, skip_device); // deprecated - KOKKOS_IMPL_DECLARE(bool, disable_warnings); - KOKKOS_IMPL_DECLARE(bool, print_configuration); - KOKKOS_IMPL_DECLARE(bool, tune_internals); - KOKKOS_IMPL_DECLARE(bool, tools_help); - KOKKOS_IMPL_DECLARE(std::string, tools_libs); - KOKKOS_IMPL_DECLARE(std::string, tools_args); - -#undef KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER_TYPE -#undef KOKKOS_IMPL_INIT_ARGS_DATA_MEMBER -#undef KOKKOS_IMPL_DECLARE - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 - public: - InitializationSettings() = default; - - InitializationSettings(InitArguments const& old) { - if (old.num_threads != -1) { - set_num_threads(old.num_threads); - } - if (old.device_id != -1) { - set_device_id(old.device_id); - } - if (old.ndevices != -1) { - set_num_devices(old.ndevices); - } - if (old.skip_device != 9999) { - set_skip_device(old.skip_device); - } - if (old.disable_warnings) { - set_disable_warnings(true); - } - if (old.tune_internals) { - set_tune_internals(true); - } - if (old.tool_help) { - set_tools_help(true); - } - if (!old.tool_lib.empty()) { - set_tools_libs(old.tool_lib); - } - if (!old.tool_args.empty()) { - set_tools_args(old.tool_args); - } - } -#endif -}; - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemoryPoolAllocator.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemoryPoolAllocator.hpp deleted file mode 100644 index 7dede48a14..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemoryPoolAllocator.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -// Experimental unified task-data parallel manycore LDRD - -#ifndef KOKKOS_IMPL_MEMORYPOOLALLOCATOR_HPP -#define KOKKOS_IMPL_MEMORYPOOLALLOCATOR_HPP - -#include - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -namespace Kokkos { -namespace Impl { - -template -class MemoryPoolAllocator { - public: - using memory_pool = MemoryPool; - - private: - memory_pool m_pool; - - public: - KOKKOS_DEFAULTED_FUNCTION - MemoryPoolAllocator() = default; - KOKKOS_DEFAULTED_FUNCTION - MemoryPoolAllocator(MemoryPoolAllocator const&) = default; - KOKKOS_DEFAULTED_FUNCTION - MemoryPoolAllocator(MemoryPoolAllocator&&) = default; - KOKKOS_DEFAULTED_FUNCTION - MemoryPoolAllocator& operator=(MemoryPoolAllocator const&) = default; - KOKKOS_DEFAULTED_FUNCTION - MemoryPoolAllocator& operator=(MemoryPoolAllocator&&) = default; - KOKKOS_DEFAULTED_FUNCTION - ~MemoryPoolAllocator() = default; - - KOKKOS_INLINE_FUNCTION - explicit MemoryPoolAllocator(memory_pool const& arg_pool) - : m_pool(arg_pool) {} - KOKKOS_INLINE_FUNCTION - explicit MemoryPoolAllocator(memory_pool&& arg_pool) - : m_pool(std::move(arg_pool)) {} - - public: - using value_type = T; - using pointer = T*; - using size_type = typename MemoryPool::memory_space::size_type; - using difference_type = std::make_signed_t; - - template - struct rebind { - using other = MemoryPoolAllocator; - }; - - KOKKOS_INLINE_FUNCTION - pointer allocate(size_t n) { - void* rv = m_pool.allocate(n * sizeof(T)); - if (rv == nullptr) { - Kokkos::abort("Kokkos MemoryPool allocator failed to allocate memory"); - } - return reinterpret_cast(rv); - } - - KOKKOS_INLINE_FUNCTION - void deallocate(T* ptr, size_t n) { m_pool.deallocate(ptr, n * sizeof(T)); } - - KOKKOS_INLINE_FUNCTION - size_type max_size() const { return m_pool.max_block_size(); } - - KOKKOS_INLINE_FUNCTION - bool operator==(MemoryPoolAllocator const& other) const { - return m_pool == other.m_pool; - } - - KOKKOS_INLINE_FUNCTION - bool operator!=(MemoryPoolAllocator const& other) const { - return !(*this == other); - } -}; - -} // end namespace Impl -} // end namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #ifndef KOKKOS_IMPL_MEMORYPOOLALLOCATOR_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemorySpace.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemorySpace.cpp deleted file mode 100644 index a80ea0a1da..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemorySpace.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/** @file Kokkos_MemorySpace.cpp - * - * Operations common to memory space instances, or at least default - * implementations thereof. - */ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#include -#include -#include - -namespace Kokkos { -namespace Impl { - -void safe_throw_allocation_with_header_failure( - std::string const& space_name, std::string const& label, - Kokkos::Experimental::RawMemoryAllocationFailure const& failure) { - auto generate_failure_message = [&](std::ostream& o) { - o << "Kokkos failed to allocate memory for label \"" << label - << "\". Allocation using MemorySpace named \"" << space_name - << "\" failed with the following error: "; - failure.print_error_message(o); - if (failure.failure_mode() == - Kokkos::Experimental::RawMemoryAllocationFailure::FailureMode:: - AllocationNotAligned) { - // TODO: delete the misaligned memory? - o << "Warning: Allocation failed due to misalignment; memory may " - "be leaked.\n"; - } - o.flush(); - }; - try { - std::ostringstream sstr; - generate_failure_message(sstr); - Kokkos::Impl::throw_runtime_exception(sstr.str()); - } catch (std::bad_alloc const&) { - // Probably failed to allocate the string because we're so close to out - // of memory. Try printing to std::cerr instead - try { - generate_failure_message(std::cerr); - } catch (std::bad_alloc const&) { - // oh well, we tried... - } - Kokkos::Impl::throw_runtime_exception( - "Kokkos encountered an allocation failure, then another allocation " - "failure while trying to create the error message."); - } -} - -} // end namespace Impl -} // end namespace Kokkos diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemorySpace.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemorySpace.hpp deleted file mode 100644 index dee11bbdb4..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_MemorySpace.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/** @file Kokkos_MemorySpace.hpp - * - * Operations common to memory space instances, or at least default - * implementations thereof. - */ - -#ifndef KOKKOS_IMPL_MEMORYSPACE_HPP -#define KOKKOS_IMPL_MEMORYSPACE_HPP - -#include -#include -#include - -#include - -namespace Kokkos { -namespace Impl { - -// Defined in implementation file to avoid having to include iostream -void safe_throw_allocation_with_header_failure( - std::string const &space_name, std::string const &label, - Kokkos::Experimental::RawMemoryAllocationFailure const &failure); - -template -SharedAllocationHeader *checked_allocation_with_header(MemorySpace const &space, - std::string const &label, - size_t alloc_size) { - try { - return reinterpret_cast(space.allocate( - label.c_str(), alloc_size + sizeof(SharedAllocationHeader), - alloc_size)); - } catch (Kokkos::Experimental::RawMemoryAllocationFailure const &failure) { - safe_throw_allocation_with_header_failure(space.name(), label, failure); - } - return nullptr; // unreachable -} - -template -SharedAllocationHeader *checked_allocation_with_header( - ExecutionSpace const &exec_space, MemorySpace const &space, - std::string const &label, size_t alloc_size) { - try { - return reinterpret_cast(space.allocate( - exec_space, label.c_str(), alloc_size + sizeof(SharedAllocationHeader), - alloc_size)); - } catch (Kokkos::Experimental::RawMemoryAllocationFailure const &failure) { - safe_throw_allocation_with_header_failure(space.name(), label, failure); - } - return nullptr; // unreachable -} - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_IMPL_MEMORYSPACE_HPP diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Memory_Fence.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Memory_Fence.hpp deleted file mode 100644 index 1df5d13b95..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Memory_Fence.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#include -#if defined(KOKKOS_ATOMIC_HPP) && !defined(KOKKOS_MEMORY_FENCE_HPP) -#define KOKKOS_MEMORY_FENCE_HPP -namespace Kokkos { - -//---------------------------------------------------------------------------- -#ifndef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS -KOKKOS_FORCEINLINE_FUNCTION -void memory_fence() { -#if defined(__CUDA_ARCH__) - __threadfence(); -#elif defined(KOKKOS_ENABLE_OPENMPTARGET) -#pragma omp flush -#elif defined(__HIP_DEVICE_COMPILE__) - __threadfence(); -#elif defined(KOKKOS_ENABLE_SYCL) && defined(__SYCL_DEVICE_ONLY__) - sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::device); -#elif defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - asm volatile("mfence" ::: "memory"); -#elif defined(KOKKOS_ENABLE_GNU_ATOMICS) || \ - (defined(KOKKOS_COMPILER_NVCC) && defined(KOKKOS_ENABLE_INTEL_ATOMICS)) - __sync_synchronize(); -#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS) - _mm_mfence(); -#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS) -#pragma omp flush -#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS) - MemoryBarrier(); -#elif !defined(KOKKOS_ENABLE_SERIAL_ATOMICS) -#error "Error: memory_fence() not defined" -#endif -} -#endif - -////////////////////////////////////////////////////// -// store_fence() -// -// If possible use a store fence on the architecture, if not run a full memory -// fence - -KOKKOS_FORCEINLINE_FUNCTION -void store_fence() { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - asm volatile("sfence" ::: "memory"); -#else - memory_fence(); -#endif -} - -////////////////////////////////////////////////////// -// load_fence() -// -// If possible use a load fence on the architecture, if not run a full memory -// fence - -KOKKOS_FORCEINLINE_FUNCTION -void load_fence() { -#if defined(KOKKOS_ENABLE_ASM) && defined(KOKKOS_ENABLE_ISA_X86_64) - asm volatile("lfence" ::: "memory"); -#else - memory_fence(); -#endif -} - -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_NumericTraits.cpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_NumericTraits.cpp deleted file mode 100644 index 5ff094032f..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_NumericTraits.cpp +++ /dev/null @@ -1,82 +0,0 @@ - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -// NOTE These out-of class definitions are only required with C++14. Since -// C++17, a static data member declared constexpr is implicitly inline. - -#if !defined(KOKKOS_ENABLE_CXX17) -namespace Kokkos { -namespace Experimental { -namespace Impl { -#define OUT_OF_CLASS_DEFINITION_FLOATING_POINT(TRAIT) \ - constexpr float TRAIT##_helper::value; \ - constexpr double TRAIT##_helper::value; \ - constexpr long double TRAIT##_helper::value - -#define OUT_OF_CLASS_DEFINITION_INTEGRAL(TRAIT) \ - constexpr bool TRAIT##_helper::value; \ - constexpr char TRAIT##_helper::value; \ - constexpr signed char TRAIT##_helper::value; \ - constexpr unsigned char TRAIT##_helper::value; \ - constexpr short TRAIT##_helper::value; \ - constexpr unsigned short TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr unsigned int TRAIT##_helper::value; \ - constexpr long int TRAIT##_helper::value; \ - constexpr unsigned long int TRAIT##_helper::value; \ - constexpr long long int TRAIT##_helper::value; \ - constexpr unsigned long long int TRAIT##_helper::value - -#define OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(TRAIT) \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value - -#define OUT_OF_CLASS_DEFINITION_INTEGRAL_2(TRAIT) \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value; \ - constexpr int TRAIT##_helper::value - -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(infinity); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(epsilon); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(round_error); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(norm_min); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(denorm_min); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(reciprocal_overflow_threshold); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(quiet_NaN); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(signaling_NaN); - -OUT_OF_CLASS_DEFINITION_INTEGRAL(finite_min); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(finite_min); -OUT_OF_CLASS_DEFINITION_INTEGRAL(finite_max); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT(finite_max); - -OUT_OF_CLASS_DEFINITION_INTEGRAL_2(digits); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(digits); -OUT_OF_CLASS_DEFINITION_INTEGRAL_2(digits10); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(digits10); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(max_digits10); -OUT_OF_CLASS_DEFINITION_INTEGRAL_2(radix); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(radix); - -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(min_exponent); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(min_exponent10); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(max_exponent); -OUT_OF_CLASS_DEFINITION_FLOATING_POINT_2(max_exponent10); -} // namespace Impl -} // namespace Experimental -} // namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ParseCommandLineArgumentsAndEnvironmentVariables.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ParseCommandLineArgumentsAndEnvironmentVariables.hpp deleted file mode 100644 index 4fdb85b6a0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ParseCommandLineArgumentsAndEnvironmentVariables.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_PARSE_COMMAND_LINE_ARGUMENTS_AND_ENVIRONMENT_VARIABLES_HPP -#define KOKKOS_PARSE_COMMAND_LINE_ARGUMENTS_AND_ENVIRONMENT_VARIABLES_HPP - -// These declaration are only provided for testing purposes -namespace Kokkos { -class InitializationSettings; -namespace Impl { -void parse_command_line_arguments(int& argc, char* argv[], - InitializationSettings& settings); -void parse_environment_variables(InitializationSettings& settings); -} // namespace Impl -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_PhysicalLayout.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_PhysicalLayout.hpp deleted file mode 100644 index bc0a7df3e0..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_PhysicalLayout.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_PHYSICAL_LAYOUT_HPP -#define KOKKOS_PHYSICAL_LAYOUT_HPP - -#include - -namespace Kokkos { -namespace Impl { - -struct PhysicalLayout { - enum LayoutType { Left, Right, Scalar, Error }; - LayoutType layout_type; - int rank; - long long int stride[9]; // distance between two neighboring elements in a - // given dimension - - template - PhysicalLayout(const View& view) - : layout_type( - is_same::array_layout, LayoutLeft>::value - ? Left - : (is_same::array_layout, - LayoutRight>::value - ? Right - : Error)), - rank(view.Rank) { - for (int i = 0; i < 9; i++) stride[i] = 0; - view.stride(stride); - } -}; - -} // namespace Impl -} // namespace Kokkos -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Profiling_DeviceInfo.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Profiling_DeviceInfo.hpp deleted file mode 100644 index be6f756d0c..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Profiling_DeviceInfo.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - //@HEADER - // ************************************************************************ - // - // Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). - // - // Under the terms of Contract DE-NA0003525 with NTESS, - // the U.S. Government retains certain rights in this software. - // - // Redistribution and use in source and binary forms, with or without - // modification, are permitted provided that the following conditions are - // met: - // - // 1. Redistributions of source code must retain the above copyright - // notice, this list of conditions and the following disclaimer. - // - // 2. Redistributions in binary form must reproduce the above copyright - // notice, this list of conditions and the following disclaimer in the - // documentation and/or other materials provided with the distribution. - // - // 3. Neither the name of the Corporation nor the names of the - // contributors may be used to endorse or promote products derived from - // this software without specific prior written permission. - // - // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY - // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE - // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - // - // Questions? Contact Christian R. Trott (crtrott@sandia.gov) - // - // ************************************************************************ - //@HEADER -*/ - -#ifndef KOKKOSP_DEVICE_INFO_HPP -#define KOKKOSP_DEVICE_INFO_HPP - -#include -#include -namespace Kokkos { -namespace Profiling { -using KokkosPDeviceInfo = Kokkos_Profiling_KokkosPDeviceInfo; -} // namespace Profiling -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Spinwait.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Spinwait.hpp deleted file mode 100644 index 085157521d..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Spinwait.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SPINWAIT_HPP -#define KOKKOS_SPINWAIT_HPP - -#include -#include - -#include - -#include - -namespace Kokkos { -namespace Impl { - -enum class WaitMode : int { - ACTIVE // Used for tight loops to keep threads active longest - , - PASSIVE // Used to quickly yield the thread to quite down the system - , - ROOT // Never sleep or yield the root thread -}; - -void host_thread_yield(const uint32_t i, const WaitMode mode); - -template -std::enable_if_t::value, void> root_spinwait_while_equal( - T const volatile& flag, const T value) { - Kokkos::store_fence(); - uint32_t i = 0; - while (value == flag) { - host_thread_yield(++i, WaitMode::ROOT); - } - Kokkos::load_fence(); -} - -template -std::enable_if_t::value, void> root_spinwait_until_equal( - T const volatile& flag, const T value) { - Kokkos::store_fence(); - uint32_t i = 0; - while (value != flag) { - host_thread_yield(++i, WaitMode::ROOT); - } - Kokkos::load_fence(); -} - -template -std::enable_if_t::value, void> spinwait_while_equal( - T const volatile& flag, const T value) { - Kokkos::store_fence(); - uint32_t i = 0; - while (value == flag) { - host_thread_yield(++i, WaitMode::ACTIVE); - } - Kokkos::load_fence(); -} - -template -std::enable_if_t::value, void> yield_while_equal( - T const volatile& flag, const T value) { - Kokkos::store_fence(); - uint32_t i = 0; - while (value == flag) { - host_thread_yield(++i, WaitMode::PASSIVE); - } - Kokkos::load_fence(); -} - -template -std::enable_if_t::value, void> spinwait_until_equal( - T const volatile& flag, const T value) { - Kokkos::store_fence(); - uint32_t i = 0; - while (value != flag) { - host_thread_yield(++i, WaitMode::ACTIVE); - } - Kokkos::load_fence(); -} - -template -std::enable_if_t::value, void> yield_until_equal( - T const volatile& flag, const T value) { - Kokkos::store_fence(); - uint32_t i = 0; - while (value != flag) { - host_thread_yield(++i, WaitMode::PASSIVE); - } - Kokkos::load_fence(); -} - -} /* namespace Impl */ -} /* namespace Kokkos */ - -#endif /* #ifndef KOKKOS_SPINWAIT_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_TaskQueueMultiple_impl.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_TaskQueueMultiple_impl.hpp deleted file mode 100644 index 1af97918b8..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_TaskQueueMultiple_impl.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPL_TASKQUEUEMULTIPLE_IMPL_HPP -#define KOKKOS_IMPL_TASKQUEUEMULTIPLE_IMPL_HPP - -#include -#if defined(KOKKOS_ENABLE_TASKDAG) - -#include - -#define KOKKOS_IMPL_DEBUG_TASKDAG_SCHEDULING_MULTIPLE 0 - -namespace Kokkos { -namespace Impl { - -template -void TaskQueueMultiple::Destroy::destroy_shared_allocation() { -// KOKKOS WORKAROUND for CUDA 10.1 with GCC 7.3.0 -#if (KOKKOS_COMPILER_CUDA_VERSION == 101) && defined(KOKKOS_COMPILER_NVCC) && \ - (KOKKOS_COMPILER_GNU >= 730) - (*m_queue).get_team_queue(0).~TaskQueueMultiple(); -#else - m_queue->get_team_queue(0).~TaskQueueMultiple(); -#endif -} - -} /* namespace Impl */ -} /* namespace Kokkos */ - -#endif /* #if defined( KOKKOS_ENABLE_TASKDAG ) */ -#endif /* #ifndef KOKKOS_IMPL_TASKQUEUEMULTIPLE_IMPL_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Timer.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Timer.hpp deleted file mode 100644 index 6edf571d78..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Timer.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_IMPLWALLTIME_HPP -#define KOKKOS_IMPLWALLTIME_HPP - -#include - -KOKKOS_IMPL_WARNING("This file is deprecated. Use instead.") - -#include - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3 -namespace Kokkos { -namespace Impl { - -/** \brief Time since construction - * Timer promoted from Impl to Kokkos ns - * This file included for backwards compatibility - */ -using Timer KOKKOS_DEPRECATED_WITH_COMMENT("Use Kokkos::Timer instead!") = - Kokkos::Timer; - -} // namespace Impl -} // namespace Kokkos -#endif - -#endif /* #ifndef KOKKOS_IMPLWALLTIME_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Tools.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Tools.hpp deleted file mode 100644 index 8d6ec64685..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_Tools.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -/** - * Header file to include all of Kokkos Tooling support - */ - -#ifndef KOKKOS_IMPL_KOKKOS_TOOLS_HPP -#define KOKKOS_IMPL_KOKKOS_TOOLS_HPP - -#include - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewArray.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewArray.hpp deleted file mode 100644 index 12a5fa288f..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewArray.hpp +++ /dev/null @@ -1,636 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_EXPERIMENTAL_VIEW_ARRAY_MAPPING_HPP -#define KOKKOS_EXPERIMENTAL_VIEW_ARRAY_MAPPING_HPP - -#include - -namespace Kokkos { -namespace Impl { - -template -struct ViewDataAnalysis> { - private: - using array_analysis = ViewArrayAnalysis; - - static_assert(std::is_void

::value, ""); - static_assert(std::is_same>::value, - ""); - static_assert(std::is_scalar::value, - "View of Array type must be of a scalar type"); - - public: - using specialize = Kokkos::Array<>; - - using dimension = typename array_analysis::dimension; - - private: - enum { - is_const = std::is_same::value - }; - - using array_scalar_dimension = typename dimension::template append::type; - - using scalar_type = std::conditional_t; - using non_const_scalar_type = V; - using const_scalar_type = const V; - - public: - using value_type = typename array_analysis::value_type; - using const_value_type = typename array_analysis::const_value_type; - using non_const_value_type = typename array_analysis::non_const_value_type; - - using type = typename ViewDataType::type; - using const_type = typename ViewDataType::type; - using non_const_type = - typename ViewDataType::type; - - using scalar_array_type = - typename ViewDataType::type; - using const_scalar_array_type = - typename ViewDataType::type; - using non_const_scalar_array_type = - typename ViewDataType::type; -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -/** \brief View mapping for non-specialized data type and standard layout */ -template -class ViewMapping> { - private: - template - friend class ViewMapping; - template - friend class Kokkos::View; - - using offset_type = ViewOffset; - - using handle_type = typename Traits::value_type::pointer; - - handle_type m_impl_handle; - offset_type m_impl_offset; - size_t m_stride = 0; - - using scalar_type = typename Traits::value_type::value_type; - - using contiguous_reference = Kokkos::Array::contiguous>; - using strided_reference = - Kokkos::Array::strided>; - - enum { - is_contiguous_reference = - (Traits::rank == 0) || (std::is_same::value) - }; - - enum { Array_N = Traits::value_type::size() }; - enum { Array_S = is_contiguous_reference ? Array_N : 1 }; - - KOKKOS_INLINE_FUNCTION - ViewMapping(const handle_type &arg_handle, const offset_type &arg_offset) - : m_impl_handle(arg_handle), - m_impl_offset(arg_offset), - m_stride(is_contiguous_reference ? 0 : arg_offset.span()) {} - - public: - //---------------------------------------- - // Domain dimensions - - enum { Rank = Traits::dimension::rank }; - - template - KOKKOS_INLINE_FUNCTION constexpr size_t extent(const iType &r) const { - return m_impl_offset.m_dim.extent(r); - } - - KOKKOS_INLINE_FUNCTION constexpr typename Traits::array_layout layout() - const { - return m_impl_offset.layout(); - } - - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const { - return m_impl_offset.dimension_0(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const { - return m_impl_offset.dimension_1(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const { - return m_impl_offset.dimension_2(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const { - return m_impl_offset.dimension_3(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const { - return m_impl_offset.dimension_4(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const { - return m_impl_offset.dimension_5(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const { - return m_impl_offset.dimension_6(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const { - return m_impl_offset.dimension_7(); - } - - // Is a regular layout with uniform striding for each index. - using is_regular = typename offset_type::is_regular; - - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { - return m_impl_offset.stride_0(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { - return m_impl_offset.stride_1(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { - return m_impl_offset.stride_2(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { - return m_impl_offset.stride_3(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { - return m_impl_offset.stride_4(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { - return m_impl_offset.stride_5(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { - return m_impl_offset.stride_6(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { - return m_impl_offset.stride_7(); - } - - //---------------------------------------- - // Range span - - /** \brief Span of the mapped range */ - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { - return m_impl_offset.span() * Array_N; - } - - /** \brief Is the mapped range span contiguous */ - KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { - return m_impl_offset.span_is_contiguous(); - } - - using reference_type = - std::conditional_t; - - using pointer_type = handle_type; - - /** \brief If data references are lvalue_reference than can query pointer to - * memory */ - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { - return m_impl_handle; - } - - //---------------------------------------- - // The View class performs all rank and bounds checking before - // calling these element reference methods. - - KOKKOS_FORCEINLINE_FUNCTION - reference_type reference() const { - return reference_type(m_impl_handle + 0, Array_N, 0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0 &i0) const { - return reference_type(m_impl_handle + m_impl_offset(i0) * Array_S, Array_N, - m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0 &i0, - const I1 &i1) const { - return reference_type(m_impl_handle + m_impl_offset(i0, i1) * Array_S, - Array_N, m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0 &i0, - const I1 &i1, - const I2 &i2) const { - return reference_type(m_impl_handle + m_impl_offset(i0, i1, i2) * Array_S, - Array_N, m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type - reference(const I0 &i0, const I1 &i1, const I2 &i2, const I3 &i3) const { - return reference_type( - m_impl_handle + m_impl_offset(i0, i1, i2, i3) * Array_S, Array_N, - m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0 &i0, - const I1 &i1, - const I2 &i2, - const I3 &i3, - const I4 &i4) const { - return reference_type( - m_impl_handle + m_impl_offset(i0, i1, i2, i3, i4) * Array_S, Array_N, - m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type - reference(const I0 &i0, const I1 &i1, const I2 &i2, const I3 &i3, - const I4 &i4, const I5 &i5) const { - return reference_type( - m_impl_handle + m_impl_offset(i0, i1, i2, i3, i4, i5) * Array_S, - Array_N, m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type - reference(const I0 &i0, const I1 &i1, const I2 &i2, const I3 &i3, - const I4 &i4, const I5 &i5, const I6 &i6) const { - return reference_type( - m_impl_handle + m_impl_offset(i0, i1, i2, i3, i4, i5, i6) * Array_S, - Array_N, m_stride); - } - - template - KOKKOS_FORCEINLINE_FUNCTION reference_type - reference(const I0 &i0, const I1 &i1, const I2 &i2, const I3 &i3, - const I4 &i4, const I5 &i5, const I6 &i6, const I7 &i7) const { - return reference_type( - m_impl_handle + m_impl_offset(i0, i1, i2, i3, i4, i5, i6, i7) * Array_S, - Array_N, m_stride); - } - - //---------------------------------------- - - private: - enum { MemorySpanMask = 8 - 1 /* Force alignment on 8 byte boundary */ }; - enum { MemorySpanSize = sizeof(scalar_type) }; - - public: - /** \brief Span, in bytes, of the referenced memory */ - KOKKOS_INLINE_FUNCTION constexpr size_t memory_span() const { - return (m_impl_offset.span() * Array_N * MemorySpanSize + MemorySpanMask) & - ~size_t(MemorySpanMask); - } - - //---------------------------------------- - - KOKKOS_DEFAULTED_FUNCTION ViewMapping() = default; - - //---------------------------------------- - - template - KOKKOS_INLINE_FUNCTION ViewMapping(pointer_type ptr, Args... args) - : m_impl_handle(ptr), - m_impl_offset(std::integral_constant(), args...), - m_stride(m_impl_offset.span()) {} - - //---------------------------------------- - - template - Kokkos::Impl::SharedAllocationRecord<> *allocate_shared( - Kokkos::Impl::ViewCtorProp const &arg_prop, - typename Traits::array_layout const &arg_layout, - bool execution_space_specified) { - using alloc_prop = Kokkos::Impl::ViewCtorProp; - - using execution_space = typename alloc_prop::execution_space; - using memory_space = typename Traits::memory_space; - using functor_type = - ViewValueFunctor; - using record_type = - Kokkos::Impl::SharedAllocationRecord; - - // Query the mapping for byte-size of allocation. - using padding = std::integral_constant< - unsigned int, alloc_prop::allow_padding ? sizeof(scalar_type) : 0>; - - m_impl_offset = offset_type(padding(), arg_layout); - - const size_t alloc_size = - (m_impl_offset.span() * Array_N * MemorySpanSize + MemorySpanMask) & - ~size_t(MemorySpanMask); - const auto &alloc_name = - static_cast const &>( - arg_prop) - .value; - const execution_space &exec_space = - static_cast const &>( - arg_prop) - .value; - const memory_space &mem_space = - static_cast const &>( - arg_prop) - .value; - - // Allocate memory from the memory space and create tracking record. - record_type *const record = - execution_space_specified - ? record_type::allocate(exec_space, mem_space, alloc_name, - alloc_size) - : record_type::allocate(mem_space, alloc_name, alloc_size); - - if (alloc_size) { - m_impl_handle = - handle_type(reinterpret_cast(record->data())); - - if (alloc_prop::initialize) { - // The functor constructs and destroys - record->m_destroy = - execution_space_specified - ? functor_type(exec_space, (pointer_type)m_impl_handle, - m_impl_offset.span() * Array_N, alloc_name) - : functor_type((pointer_type)m_impl_handle, - m_impl_offset.span() * Array_N, alloc_name); - - record->m_destroy.construct_shared_allocation(); - } - } - - return record; - } -}; - -/** \brief Assign Array to non-Array */ - -template -class ViewMapping< - DstTraits, SrcTraits, - std::enable_if_t<( - std::is_same::value && - std::is_void::value && - (std::is_same::value || - std::is_same::value || - std::is_same::value) && - std::is_same>::value && - (std::is_same::value || - std::is_same::value || - std::is_same::value))>> { - public: - // Can only convert to View::array_type - - enum { - is_assignable_data_type = - std::is_same::value && - (DstTraits::rank == SrcTraits::rank + 1) - }; - enum { - is_assignable = - std::is_same::value && - std::is_same::value - }; - - using TrackType = Kokkos::Impl::SharedAllocationTracker; - using DstType = ViewMapping; - using SrcType = ViewMapping>; - - KOKKOS_INLINE_FUNCTION - static void assign(DstType &dst, const SrcType &src, - const TrackType & /*src_track*/) { - static_assert(is_assignable, "Can only convert to array_type"); - - using dst_offset_type = typename DstType::offset_type; - - // Array dimension becomes the last dimension. - // Arguments beyond the destination rank are ignored. - if (src.span_is_contiguous()) { // not padded - dst.m_impl_offset = dst_offset_type( - std::integral_constant(), - typename DstTraits::array_layout( - (0 < SrcType::Rank ? src.dimension_0() - : SrcTraits::value_type::size()), - (1 < SrcType::Rank ? src.dimension_1() - : SrcTraits::value_type::size()), - (2 < SrcType::Rank ? src.dimension_2() - : SrcTraits::value_type::size()), - (3 < SrcType::Rank ? src.dimension_3() - : SrcTraits::value_type::size()), - (4 < SrcType::Rank ? src.dimension_4() - : SrcTraits::value_type::size()), - (5 < SrcType::Rank ? src.dimension_5() - : SrcTraits::value_type::size()), - (6 < SrcType::Rank ? src.dimension_6() - : SrcTraits::value_type::size()), - (7 < SrcType::Rank ? src.dimension_7() - : SrcTraits::value_type::size()))); - } else { // is padded - using padded = std::integral_constant< - unsigned int, sizeof(typename SrcTraits::value_type::value_type)>; - - dst.m_impl_offset = dst_offset_type( - padded(), typename DstTraits::array_layout( - (0 < SrcType::Rank ? src.dimension_0() - : SrcTraits::value_type::size()), - (1 < SrcType::Rank ? src.dimension_1() - : SrcTraits::value_type::size()), - (2 < SrcType::Rank ? src.dimension_2() - : SrcTraits::value_type::size()), - (3 < SrcType::Rank ? src.dimension_3() - : SrcTraits::value_type::size()), - (4 < SrcType::Rank ? src.dimension_4() - : SrcTraits::value_type::size()), - (5 < SrcType::Rank ? src.dimension_5() - : SrcTraits::value_type::size()), - (6 < SrcType::Rank ? src.dimension_6() - : SrcTraits::value_type::size()), - (7 < SrcType::Rank ? src.dimension_7() - : SrcTraits::value_type::size()))); - } - - dst.m_impl_handle = src.m_impl_handle; - } -}; - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -class ViewMapping< - std::enable_if_t<( - std::is_same>::value && - (std::is_same::value || - std::is_same::value || - std::is_same::value))>, - SrcTraits, Args...> { - private: - static_assert(SrcTraits::rank == sizeof...(Args), ""); - - enum : bool { - R0 = is_integral_extent<0, Args...>::value, - R1 = is_integral_extent<1, Args...>::value, - R2 = is_integral_extent<2, Args...>::value, - R3 = is_integral_extent<3, Args...>::value, - R4 = is_integral_extent<4, Args...>::value, - R5 = is_integral_extent<5, Args...>::value, - R6 = is_integral_extent<6, Args...>::value, - R7 = is_integral_extent<7, Args...>::value - }; - - enum { - rank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) + - unsigned(R4) + unsigned(R5) + unsigned(R6) + unsigned(R7) - }; - - // Whether right-most rank is a range. - enum { - R0_rev = - 0 == SrcTraits::rank - ? false - : (1 == SrcTraits::rank - ? R0 - : (2 == SrcTraits::rank - ? R1 - : (3 == SrcTraits::rank - ? R2 - : (4 == SrcTraits::rank - ? R3 - : (5 == SrcTraits::rank - ? R4 - : (6 == SrcTraits::rank - ? R5 - : (7 == SrcTraits::rank - ? R6 - : R7))))))) - }; - - // Subview's layout - using array_layout = - std::conditional_t<((rank == 0) || - (rank <= 2 && R0 && - std::is_same::value) || - (rank <= 2 && R0_rev && - std::is_same::value)), - typename SrcTraits::array_layout, - Kokkos::LayoutStride>; - - using value_type = typename SrcTraits::value_type; - - using data_type = std::conditional_t< - rank == 0, value_type, - std::conditional_t< - rank == 1, value_type *, - std::conditional_t< - rank == 2, value_type **, - std::conditional_t< - rank == 3, value_type ***, - std::conditional_t< - rank == 4, value_type ****, - std::conditional_t< - rank == 5, value_type *****, - std::conditional_t< - rank == 6, value_type ******, - std::conditional_t>>>>>>>; - - public: - using traits_type = Kokkos::ViewTraits; - - using type = - Kokkos::View; - - KOKKOS_INLINE_FUNCTION - static void assign(ViewMapping &dst, - ViewMapping const &src, Args... args) { - using DstType = ViewMapping; - - using dst_offset_type = typename DstType::offset_type; - using dst_handle_type = typename DstType::handle_type; - - const SubviewExtents extents(src.m_impl_offset.m_dim, - args...); - - dst.m_impl_offset = dst_offset_type(src.m_impl_offset, extents); - dst.m_impl_handle = dst_handle_type( - src.m_impl_handle + - src.m_impl_offset(extents.domain_offset(0), extents.domain_offset(1), - extents.domain_offset(2), extents.domain_offset(3), - extents.domain_offset(4), extents.domain_offset(5), - extents.domain_offset(6), extents.domain_offset(7))); - } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #ifndef KOKKOS_EXPERIMENTAL_VIEW_ARRAY_MAPPING_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewCtor.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewCtor.hpp deleted file mode 100644 index 8bc8f86864..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewCtor.hpp +++ /dev/null @@ -1,313 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_EXPERIMENTAL_IMPL_VIEW_CTOR_PROP_HPP -#define KOKKOS_EXPERIMENTAL_IMPL_VIEW_CTOR_PROP_HPP - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -struct WithoutInitializing_t {}; -struct AllowPadding_t {}; - -template -struct is_view_ctor_property : public std::false_type {}; - -template <> -struct is_view_ctor_property : public std::true_type {}; - -template <> -struct is_view_ctor_property : public std::true_type {}; - -//---------------------------------------------------------------------------- -/**\brief Whether a type can be used for a view label */ - -template -struct is_view_label : public std::false_type {}; - -template <> -struct is_view_label : public std::true_type {}; - -template -struct is_view_label : public std::true_type {}; - -template -struct is_view_label : public std::true_type {}; - -//---------------------------------------------------------------------------- - -template -struct ViewCtorProp; - -// Forward declare -template -struct CommonViewAllocProp; - -/* Dummy to allow for empty ViewCtorProp object - */ -template <> -struct ViewCtorProp {}; - -/* Common value_type stored as ViewCtorProp - */ -template -struct ViewCtorProp> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - using type = CommonViewAllocProp; - - KOKKOS_INLINE_FUNCTION - ViewCtorProp(const type &arg) : value(arg) {} - KOKKOS_INLINE_FUNCTION - ViewCtorProp(type &&arg) : value(arg) {} - - type value; -}; - -/* std::integral_constant are dummy arguments - * that avoid duplicate base class errors - */ -template -struct ViewCtorProp> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - template - KOKKOS_INLINE_FUNCTION ViewCtorProp(const P &) {} -}; - -/* Property flags have constexpr value */ -template -struct ViewCtorProp< - std::enable_if_t::value || - std::is_same::value>, - P> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - using type = P; - - ViewCtorProp(const type &) {} - - type value = type(); -}; - -/* Map input label type to std::string */ -template -struct ViewCtorProp::value>, Label> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - using type = std::string; - - ViewCtorProp(const type &arg) : value(arg) {} - ViewCtorProp(type &&arg) : value(arg) {} - - type value; -}; - -template -struct ViewCtorProp::value || - Kokkos::is_execution_space::value>, - Space> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - using type = Space; - - ViewCtorProp(const type &arg) : value(arg) {} - - type value; -}; - -template -struct ViewCtorProp { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - using type = T *; - - KOKKOS_INLINE_FUNCTION - ViewCtorProp(const type arg) : value(arg) {} - - type value; -}; - -// For some reason I don't understand I needed this specialization explicitly -// for NVCC/MSVC -template -struct ViewCtorProp { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; - ViewCtorProp &operator=(const ViewCtorProp &) = default; - - using type = T *; - - KOKKOS_INLINE_FUNCTION - ViewCtorProp(const type arg) : value(arg) {} - - enum : bool { has_pointer = true }; - using pointer_type = type; - type value; -}; - -// If we use `ViewCtorProp` and `ViewCtorProp...` directly -// in the parameter lists and base class initializers, respectively, as far as -// we can tell MSVC 16.5.5+CUDA 10.2 thinks that `ViewCtorProp` refers to the -// current instantiation, not the template itself, and gets all kinds of -// confused. To work around this, we just use a couple of alias templates that -// amount to the same thing. -template -using view_ctor_prop_args = ViewCtorProp; - -template -using view_ctor_prop_base = ViewCtorProp; - -template -struct ViewCtorProp : public ViewCtorProp... { - private: - using var_memory_space = - Kokkos::Impl::has_condition; - - using var_execution_space = - Kokkos::Impl::has_condition; - - struct VOIDDUMMY {}; - - using var_pointer = - Kokkos::Impl::has_condition; - - public: - /* Flags for the common properties */ - enum { has_memory_space = var_memory_space::value }; - enum { has_execution_space = var_execution_space::value }; - enum { has_pointer = var_pointer::value }; - enum { has_label = Kokkos::Impl::has_type::value }; - enum { allow_padding = Kokkos::Impl::has_type::value }; - enum { - initialize = !Kokkos::Impl::has_type::value - }; - - using memory_space = typename var_memory_space::type; - using execution_space = typename var_execution_space::type; - using pointer_type = typename var_pointer::type; - - /* Copy from a matching argument list. - * Requires std::is_same< P , ViewCtorProp< void , Args >::value ... - */ - template - inline ViewCtorProp(Args const &... args) : ViewCtorProp(args)... {} - - template - KOKKOS_INLINE_FUNCTION ViewCtorProp(pointer_type arg0, Args const &... args) - : ViewCtorProp(arg0), - ViewCtorProp::type>(args)... {} - - /* Copy from a matching property subset */ - KOKKOS_INLINE_FUNCTION ViewCtorProp(pointer_type arg0) - : ViewCtorProp(arg0) {} - - // If we use `ViewCtorProp` and `ViewCtorProp...` here - // directly, MSVC 16.5.5+CUDA 10.2 appears to think that `ViewCtorProp` refers - // to the current instantiation, not the template itself, and gets all kinds - // of confused. To work around this, we just use a couple of alias templates - // that amount to the same thing. - template - ViewCtorProp(view_ctor_prop_args const &arg) - : view_ctor_prop_base( - static_cast const &>(arg))... { - // Suppress an unused argument warning that (at least at one point) would - // show up if sizeof...(Args) == 0 - (void)arg; - } -}; - -} /* namespace Impl */ -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -namespace Impl { -struct ViewAllocateWithoutInitializingBackwardCompat {}; - -template <> -struct ViewCtorProp {}; - -// NOTE This specialization is meant to be used as the -// ViewAllocateWithoutInitializing alias below. All it does is add a -// constructor that takes the label as single argument. -template <> -struct ViewCtorProp - : ViewCtorProp, - ViewCtorProp { - ViewCtorProp(std::string label) - : ViewCtorProp( - WithoutInitializing_t(), std::move(label)) {} -}; -} /* namespace Impl */ - -using ViewAllocateWithoutInitializing = - Impl::ViewCtorProp; - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewLayoutTiled.hpp b/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewLayoutTiled.hpp deleted file mode 100644 index 8d367cebab..0000000000 --- a/bundled/kokkos-3.7.00/core/src/impl/Kokkos_ViewLayoutTiled.hpp +++ /dev/null @@ -1,1453 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_EXPERIMENTAL_VIEWLAYOUTTILE_HPP -#define KOKKOS_EXPERIMENTAL_VIEWLAYOUTTILE_HPP - -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -// View offset and mapping for tiled view's - -template -struct is_array_layout> - : public std::true_type {}; - -template -struct is_array_layout> - : public std::true_type {}; - -template -struct is_array_layout> - : public std::true_type {}; - -template -struct is_array_layout> - : public std::true_type {}; - -template -struct is_array_layout> - : public std::true_type {}; - -template -struct is_array_layout> - : public std::true_type {}; - -template -struct is_array_layout< - Kokkos::Experimental::LayoutTiled> - : public std::true_type {}; - -template -struct is_array_layout_tiled : public std::false_type {}; - -template -struct is_array_layout_tiled> : public std::true_type { -}; // Last template parameter "true" meaning this currently only supports - // powers-of-two - -namespace Impl { - -template -struct ViewOffset< - Dimension, Layout, - std::enable_if_t<((Dimension::rank <= 8) && (Dimension::rank >= 2) && - is_array_layout::value && - is_array_layout_tiled::value)>> { - public: - static constexpr Kokkos::Iterate outer_pattern = Layout::outer_pattern; - static constexpr Kokkos::Iterate inner_pattern = Layout::inner_pattern; - - static constexpr int VORank = Dimension::rank; - - static constexpr unsigned SHIFT_0 = - Kokkos::Impl::integral_power_of_two(Layout::N0); - static constexpr unsigned SHIFT_1 = - Kokkos::Impl::integral_power_of_two(Layout::N1); - static constexpr unsigned SHIFT_2 = - Kokkos::Impl::integral_power_of_two(Layout::N2); - static constexpr unsigned SHIFT_3 = - Kokkos::Impl::integral_power_of_two(Layout::N3); - static constexpr unsigned SHIFT_4 = - Kokkos::Impl::integral_power_of_two(Layout::N4); - static constexpr unsigned SHIFT_5 = - Kokkos::Impl::integral_power_of_two(Layout::N5); - static constexpr unsigned SHIFT_6 = - Kokkos::Impl::integral_power_of_two(Layout::N6); - static constexpr unsigned SHIFT_7 = - Kokkos::Impl::integral_power_of_two(Layout::N7); - static constexpr int MASK_0 = Layout::N0 - 1; - static constexpr int MASK_1 = Layout::N1 - 1; - static constexpr int MASK_2 = Layout::N2 - 1; - static constexpr int MASK_3 = Layout::N3 - 1; - static constexpr int MASK_4 = Layout::N4 - 1; - static constexpr int MASK_5 = Layout::N5 - 1; - static constexpr int MASK_6 = Layout::N6 - 1; - static constexpr int MASK_7 = Layout::N7 - 1; - - static constexpr unsigned SHIFT_2T = SHIFT_0 + SHIFT_1; - static constexpr unsigned SHIFT_3T = SHIFT_0 + SHIFT_1 + SHIFT_2; - static constexpr unsigned SHIFT_4T = SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3; - static constexpr unsigned SHIFT_5T = - SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4; - static constexpr unsigned SHIFT_6T = - SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4 + SHIFT_5; - static constexpr unsigned SHIFT_7T = - SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4 + SHIFT_5 + SHIFT_6; - static constexpr unsigned SHIFT_8T = SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + - SHIFT_4 + SHIFT_5 + SHIFT_6 + SHIFT_7; - - // Is an irregular layout that does not have uniform striding for each index. - using is_mapping_plugin = std::true_type; - using is_regular = std::false_type; - - using size_type = size_t; - using dimension_type = Dimension; - using array_layout = Layout; - - dimension_type m_dim; - size_type m_tile_N0; // Num tiles dim 0 - size_type m_tile_N1; - size_type m_tile_N2; - size_type m_tile_N3; - size_type m_tile_N4; - size_type m_tile_N5; - size_type m_tile_N6; - size_type m_tile_N7; - - //---------------------------------------- - -#define DEBUG_OUTPUT_CHECK 0 - - // Rank 2 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, - I1 const& i1) const { - auto tile_offset = - (outer_pattern == (Kokkos::Iterate::Left)) - ? (((i0 >> SHIFT_0) + m_tile_N0 * ((i1 >> SHIFT_1))) << SHIFT_2T) - : (((m_tile_N1 * (i0 >> SHIFT_0) + (i1 >> SHIFT_1))) << SHIFT_2T); - // ( num_tiles[1] * ti0 + ti1 ) * FTD - - auto local_offset = (inner_pattern == (Kokkos::Iterate::Left)) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0)) - : (((i0 & MASK_0) << SHIFT_1) + (i1 & MASK_1)); - // ( tile_dim[1] * li0 + li1 ) - -#if DEBUG_OUTPUT_CHECK - std::cout << "Am I Outer Left? " - << (outer_pattern == (Kokkos::Iterate::Left)) << std::endl; - std::cout << "Am I Inner Left? " - << (inner_pattern == (Kokkos::Iterate::Left)) << std::endl; - std::cout << "i0 = " << i0 << " i1 = " << i1 - << "\ntilei0 = " << (i0 >> SHIFT_0) - << " tilei1 = " << (i1 >> SHIFT_1) - << "locali0 = " << (i0 & MASK_0) - << "\nlocali1 = " << (i1 & MASK_1) << std::endl; -#endif - - return tile_offset + local_offset; - } - - // Rank 3 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, I1 const& i1, - I2 const& i2) const { - auto tile_offset = - (outer_pattern == Kokkos::Iterate::Left) - ? (((i0 >> SHIFT_0) + - m_tile_N0 * ((i1 >> SHIFT_1) + m_tile_N1 * (i2 >> SHIFT_2))) - << SHIFT_3T) - : ((m_tile_N2 * (m_tile_N1 * (i0 >> SHIFT_0) + (i1 >> SHIFT_1)) + - (i2 >> SHIFT_2)) - << SHIFT_3T); - - auto local_offset = (inner_pattern == Kokkos::Iterate::Left) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0) + - ((i2 & MASK_2) << (SHIFT_0 + SHIFT_1))) - : (((i0 & MASK_0) << (SHIFT_2 + SHIFT_1)) + - ((i1 & MASK_1) << (SHIFT_2)) + (i2 & MASK_2)); - -#if DEBUG_OUTPUT_CHECK - std::cout << "Am I Outer Left? " - << (outer_pattern == (Kokkos::Iterate::Left)) << std::endl; - std::cout << "Am I Inner Left? " - << (inner_pattern == (Kokkos::Iterate::Left)) << std::endl; - std::cout << "i0 = " << i0 << " i1 = " << i1 << " i2 = " << i2 - << "\ntilei0 = " << (i0 >> SHIFT_0) - << " tilei1 = " << (i1 >> SHIFT_1) - << " tilei2 = " << (i2 >> SHIFT_2) - << "\nlocali0 = " << (i0 & MASK_0) - << "locali1 = " << (i1 & MASK_1) << "locali2 = " << (i2 & MASK_2) - << std::endl; -#endif - - return tile_offset + local_offset; - } - - // Rank 4 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, I1 const& i1, - I2 const& i2, - I3 const& i3) const { - auto tile_offset = - (outer_pattern == Kokkos::Iterate::Left) - ? (((i0 >> SHIFT_0) + - m_tile_N0 * ((i1 >> SHIFT_1) + - m_tile_N1 * ((i2 >> SHIFT_2) + - m_tile_N2 * (i3 >> SHIFT_3)))) - << SHIFT_4T) - : ((m_tile_N3 * (m_tile_N2 * (m_tile_N1 * (i0 >> SHIFT_0) + - (i1 >> SHIFT_1)) + - (i2 >> SHIFT_2)) + - (i3 >> SHIFT_3)) - << SHIFT_4T); - - auto local_offset = - (inner_pattern == Kokkos::Iterate::Left) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0) + - ((i2 & MASK_2) << (SHIFT_0 + SHIFT_1)) + - ((i3 & MASK_3) << (SHIFT_0 + SHIFT_1 + SHIFT_2))) - : (((i0 & MASK_0) << (SHIFT_3 + SHIFT_2 + SHIFT_1)) + - ((i1 & MASK_1) << (SHIFT_3 + SHIFT_2)) + - ((i2 & MASK_2) << (SHIFT_3)) + (i3 & MASK_3)); - - return tile_offset + local_offset; - } - - // Rank 5 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, I1 const& i1, - I2 const& i2, I3 const& i3, - I4 const& i4) const { - auto tile_offset = - (outer_pattern == Kokkos::Iterate::Left) - ? (((i0 >> SHIFT_0) + - m_tile_N0 * - ((i1 >> SHIFT_1) + - m_tile_N1 * ((i2 >> SHIFT_2) + - m_tile_N2 * ((i3 >> SHIFT_3) + - m_tile_N3 * (i4 >> SHIFT_4))))) - << SHIFT_5T) - : ((m_tile_N4 * - (m_tile_N3 * (m_tile_N2 * (m_tile_N1 * (i0 >> SHIFT_0) + - (i1 >> SHIFT_1)) + - (i2 >> SHIFT_2)) + - (i3 >> SHIFT_3)) + - (i4 >> SHIFT_4)) - << SHIFT_5T); - - auto local_offset = - (inner_pattern == Kokkos::Iterate::Left) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0) + - ((i2 & MASK_2) << (SHIFT_0 + SHIFT_1)) + - ((i3 & MASK_3) << (SHIFT_0 + SHIFT_1 + SHIFT_2)) + - ((i4 & MASK_4) << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3))) - : (((i0 & MASK_0) << (SHIFT_4 + SHIFT_3 + SHIFT_2 + SHIFT_1)) + - ((i1 & MASK_1) << (SHIFT_4 + SHIFT_3 + SHIFT_2)) + - ((i2 & MASK_2) << (SHIFT_4 + SHIFT_3)) + - ((i3 & MASK_3) << (SHIFT_4)) + (i4 & MASK_4)); - - return tile_offset + local_offset; - } - - // Rank 6 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, I1 const& i1, - I2 const& i2, I3 const& i3, - I4 const& i4, - I5 const& i5) const { - auto tile_offset = - (outer_pattern == Kokkos::Iterate::Left) - ? (((i0 >> SHIFT_0) + - m_tile_N0 * - ((i1 >> SHIFT_1) + - m_tile_N1 * - ((i2 >> SHIFT_2) + - m_tile_N2 * - ((i3 >> SHIFT_3) + - m_tile_N3 * ((i4 >> SHIFT_4) + - m_tile_N4 * (i5 >> SHIFT_5)))))) - << SHIFT_6T) - : ((m_tile_N5 * - (m_tile_N4 * - (m_tile_N3 * - (m_tile_N2 * (m_tile_N1 * (i0 >> SHIFT_0) + - (i1 >> SHIFT_1)) + - (i2 >> SHIFT_2)) + - (i3 >> SHIFT_3)) + - (i4 >> SHIFT_4)) + - (i5 >> SHIFT_5)) - << SHIFT_6T); - - auto local_offset = - (inner_pattern == Kokkos::Iterate::Left) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0) + - ((i2 & MASK_2) << (SHIFT_0 + SHIFT_1)) + - ((i3 & MASK_3) << (SHIFT_0 + SHIFT_1 + SHIFT_2)) + - ((i4 & MASK_4) << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3)) + - ((i5 & MASK_5) - << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4))) - : (((i0 & MASK_0) - << (SHIFT_5 + SHIFT_4 + SHIFT_3 + SHIFT_2 + SHIFT_1)) + - ((i1 & MASK_1) << (SHIFT_5 + SHIFT_4 + SHIFT_3 + SHIFT_2)) + - ((i2 & MASK_2) << (SHIFT_5 + SHIFT_4 + SHIFT_3)) + - ((i3 & MASK_3) << (SHIFT_5 + SHIFT_4)) + - ((i4 & MASK_4) << (SHIFT_5)) + (i5 & MASK_5)); - - return tile_offset + local_offset; - } - - // Rank 7 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, I1 const& i1, - I2 const& i2, I3 const& i3, - I4 const& i4, I5 const& i5, - I6 const& i6) const { - auto tile_offset = - (outer_pattern == Kokkos::Iterate::Left) - ? (((i0 >> SHIFT_0) + - m_tile_N0 * - ((i1 >> SHIFT_1) + - m_tile_N1 * - ((i2 >> SHIFT_2) + - m_tile_N2 * - ((i3 >> SHIFT_3) + - m_tile_N3 * - ((i4 >> SHIFT_4) + - m_tile_N4 * - ((i5 >> SHIFT_5) + - m_tile_N5 * (i6 >> SHIFT_6))))))) - << SHIFT_7T) - : ((m_tile_N6 * - (m_tile_N5 * - (m_tile_N4 * - (m_tile_N3 * - (m_tile_N2 * (m_tile_N1 * (i0 >> SHIFT_0) + - (i1 >> SHIFT_1)) + - (i2 >> SHIFT_2)) + - (i3 >> SHIFT_3)) + - (i4 >> SHIFT_4)) + - (i5 >> SHIFT_5)) + - (i6 >> SHIFT_6)) - << SHIFT_7T); - - auto local_offset = - (inner_pattern == Kokkos::Iterate::Left) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0) + - ((i2 & MASK_2) << (SHIFT_0 + SHIFT_1)) + - ((i3 & MASK_3) << (SHIFT_0 + SHIFT_1 + SHIFT_2)) + - ((i4 & MASK_4) << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3)) + - ((i5 & MASK_5) - << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4)) + - ((i6 & MASK_6) - << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4 + SHIFT_5))) - : (((i0 & MASK_0) << (SHIFT_6 + SHIFT_5 + SHIFT_4 + SHIFT_3 + - SHIFT_2 + SHIFT_1)) + - ((i1 & MASK_1) - << (SHIFT_6 + SHIFT_5 + SHIFT_4 + SHIFT_3 + SHIFT_2)) + - ((i2 & MASK_2) << (SHIFT_6 + SHIFT_5 + SHIFT_4 + SHIFT_3)) + - ((i3 & MASK_3) << (SHIFT_6 + SHIFT_5 + SHIFT_4)) + - ((i4 & MASK_4) << (SHIFT_6 + SHIFT_5)) + - ((i5 & MASK_5) << (SHIFT_6)) + (i6 & MASK_6)); - - return tile_offset + local_offset; - } - - // Rank 8 - template - KOKKOS_INLINE_FUNCTION size_type operator()(I0 const& i0, I1 const& i1, - I2 const& i2, I3 const& i3, - I4 const& i4, I5 const& i5, - I6 const& i6, - I7 const& i7) const { - auto tile_offset = - (outer_pattern == Kokkos::Iterate::Left) - ? (((i0 >> SHIFT_0) + - m_tile_N0 * - ((i1 >> SHIFT_1) + - m_tile_N1 * - ((i2 >> SHIFT_2) + - m_tile_N2 * - ((i3 >> SHIFT_3) + - m_tile_N3 * - ((i4 >> SHIFT_4) + - m_tile_N4 * - ((i5 >> SHIFT_5) + - m_tile_N5 * - ((i6 >> SHIFT_6) + - m_tile_N6 * (i7 >> SHIFT_7)))))))) - << SHIFT_8T) - : ((m_tile_N7 * - (m_tile_N6 * - (m_tile_N5 * - (m_tile_N4 * - (m_tile_N3 * - (m_tile_N2 * - (m_tile_N1 * (i0 >> SHIFT_0) + - (i1 >> SHIFT_1)) + - (i2 >> SHIFT_2)) + - (i3 >> SHIFT_3)) + - (i4 >> SHIFT_4)) + - (i5 >> SHIFT_5)) + - (i6 >> SHIFT_6)) + - (i7 >> SHIFT_7)) - << SHIFT_8T); - - auto local_offset = - (inner_pattern == Kokkos::Iterate::Left) - ? ((i0 & MASK_0) + ((i1 & MASK_1) << SHIFT_0) + - ((i2 & MASK_2) << (SHIFT_0 + SHIFT_1)) + - ((i3 & MASK_3) << (SHIFT_0 + SHIFT_1 + SHIFT_2)) + - ((i4 & MASK_4) << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3)) + - ((i5 & MASK_5) - << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + SHIFT_4)) + - ((i6 & MASK_6) << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + - SHIFT_4 + SHIFT_5)) + - ((i7 & MASK_7) << (SHIFT_0 + SHIFT_1 + SHIFT_2 + SHIFT_3 + - SHIFT_4 + SHIFT_5 + SHIFT_6))) - : (((i0 & MASK_0) << (SHIFT_7 + SHIFT_6 + SHIFT_5 + SHIFT_4 + - SHIFT_3 + SHIFT_2 + SHIFT_1)) + - ((i1 & MASK_1) << (SHIFT_7 + SHIFT_6 + SHIFT_5 + SHIFT_4 + - SHIFT_3 + SHIFT_2)) + - ((i2 & MASK_2) - << (SHIFT_7 + SHIFT_6 + SHIFT_5 + SHIFT_4 + SHIFT_3)) + - ((i3 & MASK_3) << (SHIFT_7 + SHIFT_6 + SHIFT_5 + SHIFT_4)) + - ((i4 & MASK_4) << (SHIFT_7 + SHIFT_6 + SHIFT_5)) + - ((i5 & MASK_5) << (SHIFT_7 + SHIFT_6)) + - ((i6 & MASK_6) << (SHIFT_7)) + (i7 & MASK_7)); - - return tile_offset + local_offset; - } - - //---------------------------------------- - - KOKKOS_INLINE_FUNCTION constexpr array_layout layout() const { - return array_layout((VORank > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX), - (VORank > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX), - (VORank > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX), - (VORank > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX), - (VORank > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX), - (VORank > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX), - (VORank > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX), - (VORank > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX)); - } - - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const { - return m_dim.N0; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const { - return m_dim.N1; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const { - return m_dim.N2; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const { - return m_dim.N3; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const { - return m_dim.N4; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const { - return m_dim.N5; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const { - return m_dim.N6; - } - KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const { - return m_dim.N7; - } - - KOKKOS_INLINE_FUNCTION constexpr size_type size() const { - return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 * - m_dim.N6 * m_dim.N7; - } - - // Strides are meaningless due to irregularity - KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const { return 0; } - KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const { return 0; } - - // Stride with [ rank ] value is the total length - template - KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { - s[0] = 0; - if (0 < dimension_type::rank) { - s[1] = 0; - } - if (1 < dimension_type::rank) { - s[2] = 0; - } - if (2 < dimension_type::rank) { - s[3] = 0; - } - if (3 < dimension_type::rank) { - s[4] = 0; - } - if (4 < dimension_type::rank) { - s[5] = 0; - } - if (5 < dimension_type::rank) { - s[6] = 0; - } - if (6 < dimension_type::rank) { - s[7] = 0; - } - if (7 < dimension_type::rank) { - s[8] = 0; - } - } - - KOKKOS_INLINE_FUNCTION constexpr size_type span() const { - // Rank2: ( NumTile0 * ( NumTile1 ) ) * TileSize, etc - return (VORank == 2) - ? (m_tile_N0 * m_tile_N1) << SHIFT_2T - : (VORank == 3) - ? (m_tile_N0 * m_tile_N1 * m_tile_N2) << SHIFT_3T - : (VORank == 4) - ? (m_tile_N0 * m_tile_N1 * m_tile_N2 * m_tile_N3) - << SHIFT_4T - : (VORank == 5) - ? (m_tile_N0 * m_tile_N1 * m_tile_N2 * - m_tile_N3 * m_tile_N4) - << SHIFT_5T - : (VORank == 6) - ? (m_tile_N0 * m_tile_N1 * m_tile_N2 * - m_tile_N3 * m_tile_N4 * m_tile_N5) - << SHIFT_6T - : (VORank == 7) - ? (m_tile_N0 * m_tile_N1 * - m_tile_N2 * m_tile_N3 * - m_tile_N4 * m_tile_N5 * - m_tile_N6) - << SHIFT_7T - : (m_tile_N0 * m_tile_N1 * - m_tile_N2 * m_tile_N3 * - m_tile_N4 * m_tile_N5 * - m_tile_N6 * m_tile_N7) - << SHIFT_8T; - } - - KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { - return true; - } - - //---------------------------------------- -#ifdef KOKKOS_IMPL_WINDOWS_CUDA - KOKKOS_FUNCTION ViewOffset() {} - KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) { - m_dim = src.m_dim; - m_tile_N0 = src.m_tile_N0; - m_tile_N1 = src.m_tile_N1; - m_tile_N2 = src.m_tile_N2; - m_tile_N3 = src.m_tile_N3; - m_tile_N4 = src.m_tile_N4; - m_tile_N5 = src.m_tile_N5; - m_tile_N6 = src.m_tile_N6; - m_tile_N7 = src.m_tile_N7; - } - KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) { - m_dim = src.m_dim; - m_tile_N0 = src.m_tile_N0; - m_tile_N1 = src.m_tile_N1; - m_tile_N2 = src.m_tile_N2; - m_tile_N3 = src.m_tile_N3; - m_tile_N4 = src.m_tile_N4; - m_tile_N5 = src.m_tile_N5; - m_tile_N6 = src.m_tile_N6; - m_tile_N7 = src.m_tile_N7; - return *this; - } -#else - KOKKOS_DEFAULTED_FUNCTION ~ViewOffset() = default; - KOKKOS_DEFAULTED_FUNCTION ViewOffset() = default; - KOKKOS_DEFAULTED_FUNCTION ViewOffset(const ViewOffset&) = default; - KOKKOS_DEFAULTED_FUNCTION ViewOffset& operator=(const ViewOffset&) = default; -#endif - - template - KOKKOS_INLINE_FUNCTION constexpr ViewOffset( - std::integral_constant const&, - array_layout const arg_layout) - : m_dim(arg_layout.dimension[0], arg_layout.dimension[1], - arg_layout.dimension[2], arg_layout.dimension[3], - arg_layout.dimension[4], arg_layout.dimension[5], - arg_layout.dimension[6], arg_layout.dimension[7]), - m_tile_N0((arg_layout.dimension[0] + MASK_0) >> - SHIFT_0 /* number of tiles in first dimension */), - m_tile_N1((arg_layout.dimension[1] + MASK_1) >> SHIFT_1), - m_tile_N2((VORank > 2) ? (arg_layout.dimension[2] + MASK_2) >> SHIFT_2 - : 0), - m_tile_N3((VORank > 3) ? (arg_layout.dimension[3] + MASK_3) >> SHIFT_3 - : 0), - m_tile_N4((VORank > 4) ? (arg_layout.dimension[4] + MASK_4) >> SHIFT_4 - : 0), - m_tile_N5((VORank > 5) ? (arg_layout.dimension[5] + MASK_5) >> SHIFT_5 - : 0), - m_tile_N6((VORank > 6) ? (arg_layout.dimension[6] + MASK_6) >> SHIFT_6 - : 0), - m_tile_N7((VORank > 7) ? (arg_layout.dimension[7] + MASK_7) >> SHIFT_7 - : 0) {} -}; - -// FIXME Remove the out-of-class definitions when we require C++17 -#define KOKKOS_ITERATE_VIEW_OFFSET_ENABLE \ - std::enable_if_t<((Dimension::rank <= 8) && (Dimension::rank >= 2) && \ - is_array_layout::value && \ - is_array_layout_tiled::value)> -template -constexpr Kokkos::Iterate ViewOffset< - Dimension, Layout, KOKKOS_ITERATE_VIEW_OFFSET_ENABLE>::outer_pattern; -template -constexpr Kokkos::Iterate ViewOffset< - Dimension, Layout, KOKKOS_ITERATE_VIEW_OFFSET_ENABLE>::inner_pattern; -template -constexpr int - ViewOffset::VORank; -template -constexpr unsigned - ViewOffset::SHIFT_0; -template -constexpr unsigned - ViewOffset::SHIFT_1; -template -constexpr unsigned - ViewOffset::SHIFT_2; -template -constexpr unsigned - ViewOffset::SHIFT_3; -template -constexpr unsigned - ViewOffset::SHIFT_4; -template -constexpr unsigned - ViewOffset::SHIFT_5; -template -constexpr unsigned - ViewOffset::SHIFT_6; -template -constexpr unsigned - ViewOffset::SHIFT_7; -template -constexpr int - ViewOffset::MASK_0; -template -constexpr int - ViewOffset::MASK_1; -template -constexpr int - ViewOffset::MASK_2; -template -constexpr int - ViewOffset::MASK_3; -template -constexpr int - ViewOffset::MASK_4; -template -constexpr int - ViewOffset::MASK_5; -template -constexpr int - ViewOffset::MASK_6; -template -constexpr int - ViewOffset::MASK_7; -template -constexpr unsigned - ViewOffset::SHIFT_2T; -template -constexpr unsigned - ViewOffset::SHIFT_3T; -template -constexpr unsigned - ViewOffset::SHIFT_4T; -template -constexpr unsigned - ViewOffset::SHIFT_5T; -template -constexpr unsigned - ViewOffset::SHIFT_6T; -template -constexpr unsigned - ViewOffset::SHIFT_7T; -template -constexpr unsigned - ViewOffset::SHIFT_8T; -#undef KOKKOS_ITERATE_VIEW_OFFSET_ENABLE - -//---------------------------------------- - -// ViewMapping assign method needed in order to return a 'subview' tile as a -// proper View The outer iteration pattern determines the mapping of the pointer -// offset to the beginning of requested tile The inner iteration pattern is -// needed for the layout of the tile's View to be returned Rank 2 -template -class ViewMapping // void - , - Kokkos::ViewTraits< - T**, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = Kokkos::ViewTraits; - using type = Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left ? ((i_tile0 + src.m_impl_offset.m_tile_N0 * i_tile1) - << src_offset_type::SHIFT_2T) - : ((src.m_impl_offset.m_tile_N1 * i_tile0 + i_tile1) - << src_offset_type::SHIFT_2T)) // offset to start - // of the tile - ), - dst_offset_type()); - } -}; - -// Rank 3 -template -class ViewMapping // void - , - Kokkos::ViewTraits< - T***, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1, iType2> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = Kokkos::ViewTraits; - using type = Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1, - const iType2 i_tile2) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left - ? ((i_tile0 + - src.m_impl_offset.m_tile_N0 * - (i_tile1 + src.m_impl_offset.m_tile_N1 * i_tile2)) - << src_offset_type::SHIFT_3T) - : ((src.m_impl_offset.m_tile_N2 * - (src.m_impl_offset.m_tile_N1 * i_tile0 + i_tile1) + - i_tile2) - << src_offset_type::SHIFT_3T))) // offset to start of the - // tile - , - dst_offset_type()); - } -}; - -// Rank 4 -template -class ViewMapping< - std::enable_if_t<(N4 == 0 && N5 == 0 && N6 == 0 && N7 == 0)> // void - , - Kokkos::ViewTraits< - T****, - Kokkos::Experimental::LayoutTiled, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1, iType2, iType3> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = Kokkos::ViewTraits; - using type = Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1, - const iType2 i_tile2, const iType3 i_tile3) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left - ? ((i_tile0 + - src.m_impl_offset.m_tile_N0 * - (i_tile1 + src.m_impl_offset.m_tile_N1 * - (i_tile2 + src.m_impl_offset.m_tile_N2 * - i_tile3))) - << src_offset_type::SHIFT_4T) - : ((src.m_impl_offset.m_tile_N3 * - (src.m_impl_offset.m_tile_N2 * - (src.m_impl_offset.m_tile_N1 * i_tile0 + - i_tile1) + - i_tile2) + - i_tile3) - << src_offset_type::SHIFT_4T))) // offset to start of the - // tile - , - dst_offset_type()); - } -}; - -// Rank 5 -template -class ViewMapping // void - , - Kokkos::ViewTraits< - T*****, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1, iType2, iType3, iType4> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = Kokkos::ViewTraits; - using type = Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1, - const iType2 i_tile2, const iType3 i_tile3, const iType4 i_tile4) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left - ? ((i_tile0 + - src.m_impl_offset.m_tile_N0 * - (i_tile1 + - src.m_impl_offset.m_tile_N1 * - (i_tile2 + - src.m_impl_offset.m_tile_N2 * - (i_tile3 + - src.m_impl_offset.m_tile_N3 * i_tile4)))) - << src_offset_type::SHIFT_5T) - : ((src.m_impl_offset.m_tile_N4 * - (src.m_impl_offset.m_tile_N3 * - (src.m_impl_offset.m_tile_N2 * - (src.m_impl_offset.m_tile_N1 * i_tile0 + - i_tile1) + - i_tile2) + - i_tile3) + - i_tile4) - << src_offset_type::SHIFT_5T))) // offset to start of the - // tile - , - dst_offset_type()); - } -}; - -// Rank 6 -template -class ViewMapping // void - , - Kokkos::ViewTraits< - T******, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1, iType2, iType3, iType4, iType5> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = - Kokkos::ViewTraits; - using type = Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1, - const iType2 i_tile2, const iType3 i_tile3, const iType4 i_tile4, - const iType5 i_tile5) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left - ? ((i_tile0 + - src.m_impl_offset.m_tile_N0 * - (i_tile1 + - src.m_impl_offset.m_tile_N1 * - (i_tile2 + - src.m_impl_offset.m_tile_N2 * - (i_tile3 + - src.m_impl_offset.m_tile_N3 * - (i_tile4 + src.m_impl_offset.m_tile_N4 * - i_tile5))))) - << src_offset_type::SHIFT_6T) - : ((src.m_impl_offset.m_tile_N5 * - (src.m_impl_offset.m_tile_N4 * - (src.m_impl_offset.m_tile_N3 * - (src.m_impl_offset.m_tile_N2 * - (src.m_impl_offset.m_tile_N1 * i_tile0 + - i_tile1) + - i_tile2) + - i_tile3) + - i_tile4) + - i_tile5) - << src_offset_type::SHIFT_6T))) // offset to start of the - // tile - , - dst_offset_type()); - } -}; - -// Rank 7 -template -class ViewMapping // void - , - Kokkos::ViewTraits< - T*******, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1, iType2, iType3, iType4, iType5, iType6> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = - Kokkos::ViewTraits; - using type = Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1, - const iType2 i_tile2, const iType3 i_tile3, const iType4 i_tile4, - const iType5 i_tile5, const iType6 i_tile6) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left - ? ((i_tile0 + - src.m_impl_offset.m_tile_N0 * - (i_tile1 + - src.m_impl_offset.m_tile_N1 * - (i_tile2 + - src.m_impl_offset.m_tile_N2 * - (i_tile3 + - src.m_impl_offset.m_tile_N3 * - (i_tile4 + - src.m_impl_offset.m_tile_N4 * - (i_tile5 + - src.m_impl_offset.m_tile_N5 * - i_tile6)))))) - << src_offset_type::SHIFT_7T) - : ((src.m_impl_offset.m_tile_N6 * - (src.m_impl_offset.m_tile_N5 * - (src.m_impl_offset.m_tile_N4 * - (src.m_impl_offset.m_tile_N3 * - (src.m_impl_offset.m_tile_N2 * - (src.m_impl_offset.m_tile_N1 * - i_tile0 + - i_tile1) + - i_tile2) + - i_tile3) + - i_tile4) + - i_tile5) + - i_tile6) - << src_offset_type::SHIFT_7T))) // offset to start of the - // tile - , - dst_offset_type()); - } -}; - -// Rank 8 -template -class ViewMapping< - std::enable_if_t<(N0 != 0 && N1 != 0 && N2 != 0 && N3 != 0 && N4 != 0 && - N5 != 0 && N6 != 0 && N7 != 0)> // void - , - Kokkos::ViewTraits< - T********, - Kokkos::Experimental::LayoutTiled, - P...>, - Kokkos::Experimental::LayoutTiled, - iType0, iType1, iType2, iType3, iType4, iType5, iType6, iType7> { - public: - using src_layout = - Kokkos::Experimental::LayoutTiled; - using src_traits = Kokkos::ViewTraits; - - static constexpr bool is_outer_left = (OuterP == Kokkos::Iterate::Left); - static constexpr bool is_inner_left = (InnerP == Kokkos::Iterate::Left); - using array_layout = std::conditional_t; - using traits = - Kokkos::ViewTraits; - using type = - Kokkos::View; - - KOKKOS_INLINE_FUNCTION static void assign( - ViewMapping& dst, const ViewMapping& src, - const src_layout&, const iType0 i_tile0, const iType1 i_tile1, - const iType2 i_tile2, const iType3 i_tile3, const iType4 i_tile4, - const iType5 i_tile5, const iType6 i_tile6, const iType7 i_tile7) { - using dst_map_type = ViewMapping; - using src_map_type = ViewMapping; - using dst_handle_type = typename dst_map_type::handle_type; - using dst_offset_type = typename dst_map_type::offset_type; - using src_offset_type = typename src_map_type::offset_type; - - dst = dst_map_type( - dst_handle_type( - src.m_impl_handle + - (is_outer_left - ? ((i_tile0 + - src.m_impl_offset.m_tile_N0 * - (i_tile1 + - src.m_impl_offset.m_tile_N1 * - (i_tile2 + - src.m_impl_offset.m_tile_N2 * - (i_tile3 + - src.m_impl_offset.m_tile_N3 * - (i_tile4 + - src.m_impl_offset.m_tile_N4 * - (i_tile5 + - src.m_impl_offset.m_tile_N5 * - (i_tile6 + - src.m_impl_offset.m_tile_N6 * - i_tile7))))))) - << src_offset_type::SHIFT_8T) - : ((src.m_impl_offset.m_tile_N7 * - (src.m_impl_offset.m_tile_N6 * - (src.m_impl_offset.m_tile_N5 * - (src.m_impl_offset.m_tile_N4 * - (src.m_impl_offset.m_tile_N3 * - (src.m_impl_offset.m_tile_N2 * - (src.m_impl_offset.m_tile_N1 * - i_tile0 + - i_tile1) + - i_tile2) + - i_tile3) + - i_tile4) + - i_tile5) + - i_tile6) + - i_tile7) - << src_offset_type::SHIFT_8T))) // offset to start of the - // tile - , - dst_offset_type()); - } -}; - -} /* namespace Impl */ -} /* namespace Kokkos */ - -//---------------------------------------- - -namespace Kokkos { - -// Rank 2 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T**, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View(src, SrcLayout(), i_tile0, - i_tile1); -} - -// Rank 3 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T***, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1, - const size_t i_tile2) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View( - src, SrcLayout(), i_tile0, i_tile1, i_tile2); -} - -// Rank 4 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T****, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1, - const size_t i_tile2, const size_t i_tile3) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View( - src, SrcLayout(), i_tile0, i_tile1, i_tile2, i_tile3); -} - -// Rank 5 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T*****, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1, - const size_t i_tile2, const size_t i_tile3, - const size_t i_tile4) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View( - src, SrcLayout(), i_tile0, i_tile1, i_tile2, i_tile3, i_tile4); -} - -// Rank 6 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T******, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1, - const size_t i_tile2, const size_t i_tile3, - const size_t i_tile4, const size_t i_tile5) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View( - src, SrcLayout(), i_tile0, i_tile1, i_tile2, i_tile3, i_tile4, i_tile5); -} - -// Rank 7 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T*******, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1, - const size_t i_tile2, const size_t i_tile3, - const size_t i_tile4, const size_t i_tile5, - const size_t i_tile6) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View( - src, SrcLayout(), i_tile0, i_tile1, i_tile2, i_tile3, i_tile4, i_tile5, - i_tile6); -} - -// Rank 8 -template -KOKKOS_INLINE_FUNCTION - Kokkos::View, - P...> - tile_subview(const Kokkos::View< - T********, - Kokkos::Experimental::LayoutTiled< - OuterP, InnerP, N0, N1, N2, N3, N4, N5, N6, N7, true>, - P...>& src, - const size_t i_tile0, const size_t i_tile1, - const size_t i_tile2, const size_t i_tile3, - const size_t i_tile4, const size_t i_tile5, - const size_t i_tile6, const size_t i_tile7) { - // Force the specialized ViewMapping for extracting a tile - // by using the first subview argument as the layout. - using array_layout = - std::conditional_t<(InnerP == Kokkos::Iterate::Left), Kokkos::LayoutLeft, - Kokkos::LayoutRight>; - using SrcLayout = - Kokkos::Experimental::LayoutTiled; - - return Kokkos::View( - src, SrcLayout(), i_tile0, i_tile1, i_tile2, i_tile3, i_tile4, i_tile5, - i_tile6, i_tile7); -} - -} /* namespace Kokkos */ -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #ifndef KOKKOS_EXPERIENTAL_VIEWLAYOUTTILE_HPP */ diff --git a/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_Cuda.hpp b/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_Cuda.hpp deleted file mode 100644 index 983a71a561..0000000000 --- a/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_Cuda.hpp +++ /dev/null @@ -1,133 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_CUDA_SETUP_HPP_ -#define KOKKOS_CUDA_SETUP_HPP_ - -#if !defined(KOKKOS_ENABLE_CUDA) -#error \ - "KOKKOS_ENABLE_CUDA was not defined, but Kokkos_Setup_Cuda.hpp was included anyway." -#endif - -#if defined(KOKKOS_ENABLE_CUDA) && !defined(__CUDACC__) -#error \ - "KOKKOS_ENABLE_CUDA defined but the compiler is not defining the __CUDACC__ macro as expected" -// Some tooling environments will still function better if we do this here. -#define __CUDACC__ -#endif /* defined(KOKKOS_ENABLE_CUDA) && !defined(__CUDACC__) */ - -// Compiling with a CUDA compiler. -// -// Include to pick up the CUDA_VERSION macro defined as: -// CUDA_VERSION = ( MAJOR_VERSION * 1000 ) + ( MINOR_VERSION * 10 ) -// -// When generating device code the __CUDA_ARCH__ macro is defined as: -// __CUDA_ARCH__ = ( MAJOR_CAPABILITY * 100 ) + ( MINOR_CAPABILITY * 10 ) - -#include -#include - -#if defined(_WIN32) -#define KOKKOS_IMPL_WINDOWS_CUDA -#endif - -#if !defined(CUDA_VERSION) -#error "#include did not define CUDA_VERSION." -#endif - -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 300) -// Compiling with CUDA compiler for device code. -#error "Cuda device capability >= 3.0 is required." -#endif - -#ifdef KOKKOS_ENABLE_CUDA_LAMBDA -#define KOKKOS_LAMBDA [=] __host__ __device__ - -#if defined(KOKKOS_ENABLE_CXX17) || defined(KOKKOS_ENABLE_CXX20) -#define KOKKOS_CLASS_LAMBDA [ =, *this ] __host__ __device__ -#endif - -#else // !defined(KOKKOS_ENABLE_CUDA_LAMBDA) -#undef KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA -#endif // !defined(KOKKOS_ENABLE_CUDA_LAMBDA) - -#if (10000 > CUDA_VERSION) -#define KOKKOS_ENABLE_PRE_CUDA_10_DEPRECATION_API -#endif - -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700) -// PTX atomics with memory order semantics are only available on volta and later -#if !defined(KOKKOS_DISABLE_CUDA_ASM) -#if !defined(KOKKOS_ENABLE_CUDA_ASM) -#define KOKKOS_ENABLE_CUDA_ASM -#if !defined(KOKKOS_DISABLE_CUDA_ASM_ATOMICS) && \ - defined(KOKKOS_ENABLE_GNU_ATOMICS) -#define KOKKOS_ENABLE_CUDA_ASM_ATOMICS -#endif -#endif -#endif -#endif - -#define KOKKOS_IMPL_FORCEINLINE_FUNCTION __device__ __host__ __forceinline__ -#define KOKKOS_IMPL_FORCEINLINE __forceinline__ -#define KOKKOS_IMPL_INLINE_FUNCTION __device__ __host__ inline -#define KOKKOS_IMPL_FUNCTION __device__ __host__ -#define KOKKOS_IMPL_HOST_FUNCTION __host__ -#define KOKKOS_IMPL_DEVICE_FUNCTION __device__ -#if defined(KOKKOS_COMPILER_NVCC) -#define KOKKOS_INLINE_FUNCTION_DELETED inline -#else -#define KOKKOS_INLINE_FUNCTION_DELETED __device__ __host__ inline -#endif -#if (CUDA_VERSION < 10000) -#define KOKKOS_DEFAULTED_FUNCTION __host__ __device__ inline -#else -#define KOKKOS_DEFAULTED_FUNCTION inline -#endif - -#if (CUDA_VERSION >= 10000) -#define KOKKOS_CUDA_ENABLE_GRAPHS -#endif - -#endif /* KOKKOS_CUDA_SETUP_HPP_ */ diff --git a/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_HIP.hpp b/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_HIP.hpp deleted file mode 100644 index f1df2f87bb..0000000000 --- a/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_HIP.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SETUP_HIP_HPP_ -#define KOKKOS_SETUP_HIP_HPP_ - -#if defined(KOKKOS_ENABLE_HIP) - -#define KOKKOS_IMPL_HIP_CLANG_WORKAROUND - -#define HIP_ENABLE_PRINTF -#include -#include - -#define KOKKOS_LAMBDA [=] __host__ __device__ -#if defined(KOKKOS_ENABLE_CXX17) || defined(KOKKOS_ENABLE_CXX20) -#define KOKKOS_CLASS_LAMBDA [ =, *this ] __host__ __device__ -#endif - -#define KOKKOS_IMPL_FORCEINLINE_FUNCTION __device__ __host__ __forceinline__ -#define KOKKOS_IMPL_INLINE_FUNCTION __device__ __host__ inline -#define KOKKOS_DEFAULTED_FUNCTION __device__ __host__ inline -#define KOKKOS_INLINE_FUNCTION_DELETED __device__ __host__ inline -#define KOKKOS_IMPL_FUNCTION __device__ __host__ -#define KOKKOS_IMPL_HOST_FUNCTION __host__ -#define KOKKOS_IMPL_DEVICE_FUNCTION __device__ - -#endif // #if defined( KOKKOS_ENABLE_HIP ) - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_SYCL.hpp b/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_SYCL.hpp deleted file mode 100644 index b203e9afb2..0000000000 --- a/bundled/kokkos-3.7.00/core/src/setup/Kokkos_Setup_SYCL.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SETUP_SYCL_HPP_ -#define KOKKOS_SETUP_SYCL_HPP_ - -// FIXME_SYCL the fallback assert is temporarily disabled by default in the -// compiler so we need to force it -#ifndef SYCL_ENABLE_FALLBACK_ASSERT -#define SYCL_ENABLE_FALLBACK_ASSERT -#endif -#ifndef SYCL_FALLBACK_ASSERT -#define SYCL_FALLBACK_ASSERT 1 -#endif - -#include - -#ifdef __SYCL_DEVICE_ONLY__ -#define KOKKOS_IMPL_DO_NOT_USE_PRINTF(format, ...) \ - do { \ - const __attribute__((opencl_constant)) char fmt[] = (format); \ - sycl::ext::oneapi::experimental::printf(fmt, ##__VA_ARGS__); \ - } while (0) -#endif - -#endif diff --git a/bundled/kokkos-3.7.00/core/src/traits/Kokkos_GraphKernelTrait.hpp b/bundled/kokkos-3.7.00/core/src/traits/Kokkos_GraphKernelTrait.hpp deleted file mode 100644 index b16a7777cd..0000000000 --- a/bundled/kokkos-3.7.00/core/src/traits/Kokkos_GraphKernelTrait.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_KOKKOS_GRAPHKERNELTRAIT_HPP -#define KOKKOS_KOKKOS_GRAPHKERNELTRAIT_HPP - -#include -#include -#include // IsGraphKernelTag -#include -#include - -namespace Kokkos { -namespace Impl { - -//============================================================================== -// {{{1 - -struct GraphKernelTrait : TraitSpecificationBase { - struct base_traits { - using is_graph_kernel = std::false_type; - KOKKOS_IMPL_MSVC_NVCC_EBO_WORKAROUND - }; - template - struct mixin_matching_trait : AnalyzeNextTrait { - using base_t = AnalyzeNextTrait; - using base_t::base_t; - using is_graph_kernel = std::true_type; - }; - template - using trait_matches_specification = std::is_same; -}; - -// end trait specification }}}1 -//============================================================================== - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_KOKKOS_GRAPHKERNELTRAIT_HPP diff --git a/bundled/kokkos-3.7.00/core/src/traits/Kokkos_PolicyTraitMatcher.hpp b/bundled/kokkos-3.7.00/core/src/traits/Kokkos_PolicyTraitMatcher.hpp deleted file mode 100644 index 31927320bf..0000000000 --- a/bundled/kokkos-3.7.00/core/src/traits/Kokkos_PolicyTraitMatcher.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#include // type_list - -#include - -#ifndef KOKKOS_KOKKOS_POLICYTRAITMATCHER_HPP -#define KOKKOS_KOKKOS_POLICYTRAITMATCHER_HPP - -namespace Kokkos { -namespace Impl { - -//============================================================================== -// {{{1 - -// To handle the WorkTag case, we need more than just a predicate; we need -// something that we can default to in the unspecialized case, just like we -// do for AnalyzeExecPolicy -template -struct PolicyTraitMatcher : std::false_type {}; - -template -struct PolicyTraitMatcher< - TraitSpec, Trait, - std::enable_if_t< - TraitSpec::template trait_matches_specification::value>> - : std::true_type {}; - -// end PolicyTraitMatcher }}}1 -//============================================================================== - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_KOKKOS_POLICYTRAITMATCHER_HPP diff --git a/bundled/kokkos-3.7.00/simd/cmake/Dependencies.cmake b/bundled/kokkos-3.7.00/simd/cmake/Dependencies.cmake deleted file mode 100644 index 5e29157369..0000000000 --- a/bundled/kokkos-3.7.00/simd/cmake/Dependencies.cmake +++ /dev/null @@ -1,5 +0,0 @@ -TRIBITS_PACKAGE_DEFINE_DEPENDENCIES( - LIB_REQUIRED_PACKAGES KokkosCore - LIB_OPTIONAL_TPLS Pthread CUDA HWLOC HPX - TEST_OPTIONAL_TPLS CUSPARSE - ) diff --git a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD.hpp b/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD.hpp deleted file mode 100644 index a15a1b8ebf..0000000000 --- a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD.hpp +++ /dev/null @@ -1,161 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SIMD_HPP -#define KOKKOS_SIMD_HPP - -#include - -#include - -#ifdef KOKKOS_ARCH_AVX512XEON -#include -#endif - -namespace Kokkos { -namespace Experimental { - -namespace simd_abi { - -namespace Impl { - -#if defined(KOKKOS_ARCH_AVX512XEON) -using host_native = avx512_fixed_size<8>; -#else -using host_native = scalar; -#endif - -template -struct ForSpace; - -#ifdef KOKKOS_ENABLE_SERIAL -template <> -struct ForSpace { - using type = host_native; -}; -#endif - -#ifdef KOKKOS_ENABLE_CUDA -template <> -struct ForSpace { - using type = scalar; -}; -#endif - -#ifdef KOKKOS_ENABLE_THREADS -template <> -struct ForSpace { - using type = host_native; -}; -#endif - -#ifdef KOKKOS_ENABLE_HPX -template <> -struct ForSpace { - using type = scalar; -}; -#endif - -#ifdef KOKKOS_ENABLE_OPENMP -template <> -struct ForSpace { - using type = host_native; -}; -#endif - -#ifdef KOKKOS_ENABLE_OPENMPTARGET -template <> -struct ForSpace { - using type = scalar; -}; -#endif - -#ifdef KOKKOS_ENABLE_HIP -template <> -struct ForSpace { - using type = scalar; -}; -#endif - -#ifdef KOKKOS_ENABLE_SYCL -template <> -struct ForSpace { - using type = scalar; -}; -#endif - -} // namespace Impl - -template -using ForSpace = typename Impl::ForSpace::type; - -template -using native = ForSpace; - -} // namespace simd_abi - -template -using native_simd = simd>; -template -using native_simd_mask = simd_mask>; - -namespace Impl { - -template -class abi_set {}; - -#ifdef KOKKOS_ARCH_AVX512XEON -using host_abi_set = abi_set>; -#else -using host_abi_set = abi_set; -#endif - -using device_abi_set = abi_set; - -} // namespace Impl - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_AVX512.hpp b/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_AVX512.hpp deleted file mode 100644 index 1df0730ac4..0000000000 --- a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_AVX512.hpp +++ /dev/null @@ -1,1023 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SIMD_AVX512_HPP -#define KOKKOS_SIMD_AVX512_HPP - -#include -#include - -#include - -#include - -namespace Kokkos { -namespace Experimental { - -namespace simd_abi { - -template -class avx512_fixed_size {}; - -} // namespace simd_abi - -template -class simd_mask> { - __mmask8 m_value; - - public: - class reference { - __mmask8& m_mask; - int m_lane; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION __mmask8 bit_mask() const { - return __mmask8(std::int16_t(1 << m_lane)); - } - - public: - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference(__mmask8& mask_arg, - int lane_arg) - : m_mask(mask_arg), m_lane(lane_arg) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference - operator=(bool value) const { - if (value) { - m_mask |= bit_mask(); - } else { - m_mask &= ~bit_mask(); - } - return *this; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION operator bool() const { - return (m_mask & bit_mask()) != 0; - } - }; - using value_type = bool; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd_mask() = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd_mask(value_type value) - : m_value(-std::int16_t(value)) {} - template - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd_mask( - simd_mask> const& other) - : m_value(static_cast<__mmask8>(other)) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION static constexpr std::size_t size() { - return 8; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit simd_mask( - __mmask8 const& value_in) - : m_value(value_in) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit operator __mmask8() - const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference operator[](std::size_t i) { - return reference(m_value, int(i)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION value_type - operator[](std::size_t i) const { - return static_cast(reference(m_value, int(i))); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd_mask - operator||(simd_mask const& other) const { - return simd_mask(_kor_mask8(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd_mask - operator&&(simd_mask const& other) const { - return simd_mask(_kand_mask8(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd_mask operator!() const { - static const __mmask8 true_value(static_cast<__mmask8>(simd_mask(true))); - return simd_mask(_kxor_mask8(true_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION bool operator==( - simd_mask const& other) const { - return m_value == other.m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION bool operator!=( - simd_mask const& other) const { - return m_value != other.m_value; - } -}; - -template <> -class simd> { - __m256i m_value; - - public: - using value_type = std::int32_t; - using abi_type = simd_abi::avx512_fixed_size<8>; - using mask_type = simd_mask; - using reference = value_type&; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd() = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION static constexpr std::size_t size() { - return 8; - } - template , - bool> = false> - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(U&& value) - : m_value(_mm256_set1_epi32(value_type(value))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit simd( - __m256i const& value_in) - : m_value(value_in) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd( - simd const& other); - template ()); } - std::is_invocable_r_v>, - bool> = false> - KOKKOS_FORCEINLINE_FUNCTION simd(G&& gen) - : m_value( - _mm256_setr_epi32(gen(std::integral_constant()), - gen(std::integral_constant()), - gen(std::integral_constant()), - gen(std::integral_constant()), - gen(std::integral_constant()), - gen(std::integral_constant()), - gen(std::integral_constant()), - gen(std::integral_constant()))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference operator[](std::size_t i) { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION value_type - operator[](std::size_t i) const { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION void copy_to( - value_type* ptr, element_aligned_tag) const { - _mm256_mask_storeu_epi32(ptr, static_cast<__mmask8>(mask_type(true)), - m_value); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION void copy_from(value_type const* ptr, - element_aligned_tag) { - m_value = _mm256_mask_loadu_epi32( - _mm256_set1_epi32(0), static_cast<__mmask8>(mask_type(true)), ptr); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit operator __m256i() - const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<(simd const& other) const { - return mask_type(_mm256_cmplt_epi32_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>(simd const& other) const { - return mask_type(_mm256_cmplt_epi32_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<=(simd const& other) const { - return mask_type(_mm256_cmple_epi32_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>=(simd const& other) const { - return mask_type(_mm256_cmple_epi32_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator==(simd const& other) const { - return mask_type(_mm256_cmpeq_epi32_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator!=(simd const& other) const { - return mask_type(_mm256_cmpneq_epi32_mask(m_value, other.m_value)); - } -}; - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator*(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm256_mullo_epi32(static_cast<__m256i>(lhs), static_cast<__m256i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator+(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm256_add_epi32(static_cast<__m256i>(lhs), static_cast<__m256i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm256_sub_epi32(static_cast<__m256i>(lhs), static_cast<__m256i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& a) { - return simd>(0) - a; -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> condition( - simd_mask> const& a, - simd> const& b, - simd> const& c) { - return simd>( - _mm256_mask_blend_epi32(static_cast<__mmask8>(a), static_cast<__m256i>(c), - static_cast<__m256i>(b))); -} - -template <> -class simd> { - __m256i m_value; - - public: - using value_type = std::uint32_t; - using abi_type = simd_abi::avx512_fixed_size<8>; - using mask_type = simd_mask; - using reference = value_type&; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd() = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION static constexpr std::size_t size() { - return 8; - } - template , - bool> = false> - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(U&& value) - : m_value(_mm256_set1_epi32(bit_cast(value_type(value)))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit simd( - __m256i const& value_in) - : m_value(value_in) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd( - simd> const& other) - : m_value(static_cast<__m256i>(other)) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference operator[](std::size_t i) { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION value_type - operator[](std::size_t i) const { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit operator __m256i() - const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<(simd const& other) const { - return mask_type(_mm256_cmplt_epu32_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>(simd const& other) const { - return mask_type(_mm256_cmplt_epu32_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<=(simd const& other) const { - return mask_type(_mm256_cmple_epu32_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>=(simd const& other) const { - return mask_type(_mm256_cmple_epu32_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator==(simd const& other) const { - return mask_type(_mm256_cmpeq_epu32_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator!=(simd const& other) const { - return mask_type(_mm256_cmpneq_epu32_mask(m_value, other.m_value)); - } -}; - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator*(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm256_mullo_epi32(static_cast<__m256i>(lhs), static_cast<__m256i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator+(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm256_add_epi32(static_cast<__m256i>(lhs), static_cast<__m256i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm256_sub_epi32(static_cast<__m256i>(lhs), static_cast<__m256i>(rhs))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> condition( - simd_mask> const& a, - simd> const& b, - simd> const& c) { - return simd>( - _mm256_mask_blend_epi32(static_cast<__mmask8>(a), static_cast<__m256i>(c), - static_cast<__m256i>(b))); -} - -template <> -class simd> { - __m512i m_value; - - public: - using value_type = std::int64_t; - using abi_type = simd_abi::avx512_fixed_size<8>; - using mask_type = simd_mask; - using reference = value_type&; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd() = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION static constexpr std::size_t size() { - return 8; - } - template , - bool> = false> - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(U&& value) - : m_value(_mm512_set1_epi64(value_type(value))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd( - simd> const& other) - : m_value(_mm512_cvtepi32_epi64(static_cast<__m256i>(other))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd( - simd> const& other); - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr simd(__m512i const& value_in) - : m_value(value_in) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference operator[](std::size_t i) { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION value_type - operator[](std::size_t i) const { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION void copy_to( - value_type* ptr, element_aligned_tag) const { - _mm512_mask_storeu_epi64(ptr, static_cast<__mmask8>(mask_type(true)), - m_value); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd operator>>(int rhs) const { - return _mm512_srai_epi64(m_value, rhs); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd - operator>>(simd> const& rhs) const { - return _mm512_srav_epi64(m_value, - _mm512_cvtepi32_epi64(static_cast<__m256i>(rhs))); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd operator<<(int rhs) const { - return _mm512_slli_epi64(m_value, rhs); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd - operator<<(simd> const& rhs) const { - return _mm512_sllv_epi64(m_value, - _mm512_cvtepi32_epi64(static_cast<__m256i>(rhs))); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit operator __m512i() - const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<(simd const& other) const { - return mask_type(_mm512_cmplt_epi64_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>(simd const& other) const { - return mask_type(_mm512_cmplt_epi64_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<=(simd const& other) const { - return mask_type(_mm512_cmple_epi64_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>=(simd const& other) const { - return mask_type(_mm512_cmple_epi64_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator==(simd const& other) const { - return mask_type(_mm512_cmpeq_epi64_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator!=(simd const& other) const { - return mask_type(_mm512_cmpneq_epi64_mask(m_value, other.m_value)); - } -}; - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator*(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_mullo_epi64(static_cast<__m512i>(lhs), static_cast<__m512i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator+(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_add_epi64(static_cast<__m512i>(lhs), static_cast<__m512i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_sub_epi64(static_cast<__m512i>(lhs), static_cast<__m512i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& a) { - return simd>(0) - a; -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> condition( - simd_mask> const& a, - simd> const& b, - simd> const& c) { - return simd>( - _mm512_mask_blend_epi64(static_cast<__mmask8>(a), static_cast<__m512i>(c), - static_cast<__m512i>(b))); -} - -template <> -class simd> { - __m512i m_value; - - public: - using value_type = std::uint64_t; - using abi_type = simd_abi::avx512_fixed_size<8>; - using mask_type = simd_mask; - using reference = value_type&; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd() = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION static constexpr std::size_t size() { - return 8; - } - template , - bool> = false> - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(U&& value) - : m_value(_mm512_set1_epi64(bit_cast(value_type(value)))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr simd(__m512i const& value_in) - : m_value(value_in) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd( - simd const& other) - : m_value(_mm512_cvtepi32_epi64(static_cast<__m256i>(other))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION explicit simd( - simd const& other) - : m_value(static_cast<__m512i>(other)) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference operator[](std::size_t i) { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION value_type - operator[](std::size_t i) const { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd - operator>>(unsigned int rhs) const { - return _mm512_srli_epi64(m_value, rhs); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd operator>>( - simd> const& rhs) const { - return _mm512_srlv_epi64(m_value, - _mm512_cvtepi32_epi64(static_cast<__m256i>(rhs))); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd - operator<<(unsigned int rhs) const { - return _mm512_slli_epi64(m_value, rhs); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd operator<<( - simd> const& rhs) const { - return _mm512_sllv_epi64(m_value, - _mm512_cvtepi32_epi64(static_cast<__m256i>(rhs))); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd - operator&(simd const& other) const { - return _mm512_and_epi64(m_value, other.m_value); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd - operator|(simd const& other) const { - return _mm512_or_epi64(m_value, other.m_value); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit operator __m512i() - const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<(simd const& other) const { - return mask_type(_mm512_cmplt_epu64_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>(simd const& other) const { - return mask_type(_mm512_cmplt_epu64_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<=(simd const& other) const { - return mask_type(_mm512_cmple_epu64_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>=(simd const& other) const { - return mask_type(_mm512_cmple_epu64_mask(other.m_value, m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator==(simd const& other) const { - return mask_type(_mm512_cmpeq_epu64_mask(m_value, other.m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator!=(simd const& other) const { - return mask_type(_mm512_cmpneq_epu64_mask(m_value, other.m_value)); - } -}; - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator*(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_mullo_epi64(static_cast<__m512i>(lhs), static_cast<__m512i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator+(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_add_epi64(static_cast<__m512i>(lhs), static_cast<__m512i>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_sub_epi64(static_cast<__m512i>(lhs), static_cast<__m512i>(rhs))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> condition( - simd_mask> const& a, - simd> const& b, - simd> const& c) { - return simd>( - _mm512_mask_blend_epi64(static_cast<__mmask8>(a), static_cast<__m512i>(c), - static_cast<__m512i>(b))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd>::simd( - simd> const& other) - : m_value(_mm512_cvtepi64_epi32(static_cast<__m512i>(other))) {} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd>::simd( - simd> const& other) - : m_value(static_cast<__m512i>(other)) {} - -template <> -class simd> { - __m512d m_value; - - public: - using value_type = double; - using abi_type = simd_abi::avx512_fixed_size<8>; - using mask_type = simd_mask; - using reference = value_type&; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd() = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd const&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd& operator=(simd&&) = default; - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION static constexpr std::size_t size() { - return 8; - } - template , - bool> = false> - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(U&& value) - : m_value(_mm512_set1_pd(value_type(value))) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION simd(double a, double b, double c, - double d, double e, double f, - double g, double h) - : m_value(_mm512_setr_pd(a, b, c, d, e, f, g, h)) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit simd( - __m512d const& value_in) - : m_value(value_in) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION reference operator[](std::size_t i) { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION value_type - operator[](std::size_t i) const { - return reinterpret_cast(&m_value)[i]; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION void copy_from(value_type const* ptr, - element_aligned_tag) { - m_value = _mm512_loadu_pd(ptr); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION void copy_to( - value_type* ptr, element_aligned_tag) const { - _mm512_storeu_pd(ptr, m_value); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr explicit operator __m512d() - const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<(simd const& other) const { - return mask_type(_mm512_cmp_pd_mask(m_value, other.m_value, _CMP_LT_OS)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>(simd const& other) const { - return mask_type(_mm512_cmp_pd_mask(m_value, other.m_value, _CMP_GT_OS)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator<=(simd const& other) const { - return mask_type(_mm512_cmp_pd_mask(m_value, other.m_value, _CMP_LE_OS)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator>=(simd const& other) const { - return mask_type(_mm512_cmp_pd_mask(m_value, other.m_value, _CMP_GE_OS)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator==(simd const& other) const { - return mask_type(_mm512_cmp_pd_mask(m_value, other.m_value, _CMP_EQ_OS)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION mask_type - operator!=(simd const& other) const { - return mask_type(_mm512_cmp_pd_mask(m_value, other.m_value, _CMP_NEQ_OS)); - } -}; - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator*(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_mul_pd(static_cast<__m512d>(lhs), static_cast<__m512d>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator/(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_div_pd(static_cast<__m512d>(lhs), static_cast<__m512d>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator+(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_add_pd(static_cast<__m512d>(lhs), static_cast<__m512d>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& lhs, - simd> const& rhs) { - return simd>( - _mm512_sub_pd(static_cast<__m512d>(lhs), static_cast<__m512d>(rhs))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - simd> - operator-(simd> const& a) { - return simd>( - _mm512_sub_pd(_mm512_set1_pd(0.0), static_cast<__m512d>(a))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> copysign( - simd> const& a, - simd> const& b) { - static const __m512i sign_mask = reinterpret_cast<__m512i>( - static_cast<__m512d>(simd>(-0.0))); - return simd>( - reinterpret_cast<__m512d>(_mm512_xor_epi64( - _mm512_andnot_epi64( - sign_mask, reinterpret_cast<__m512i>(static_cast<__m512d>(a))), - _mm512_and_epi64( - sign_mask, reinterpret_cast<__m512i>(static_cast<__m512d>(b)))))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> abs( - simd> const& a) { - __m512d const rhs = static_cast<__m512d>(a); - return simd>(reinterpret_cast<__m512d>( - _mm512_and_epi64(_mm512_set1_epi64(0x7FFFFFFFFFFFFFFF), - reinterpret_cast<__m512i>(rhs)))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> sqrt( - simd> const& a) { - return simd>( - _mm512_sqrt_pd(static_cast<__m512d>(a))); -} - -#ifdef __INTEL_COMPILER - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> cbrt( - simd> const& a) { - return simd>( - _mm512_cbrt_pd(static_cast<__m512d>(a))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> exp( - simd> const& a) { - return simd>( - _mm512_exp_pd(static_cast<__m512d>(a))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> log( - simd> const& a) { - return simd>( - _mm512_log_pd(static_cast<__m512d>(a))); -} - -#endif - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> fma( - simd> const& a, - simd> const& b, - simd> const& c) { - return simd>( - _mm512_fmadd_pd(static_cast<__m512d>(a), static_cast<__m512d>(b), - static_cast<__m512d>(c))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> max( - simd> const& a, - simd> const& b) { - return simd>( - _mm512_max_pd(static_cast<__m512d>(a), static_cast<__m512d>(b))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> min( - simd> const& a, - simd> const& b) { - return simd>( - _mm512_min_pd(static_cast<__m512d>(a), static_cast<__m512d>(b))); -} - -KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION -simd> condition( - simd_mask> const& a, - simd> const& b, - simd> const& c) { - return simd>( - _mm512_mask_blend_pd(static_cast<__mmask8>(a), static_cast<__m512d>(c), - static_cast<__m512d>(b))); -} - -template <> -class const_where_expression>, - simd>> { - public: - using abi_type = simd_abi::avx512_fixed_size<8>; - using value_type = simd; - using mask_type = simd_mask; - - protected: - value_type& m_value; - mask_type const& m_mask; - - public: - const_where_expression(mask_type const& mask_arg, value_type const& value_arg) - : m_value(const_cast(value_arg)), m_mask(mask_arg) {} - [[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr mask_type const& - mask() const { - return m_mask; - } - [[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr value_type const& - value() const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - void copy_to(double* mem, element_aligned_tag) const { - _mm512_mask_storeu_pd(mem, static_cast<__mmask8>(m_mask), - static_cast<__m512d>(m_value)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - void scatter_to( - double* mem, - simd> const& index) const { - _mm512_mask_i32scatter_pd(mem, static_cast<__mmask8>(m_mask), - static_cast<__m256i>(index), - static_cast<__m512d>(m_value), 8); - } -}; - -template <> -class where_expression>, - simd>> - : public const_where_expression< - simd_mask>, - simd>> { - public: - where_expression( - simd_mask> const& mask_arg, - simd>& value_arg) - : const_where_expression(mask_arg, value_arg) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - void copy_from(double const* mem, element_aligned_tag) { - m_value = value_type(_mm512_mask_loadu_pd( - _mm512_set1_pd(0.0), static_cast<__mmask8>(m_mask), mem)); - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - void gather_from( - double const* mem, - simd> const& index) { - m_value = value_type(_mm512_mask_i32gather_pd( - _mm512_set1_pd(0.0), static_cast<__mmask8>(m_mask), - static_cast<__m256i>(index), mem, 8)); - } - template >>, - bool> = false> - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION void operator=(U&& x) { - auto const x_as_value_type = - static_cast>>( - std::forward(x)); - m_value = simd>(_mm512_mask_blend_pd( - static_cast<__mmask8>(m_mask), static_cast<__m512d>(m_value), - static_cast<__m512d>(x_as_value_type))); - } -}; - -template <> -class const_where_expression< - simd_mask>, - simd>> { - public: - using abi_type = simd_abi::avx512_fixed_size<8>; - using value_type = simd; - using mask_type = simd_mask; - - protected: - value_type& m_value; - mask_type const& m_mask; - - public: - const_where_expression(mask_type const& mask_arg, value_type const& value_arg) - : m_value(const_cast(value_arg)), m_mask(mask_arg) {} - [[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr mask_type const& - mask() const { - return m_mask; - } - [[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr value_type const& - value() const { - return m_value; - } - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - void copy_to(std::int32_t* mem, element_aligned_tag) const { - _mm256_mask_storeu_epi32(mem, static_cast<__mmask8>(m_mask), - static_cast<__m256i>(m_value)); - } -}; - -template <> -class where_expression>, - simd>> - : public const_where_expression< - simd_mask>, - simd>> { - public: - where_expression( - simd_mask> const& mask_arg, - simd>& value_arg) - : const_where_expression(mask_arg, value_arg) {} - KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION - void copy_from(std::int32_t const* mem, element_aligned_tag) { - m_value = value_type(_mm256_mask_loadu_epi32( - _mm256_set1_epi32(0), static_cast<__mmask8>(m_mask), mem)); - } -}; - -template <> -class const_where_expression< - simd_mask>, - simd>> { - public: - using abi_type = simd_abi::avx512_fixed_size<8>; - using value_type = simd; - using mask_type = simd_mask; - - protected: - value_type& m_value; - mask_type const& m_mask; - - public: - const_where_expression(mask_type const& mask_arg, value_type const& value_arg) - : m_value(const_cast(value_arg)), m_mask(mask_arg) {} - [[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr mask_type const& - mask() const { - return m_mask; - } - [[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION constexpr value_type const& - value() const { - return m_value; - } -}; - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION std::int32_t hmax( - const_where_expression< - simd_mask>, - simd>> const& x) { - return _mm512_mask_reduce_max_epi32( - static_cast<__mmask8>(x.mask()), - _mm512_castsi256_si512(static_cast<__m256i>(x.value()))); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION double hmin( - const_where_expression>, - simd>> const& - x) { - return _mm512_mask_reduce_min_pd(static_cast<__mmask8>(x.mask()), - static_cast<__m512d>(x.value())); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION std::int64_t reduce( - const_where_expression< - simd_mask>, - simd>> const& x, - std::int64_t, std::plus<>) { - return _mm512_mask_reduce_add_epi64(static_cast<__mmask8>(x.mask()), - static_cast<__m512i>(x.value())); -} - -[[nodiscard]] KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION double reduce( - const_where_expression>, - simd>> const& - x, - double, std::plus<>) { - return _mm512_mask_reduce_add_pd(static_cast<__mmask8>(x.mask()), - static_cast<__m512d>(x.value())); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_Scalar.hpp b/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_Scalar.hpp deleted file mode 100644 index f0d06695e5..0000000000 --- a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_Scalar.hpp +++ /dev/null @@ -1,353 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_SIMD_SCALAR_HPP -#define KOKKOS_SIMD_SCALAR_HPP - -#include -#include -#include - -#include - -namespace Kokkos { -namespace Experimental { - -namespace simd_abi { - -class scalar {}; - -} // namespace simd_abi - -template -class simd_mask { - bool m_value; - - public: - using value_type = bool; - using simd_type = simd; - using abi_type = simd_abi::scalar; - using reference = value_type&; - KOKKOS_DEFAULTED_FUNCTION simd_mask() = default; - KOKKOS_FORCEINLINE_FUNCTION static constexpr std::size_t size() { return 1; } - KOKKOS_FORCEINLINE_FUNCTION explicit simd_mask(value_type value) - : m_value(value) {} - template - KOKKOS_FORCEINLINE_FUNCTION simd_mask( - simd_mask const& other) - : m_value(static_cast(other)) {} - KOKKOS_FORCEINLINE_FUNCTION constexpr explicit operator bool() const { - return m_value; - } - KOKKOS_FORCEINLINE_FUNCTION reference operator[](std::size_t) { - return m_value; - } - KOKKOS_FORCEINLINE_FUNCTION value_type operator[](std::size_t) const { - return m_value; - } - KOKKOS_FORCEINLINE_FUNCTION simd_mask - operator||(simd_mask const& other) const { - return simd_mask(m_value || other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION simd_mask - operator&&(simd_mask const& other) const { - return simd_mask(m_value && other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION simd_mask operator!() const { - return simd_mask(!m_value); - } - KOKKOS_FORCEINLINE_FUNCTION bool operator==(simd_mask const& other) const { - return m_value == other.m_value; - } - KOKKOS_FORCEINLINE_FUNCTION bool operator!=(simd_mask const& other) const { - return m_value != other.m_value; - } -}; - -template -class simd { - T m_value; - - public: - using value_type = T; - using abi_type = simd_abi::scalar; - using mask_type = simd_mask; - using reference = value_type&; - KOKKOS_DEFAULTED_FUNCTION simd() = default; - KOKKOS_DEFAULTED_FUNCTION simd(simd const&) = default; - KOKKOS_DEFAULTED_FUNCTION simd(simd&&) = default; - KOKKOS_DEFAULTED_FUNCTION simd& operator=(simd const&) = default; - KOKKOS_DEFAULTED_FUNCTION simd& operator=(simd&&) = default; - KOKKOS_FORCEINLINE_FUNCTION static constexpr std::size_t size() { return 1; } - template , - bool> = false> - KOKKOS_FORCEINLINE_FUNCTION simd(U&& value) : m_value(value) {} - template , - bool> = false> - KOKKOS_FORCEINLINE_FUNCTION explicit simd(simd const& other) - : m_value(static_cast(other)) {} - template ()); } - std::is_invocable_r_v>, - bool> = false> - KOKKOS_FORCEINLINE_FUNCTION simd(G&& gen) - : m_value(gen(std::integral_constant())) {} - KOKKOS_FORCEINLINE_FUNCTION simd operator-() const { return simd(-m_value); } - KOKKOS_FORCEINLINE_FUNCTION simd operator>>(int rhs) const { - return simd(m_value >> rhs); - } - KOKKOS_FORCEINLINE_FUNCTION simd - operator>>(simd const& rhs) const { - return simd(m_value >> static_cast(rhs)); - } - KOKKOS_FORCEINLINE_FUNCTION simd operator<<(int rhs) const { - return simd(m_value << rhs); - } - KOKKOS_FORCEINLINE_FUNCTION simd - operator<<(simd const& rhs) const { - return simd(m_value << static_cast(rhs)); - } - KOKKOS_FORCEINLINE_FUNCTION simd operator&(simd const& other) const { - return m_value & other.m_value; - } - KOKKOS_FORCEINLINE_FUNCTION simd operator|(simd const& other) const { - return m_value | other.m_value; - } - KOKKOS_FORCEINLINE_FUNCTION constexpr explicit operator T() const { - return m_value; - } - KOKKOS_FORCEINLINE_FUNCTION mask_type operator<(simd const& other) const { - return mask_type(m_value < other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION mask_type operator>(simd const& other) const { - return mask_type(m_value > other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION mask_type operator<=(simd const& other) const { - return mask_type(m_value <= other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION mask_type operator>=(simd const& other) const { - return mask_type(m_value >= other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION mask_type operator==(simd const& other) const { - return mask_type(m_value == other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION mask_type operator!=(simd const& other) const { - return mask_type(m_value != other.m_value); - } - KOKKOS_FORCEINLINE_FUNCTION void copy_from(T const* ptr, - element_aligned_tag) { - m_value = *ptr; - } - KOKKOS_FORCEINLINE_FUNCTION void copy_to(T* ptr, element_aligned_tag) const { - *ptr = m_value; - } - KOKKOS_FORCEINLINE_FUNCTION reference operator[](std::size_t) { - return m_value; - } - KOKKOS_FORCEINLINE_FUNCTION value_type operator[](std::size_t) const { - return m_value; - } -}; - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd operator*( - simd const& lhs, - simd const& rhs) { - return simd(static_cast(lhs) * static_cast(rhs)); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd operator/( - simd const& lhs, - simd const& rhs) { - return simd(static_cast(lhs) / static_cast(rhs)); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd operator+( - simd const& lhs, - simd const& rhs) { - return simd(static_cast(lhs) + static_cast(rhs)); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd operator-( - simd const& lhs, - simd const& rhs) { - return simd(static_cast(lhs) - static_cast(rhs)); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd abs( - simd const& a) { - return simd(std::abs(static_cast(a))); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd sqrt( - simd const& a) { - return simd(std::sqrt(static_cast(a))); -} - -template -KOKKOS_FORCEINLINE_FUNCTION simd fma( - simd const& x, simd const& y, - simd const& z) { - return simd((static_cast(x) * static_cast(y)) + - static_cast(z)); -} - -template -KOKKOS_FORCEINLINE_FUNCTION simd condition( - desul::Impl::dont_deduce_this_parameter_t< - simd_mask> const& a, - simd const& b, simd const& c) { - return simd(static_cast(a) ? static_cast(b) - : static_cast(c)); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd copysign( - simd const& a, simd const& b) { - return std::copysign(static_cast(a), static_cast(b)); -} - -template -class const_where_expression, - simd> { - public: - using abi_type = simd_abi::scalar; - using value_type = simd; - using mask_type = simd_mask; - - protected: - value_type& m_value; - mask_type const& m_mask; - - public: - KOKKOS_FORCEINLINE_FUNCTION - const_where_expression(mask_type const& mask_arg, value_type const& value_arg) - : m_value(const_cast(value_arg)), m_mask(mask_arg) {} - KOKKOS_FORCEINLINE_FUNCTION - mask_type const& mask() const { return m_mask; } - KOKKOS_FORCEINLINE_FUNCTION - value_type const& value() const { return m_value; } - KOKKOS_FORCEINLINE_FUNCTION - void copy_to(T* mem, element_aligned_tag) const { - if (static_cast(m_mask)) *mem = static_cast(m_value); - } - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t> - scatter_to(T* mem, simd const& index) const { - if (static_cast(m_mask)) - mem[static_cast(index)] = static_cast(m_value); - } -}; - -template -class where_expression, - simd> - : public const_where_expression, - simd> { - using base_type = const_where_expression, - simd>; - - public: - using typename base_type::value_type; - KOKKOS_FORCEINLINE_FUNCTION - where_expression(simd_mask const& mask_arg, - simd& value_arg) - : base_type(mask_arg, value_arg) {} - KOKKOS_FORCEINLINE_FUNCTION - void copy_from(T const* mem, element_aligned_tag) { - if (static_cast(this->m_mask)) this->m_value = *mem; - } - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t> - gather_from(T const* mem, simd const& index) { - if (static_cast(this->m_mask)) - this->m_value = mem[static_cast(index)]; - } - template >, - bool> = false> - KOKKOS_FORCEINLINE_FUNCTION void operator=(U&& x) { - if (static_cast(this->m_mask)) - this->m_value = - static_cast>(std::forward(x)); - } -}; - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION T -reduce(const_where_expression, - simd> const& x, - T identity_element, BinaryOp) { - return static_cast(x.mask()) ? static_cast(x.value()) - : identity_element; -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION T -hmax(const_where_expression, - simd> const& x) { - return static_cast(x.mask()) ? static_cast(x.value()) - : Kokkos::reduction_identity::max(); -} - -template -[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION T -hmin(const_where_expression, - simd> const& x) { - return static_cast(x.mask()) ? static_cast(x.value()) - : Kokkos::reduction_identity::min(); -} - -} // namespace Experimental -} // namespace Kokkos - -#endif diff --git a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_dummy.cpp b/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_dummy.cpp deleted file mode 100644 index d273cb6d54..0000000000 --- a/bundled/kokkos-3.7.00/simd/src/Kokkos_SIMD_dummy.cpp +++ /dev/null @@ -1,7 +0,0 @@ -// This file is needed in order to get the linker language -// for the header only submodule. -// While we set the language properties in our normal cmake -// path it does not get set in the Trilinos environment. -// Furthermore, setting LINKER_LANGUAGE is only supported -// in CMAKE 3.19 and up. -void KOKKOS_SIMD_SRC_DUMMY_PREVENT_LINK_ERROR() {} diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/.clang-format b/bundled/kokkos-3.7.00/tpls/desul/include/desul/.clang-format deleted file mode 100644 index 7968b43c1d..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/.clang-format +++ /dev/null @@ -1,3 +0,0 @@ -DisableFormat: true -SortIncludes: false - diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Atomic_Ref.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Atomic_Ref.hpp deleted file mode 100644 index fbf2dcf6b7..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Atomic_Ref.hpp +++ /dev/null @@ -1,541 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMIC_REF_IMPL_HPP_ -#define DESUL_ATOMIC_REF_IMPL_HPP_ - -#include -#include -#include - -#include "desul/atomics/Common.hpp" -#include "desul/atomics/Generic.hpp" -#include "desul/atomics/Macros.hpp" - -namespace desul { -namespace Impl { - -// TODO current implementation is missing the following: -// * member functions -// * wait -// * notify_one -// * notify_all - -template {}, - bool = std::is_floating_point{}> -struct basic_atomic_ref; - -// base class for non-integral, non-floating-point, non-pointer types -template -struct basic_atomic_ref { - static_assert(std::is_trivially_copyable{}, ""); - - private: - T* _ptr; - - // 1/2/4/8/16-byte types must be aligned to at least their size - static constexpr int _min_alignment = (sizeof(T) & (sizeof(T) - 1)) || sizeof(T) > 16 - ? 0 - : sizeof(T); - - public: - using value_type = T; - - static constexpr bool is_always_lock_free = atomic_always_lock_free(sizeof(T)); - - static constexpr std::size_t required_alignment = _min_alignment > alignof(T) - ? _min_alignment - : alignof(T); - - basic_atomic_ref() = delete; - basic_atomic_ref& operator=(basic_atomic_ref const&) = delete; - - basic_atomic_ref(basic_atomic_ref const&) = default; - - explicit basic_atomic_ref(T& obj) : _ptr(std::addressof(obj)) {} - - T operator=(T desired) const noexcept { - this->store(desired); - return desired; - } - - operator T() const noexcept { return this->load(); } - - template - DESUL_FUNCTION void store(T desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - atomic_store(_ptr, desired, order, MemoryScope()); - } - - template - DESUL_FUNCTION T load(_MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, order, MemoryScope()); - } - - template - DESUL_FUNCTION T exchange(T desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, desired, order, MemoryScope()); - } - - DESUL_FUNCTION bool is_lock_free() const noexcept { - return atomic_is_lock_free(); - } - - template - DESUL_FUNCTION bool compare_exchange_weak(T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_weak( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_weak( - T& expected, T desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_weak(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_strong( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T& expected, T desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_strong(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } -}; - -// base class for atomic_ref -template -struct basic_atomic_ref { - static_assert(std::is_integral{}, ""); - - private: - T* _ptr; - - public: - using value_type = T; - using difference_type = value_type; - - static constexpr bool is_always_lock_free = atomic_always_lock_free(sizeof(T)); - - static constexpr std::size_t required_alignment = sizeof(T) > alignof(T) ? sizeof(T) - : alignof(T); - - basic_atomic_ref() = delete; - basic_atomic_ref& operator=(basic_atomic_ref const&) = delete; - - explicit basic_atomic_ref(T& obj) : _ptr(&obj) {} - - basic_atomic_ref(basic_atomic_ref const&) = default; - - T operator=(T desired) const noexcept { - this->store(desired); - return desired; - } - - operator T() const noexcept { return this->load(); } - - template - DESUL_FUNCTION void store(T desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - atomic_store(_ptr, desired, order, MemoryScope()); - } - - template - DESUL_FUNCTION T load(_MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, order, MemoryScope()); - } - - template - DESUL_FUNCTION T exchange(T desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, desired, order, MemoryScope()); - } - - DESUL_FUNCTION bool is_lock_free() const noexcept { - return atomic_is_lock_free(); - } - - template - DESUL_FUNCTION bool compare_exchange_weak(T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_weak( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_weak( - T& expected, T desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_weak(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_strong( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T& expected, T desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_strong(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_add(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_add(_ptr, arg, order, MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_sub(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_sub(_ptr, arg, order, MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_and(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_and(_ptr, arg, order, MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_or(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_or(_ptr, arg, order, MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_xor(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_xor(_ptr, arg, order, MemoryScope()); - } - - DESUL_FUNCTION value_type operator++() const noexcept { - return atomic_add_fetch(_ptr, value_type(1), MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator++(int) const noexcept { return fetch_add(1); } - - DESUL_FUNCTION value_type operator--() const noexcept { - return atomic_sub_fetch(_ptr, value_type(1), MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator--(int) const noexcept { return fetch_sub(1); } - - DESUL_FUNCTION value_type operator+=(value_type arg) const noexcept { - atomic_add_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator-=(value_type arg) const noexcept { - atomic_sub_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator&=(value_type arg) const noexcept { - atomic_and_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator|=(value_type arg) const noexcept { - atomic_or_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator^=(value_type arg) const noexcept { - atomic_xor_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } -}; - -// base class for atomic_ref -template -struct basic_atomic_ref { - static_assert(std::is_floating_point{}, ""); - - private: - T* _ptr; - - public: - using value_type = T; - using difference_type = value_type; - - static constexpr bool is_always_lock_free = atomic_always_lock_free(sizeof(T)); - - static constexpr std::size_t required_alignment = alignof(T); - - basic_atomic_ref() = delete; - basic_atomic_ref& operator=(basic_atomic_ref const&) = delete; - - explicit basic_atomic_ref(T& obj) : _ptr(&obj) {} - - basic_atomic_ref(basic_atomic_ref const&) = default; - - T operator=(T desired) const noexcept { - this->store(desired); - return desired; - } - - operator T() const noexcept { return this->load(); } - - template - DESUL_FUNCTION void store(T desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - atomic_store(_ptr, desired, order, MemoryScope()); - } - - template - DESUL_FUNCTION T load(_MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, order, MemoryScope()); - } - - template - DESUL_FUNCTION T exchange(T desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, desired, order, MemoryScope()); - } - - DESUL_FUNCTION bool is_lock_free() const noexcept { - return atomic_is_lock_free(); - } - - template - DESUL_FUNCTION bool compare_exchange_weak(T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_weak( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_weak( - T& expected, T desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_weak(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_strong( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T& expected, T desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_strong(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_add(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_add(_ptr, arg, order, MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_sub(value_type arg, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_sub(_ptr, arg, order, MemoryScope()); - } - - DESUL_FUNCTION value_type operator+=(value_type arg) const noexcept { - atomic_add_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator-=(value_type arg) const noexcept { - atomic_sub_fetch(_ptr, arg, MemoryOrder(), MemoryScope()); - } -}; - -// base class for atomic_ref -template -struct basic_atomic_ref { - private: - T** _ptr; - - public: - using value_type = T*; - using difference_type = std::ptrdiff_t; - - static constexpr bool is_always_lock_free = atomic_always_lock_free(sizeof(T)); - - static constexpr std::size_t required_alignment = alignof(T*); - - basic_atomic_ref() = delete; - basic_atomic_ref& operator=(basic_atomic_ref const&) = delete; - - explicit basic_atomic_ref(T*& arg) : _ptr(std::addressof(arg)) {} - - basic_atomic_ref(basic_atomic_ref const&) = default; - - T* operator=(T* desired) const noexcept { - this->store(desired); - return desired; - } - - operator T*() const noexcept { return this->load(); } - - template - DESUL_FUNCTION void store(T* desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - atomic_store(_ptr, desired, order, MemoryScope()); - } - - template - DESUL_FUNCTION T* load(_MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, order, MemoryScope()); - } - - template - DESUL_FUNCTION T* exchange(T* desired, - _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_load(_ptr, desired, order, MemoryScope()); - } - - DESUL_FUNCTION bool is_lock_free() const noexcept { - return atomic_is_lock_free(); - } - - template - DESUL_FUNCTION bool compare_exchange_weak(T*& expected, - T* desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_weak( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_weak( - T*& expected, T* desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_weak(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T*& expected, - T* desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure) const noexcept { - return atomic_compare_exchange_strong( - _ptr, expected, desired, success, failure, MemoryScope()); - } - - template - DESUL_FUNCTION bool compare_exchange_strong( - T*& expected, T* desired, _MemoryOrder order = _MemoryOrder()) const noexcept { - return compare_exchange_strong(expected, - desired, - order, - cmpexch_failure_memory_order<_MemoryOrder>(), - MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_add(difference_type d, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_add(_ptr, _type_size(d), order, MemoryScope()); - } - - template - DESUL_FUNCTION value_type - fetch_sub(difference_type d, _MemoryOrder order = _MemoryOrder()) const noexcept { - return atomic_fetch_sub(_ptr, _type_size(d), order, MemoryScope()); - } - - DESUL_FUNCTION value_type operator++() const noexcept { - return atomic_add_fetch(_ptr, _type_size(1), MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator++(int) const noexcept { return fetch_add(1); } - - DESUL_FUNCTION value_type operator--() const noexcept { - return atomic_sub_fetch(_ptr, _type_size(1), MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator--(int) const noexcept { return fetch_sub(1); } - - DESUL_FUNCTION value_type operator+=(difference_type d) const noexcept { - atomic_add_fetch(_ptr, _type_size(d), MemoryOrder(), MemoryScope()); - } - - DESUL_FUNCTION value_type operator-=(difference_type d) const noexcept { - atomic_sub_fetch(_ptr, _type_size(d), MemoryOrder(), MemoryScope()); - } - - private: - static constexpr std::ptrdiff_t _type_size(std::ptrdiff_t d) noexcept { - static_assert(std::is_object{}, ""); - return d * sizeof(T); - } -}; - -} // namespace Impl - -template -struct scoped_atomic_ref : Impl::basic_atomic_ref { - explicit scoped_atomic_ref(T& obj) noexcept - : Impl::basic_atomic_ref(obj) {} - - scoped_atomic_ref& operator=(scoped_atomic_ref const&) = delete; - - scoped_atomic_ref(scoped_atomic_ref const&) = default; - - using Impl::basic_atomic_ref::operator=; -}; - -} // namespace desul - -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/CUDA.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/CUDA.hpp deleted file mode 100644 index 87c0df4af9..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/CUDA.hpp +++ /dev/null @@ -1,664 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ -#ifndef DESUL_ATOMICS_CUDA_HPP_ -#define DESUL_ATOMICS_CUDA_HPP_ - -#ifdef DESUL_HAVE_CUDA_ATOMICS -// When building with Clang we need to include the device functions always since Clang -// must see a consistent overload set in both device and host compilation, but that -// means we need to know on the host what to make visible, i.e. we need a host side -// compile knowledge of architecture. -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)) || \ - (!defined(__NVCC__) && !defined(DESUL_CUDA_ARCH_IS_PRE_VOLTA)) -#define DESUL_HAVE_CUDA_ATOMICS_ASM -#include -#endif - -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 700)) || \ - (!defined(__NVCC__) && !defined(DESUL_HAVE_CUDA_ATOMICS_ASM)) -namespace desul { -namespace Impl { -template -struct is_cuda_atomic_integer_type { - static constexpr bool value = std::is_same::value || - std::is_same::value || - std::is_same::value; -}; - -template -struct is_cuda_atomic_add_type { - static constexpr bool value = is_cuda_atomic_integer_type::value || -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 600) - std::is_same::value || -#endif - std::is_same::value; -}; - -template -struct is_cuda_atomic_sub_type { - static constexpr bool value = - std::is_same::value || std::is_same::value; -}; -} // namespace Impl - -// Atomic Add -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_add(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicAdd(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_add(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicAdd(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_add(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_add(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic Sub -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_sub(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicSub(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_sub(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicSub(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_sub(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_sub(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Wrap around atomic add -__device__ inline unsigned int atomic_fetch_inc_mod(unsigned int* dest, - unsigned int val, - MemoryOrderRelaxed, - MemoryScopeDevice) { - return atomicInc(dest, val); -} - -template -__device__ inline unsigned int atomic_fetch_inc_mod(unsigned int* dest, - unsigned int val, - MemoryOrder, - MemoryScopeDevice) { - __threadfence(); - unsigned int return_val = atomicInc(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline unsigned int atomic_fetch_inc_mod(unsigned int* dest, - unsigned int val, - MemoryOrder, - MemoryScopeCore) { - return atomic_fetch_inc_mod(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Wrap around atomic sub -__device__ inline unsigned int atomic_fetch_dec_mod(unsigned int* dest, - unsigned int val, - MemoryOrderRelaxed, - MemoryScopeDevice) { - return atomicDec(dest, val); -} - -template -__device__ inline unsigned int atomic_fetch_dec_mod(unsigned int* dest, - unsigned int val, - MemoryOrder, - MemoryScopeDevice) { - __threadfence(); - unsigned int return_val = atomicDec(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline unsigned int atomic_fetch_dec_mod(unsigned int* dest, - unsigned int val, - MemoryOrder, - MemoryScopeCore) { - return atomic_fetch_dec_mod(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic Inc -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_inc(T* dest, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicAdd(dest, T(1)); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_inc(T* dest, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicAdd(dest, T(1)); - __threadfence(); - - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_inc(T* dest, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_add(dest, T(1), MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic Dec -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_dec(T* dest, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicSub(dest, T(1)); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_dec(T* dest, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicSub(dest, T(1)); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_dec(T* dest, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_sub(dest, T(1), MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic Max -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_max(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicMax(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_max(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicMax(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_max(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_max(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic Min -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_min(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicMin(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_min(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicMin(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_min(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_min(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic And -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_and(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicAnd(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_and(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicAnd(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_and(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_and(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic XOR -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_xor(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicXor(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_xor(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicXor(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_xor(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_xor(dest, val, MemoryOrder(), MemoryScopeDevice()); -} - -// Atomic OR -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_or(T* dest, T val, MemoryOrderRelaxed, MemoryScopeDevice) { - return atomicOr(dest, val); -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_or(T* dest, T val, MemoryOrder, MemoryScopeDevice) { - __threadfence(); - T return_val = atomicOr(dest, val); - __threadfence(); - return return_val; -} - -template -__device__ inline - std::enable_if_t::value, T> - atomic_fetch_or(T* dest, T val, MemoryOrder, MemoryScopeCore) { - return atomic_fetch_or(dest, val, MemoryOrder(), MemoryScopeDevice()); -} -} // namespace desul -#endif - -#if !defined(__NVCC__) -// Functions defined as device functions in CUDA which don't exist in the GCC overload -// set -namespace desul { - -#if defined(DESUL_HAVE_CUDA_ATOMICS_ASM) -#define DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(TYPE, ORDER, SCOPE) \ - inline void atomic_add(TYPE* const dest, TYPE val, ORDER order, SCOPE scope) { \ - (void)atomic_fetch_add(dest, val, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(int32_t, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(long, - MemoryOrderRelaxed, - MemoryScopeDevice); // only for ASM? -DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(unsigned int, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(unsigned long long, - MemoryOrderRelaxed, - MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(float, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_ADD(double, MemoryOrderRelaxed, MemoryScopeDevice); - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_SUB(TYPE, ORDER, SCOPE) \ - inline void atomic_sub(TYPE* const dest, TYPE val, ORDER order, SCOPE scope) { \ - (void)atomic_fetch_sub(dest, val, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_SUB(int32_t, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_SUB(long, - MemoryOrderRelaxed, - MemoryScopeDevice); // only for ASM? -DESUL_IMPL_CUDA_HOST_ATOMIC_SUB(unsigned int, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_SUB(float, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_SUB(double, MemoryOrderRelaxed, MemoryScopeDevice); - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_INC(TYPE, ORDER, SCOPE) \ - inline void atomic_inc(TYPE* const dest, ORDER order, SCOPE scope) { \ - (void)atomic_fetch_inc(dest, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_INC(unsigned int, - MemoryOrderRelaxed, - MemoryScopeDevice); // only for ASM? - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_DEC(TYPE, ORDER, SCOPE) \ - inline void atomic_dec(TYPE* const dest, ORDER order, SCOPE scope) { \ - (void)atomic_fetch_dec(dest, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_DEC(unsigned, - MemoryOrderRelaxed, - MemoryScopeDevice); // only for ASM? - -#endif // DESUL_HAVE_CUDA_ATOMICS_ASM - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_INC_MOD(TYPE, ORDER, SCOPE) \ - inline TYPE atomic_fetch_inc_mod(TYPE* dest, TYPE val, ORDER order, SCOPE scope) { \ - using cas_t = typename Impl::atomic_compare_exchange_type::type; \ - cas_t oldval = reinterpret_cast(*dest); \ - cas_t assume = oldval; \ - do { \ - assume = oldval; \ - TYPE newval = (reinterpret_cast(assume) >= val) \ - ? static_cast(0) \ - : reinterpret_cast(assume) + static_cast(1); \ - oldval = desul::atomic_compare_exchange(reinterpret_cast(dest), \ - assume, \ - reinterpret_cast(newval), \ - order, \ - scope); \ - } while (assume != oldval); \ - return reinterpret_cast(oldval); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_INC_MOD(unsigned int, - MemoryOrderRelaxed, - MemoryScopeDevice); -#define DESUL_IMPL_CUDA_HOST_ATOMIC_DEC_MOD(TYPE, ORDER, SCOPE) \ - inline TYPE atomic_fetch_dec_mod(TYPE* dest, TYPE val, ORDER order, SCOPE scope) { \ - using cas_t = typename Impl::atomic_compare_exchange_type::type; \ - cas_t oldval = reinterpret_cast(*dest); \ - cas_t assume = oldval; \ - do { \ - assume = oldval; \ - TYPE newval = ((reinterpret_cast(assume) == static_cast(0)) | \ - (reinterpret_cast(assume) > val)) \ - ? val \ - : reinterpret_cast(assume) - static_cast(1); \ - oldval = desul::atomic_compare_exchange(reinterpret_cast(dest), \ - assume, \ - reinterpret_cast(newval), \ - order, \ - scope); \ - } while (assume != oldval); \ - return reinterpret_cast(oldval); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_DEC_MOD(unsigned int, - MemoryOrderRelaxed, - MemoryScopeDevice); - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_ADD(TYPE, ORDER, SCOPE) \ - inline TYPE atomic_fetch_add(TYPE* const dest, TYPE val, ORDER order, SCOPE scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::AddOper(), dest, val, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_ADD(float, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_ADD(double, MemoryOrderRelaxed, MemoryScopeDevice); - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_SUB(TYPE, ORDER, SCOPE) \ - inline TYPE atomic_fetch_sub(TYPE* const dest, TYPE val, ORDER order, SCOPE scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::SubOper(), dest, val, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_SUB(float, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_SUB(double, MemoryOrderRelaxed, MemoryScopeDevice); - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MAX(TYPE, ORDER, SCOPE) \ - inline TYPE atomic_fetch_max(TYPE* const dest, TYPE val, ORDER order, SCOPE scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::MaxOper(), dest, val, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MAX(int, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MAX(long, - MemoryOrderRelaxed, - MemoryScopeDevice); // only for ASM? -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MAX(unsigned int, - MemoryOrderRelaxed, - MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MAX(unsigned long, - MemoryOrderRelaxed, - MemoryScopeDevice); -// DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MAX(unsigned long -// long,MemoryOrderRelaxed,MemoryScopeDevice); - -#define DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MIN(TYPE, ORDER, SCOPE) \ - inline TYPE atomic_fetch_min(TYPE* const dest, TYPE val, ORDER order, SCOPE scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::MinOper(), dest, val, order, scope); \ - } -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MIN(int, MemoryOrderRelaxed, MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MIN(long, - MemoryOrderRelaxed, - MemoryScopeDevice); // only for ASM? -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MIN(unsigned int, - MemoryOrderRelaxed, - MemoryScopeDevice); -DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MIN(unsigned long, - MemoryOrderRelaxed, - MemoryScopeDevice); -// DESUL_IMPL_CUDA_HOST_ATOMIC_FETCH_MIN(unsigned long -// long,MemoryOrderRelaxed,MemoryScopeDevice); inline void atomic_fetch_max(int32_t* -// const dest, int32_t val, MemoryOrderRelaxed order, MemoryScopeDevice scope) { - -} // namespace desul - -// Functions defined int the GCC overload set but not in the device overload set -namespace desul { -__device__ inline unsigned long long atomic_fetch_add(unsigned long long* const dest, - unsigned long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::AddOper(), - dest, - val, - order, - scope); -} -__device__ inline long long atomic_fetch_add(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::AddOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_add(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::AddOper(), dest, val, order, scope); -} -__device__ inline long long atomic_fetch_sub(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::SubOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_sub(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::SubOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_max(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::MaxOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_min(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::MinOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_or(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::OrOper(), dest, val, order, scope); -} -__device__ inline long long atomic_fetch_or(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::OrOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_xor(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::XorOper(), dest, val, order, scope); -} -__device__ inline long long atomic_fetch_xor(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::XorOper(), dest, val, order, scope); -} -__device__ inline long atomic_fetch_and(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::AndOper(), dest, val, order, scope); -} -__device__ inline long long atomic_fetch_and(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_fetch_oper( - Impl::AndOper(), dest, val, order, scope); -} - -__device__ inline unsigned long long atomic_add_fetch(unsigned long long* const dest, - unsigned long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::AddOper(), - dest, - val, - order, - scope); -} -__device__ inline long long atomic_add_fetch(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::AddOper(), dest, val, order, scope); -} -__device__ inline long atomic_add_fetch(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::AddOper(), dest, val, order, scope); -} -__device__ inline long long atomic_sub_fetch(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::SubOper(), dest, val, order, scope); -} -__device__ inline long atomic_sub_fetch(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::SubOper(), dest, val, order, scope); -} -__device__ inline long long atomic_or_fetch(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::OrOper(), dest, val, order, scope); -} -__device__ inline long atomic_or_fetch(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::OrOper(), dest, val, order, scope); -} -__device__ inline long long atomic_xor_fetch(long long* const dest, - long long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::XorOper(), dest, val, order, scope); -} -__device__ inline long atomic_xor_fetch(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::XorOper(), dest, val, order, scope); -} -__device__ inline long long atomic_and_fetch(long long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::AndOper(), dest, val, order, scope); -} -__device__ inline long atomic_and_fetch(long* const dest, - long val, - MemoryOrderRelaxed order, - MemoryScopeDevice scope) { - return Impl::atomic_oper_fetch( - Impl::AndOper(), dest, val, order, scope); -} -} // namespace desul -#endif - -#endif // DESUL_HAVE_CUDA_ATOMICS -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_CUDA.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_CUDA.hpp deleted file mode 100644 index 310c59f55e..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_CUDA.hpp +++ /dev/null @@ -1,283 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_CUDA_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_CUDA_HPP_ -#include "desul/atomics/Common.hpp" -#include "desul/atomics/Lock_Array_Cuda.hpp" - -#ifdef DESUL_HAVE_CUDA_ATOMICS -namespace desul { -// Only include if compiling device code, or the CUDA compiler is not NVCC (i.e. Clang) -// atomic_thread_fence implementation -#if defined(__CUDA_ARCH__) || !defined(__NVCC__) -__device__ inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeDevice) { - __threadfence(); -} -__device__ inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeDevice) { - __threadfence(); -} -__device__ inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeDevice) { - __threadfence(); -} -__device__ inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeDevice) { - __threadfence(); -} -__device__ inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeCore) { - __threadfence_block(); -} -__device__ inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeCore) { - __threadfence_block(); -} -__device__ inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeCore) { - __threadfence_block(); -} -__device__ inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeCore) { - __threadfence_block(); -} -#if (__CUDA_ARCH__ >= 600) || !defined(__NVCC__) -__device__ inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeNode) { - __threadfence_system(); -} -__device__ inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeNode) { - __threadfence_system(); -} -__device__ inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeNode) { - __threadfence_system(); -} -__device__ inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeNode) { - __threadfence_system(); -} -#endif -#endif -} // namespace desul - -// Compare Exchange for PRE Volta, not supported with CLANG as CUDA compiler, since we -// do NOT have a way of having the code included for clang only when the CC is smaller -// than 700 But on Clang the device side symbol list must be independent of -// __CUDA_ARCH__ -// FIXME temporary fix for https://github.com/kokkos/kokkos/issues/4390 -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 700) || \ - (!defined(__NVCC__) && defined(DESUL_CUDA_ARCH_IS_PRE_VOLTA) && 0) -namespace desul { -template -__device__ typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned int) == 4, - "this function assumes an unsigned int is 32-bit"); - unsigned int return_val = atomicCAS(reinterpret_cast(dest), - reinterpret_cast(compare), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} -template -__device__ typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned long long int) == 8, - "this function assumes an unsigned long long is 64-bit"); - unsigned long long int return_val = - atomicCAS(reinterpret_cast(dest), - reinterpret_cast(compare), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelease, MemoryScope) { - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderAcquire, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderAcqRel, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if::type atomic_exchange( - T* const dest, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned int) == 4, - "this function assumes an unsigned int is 32-bit"); - unsigned int return_val = atomicExch(reinterpret_cast(dest), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} -template -__device__ typename std::enable_if::type atomic_exchange( - T* const dest, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned long long int) == 8, - "this function assumes an unsigned long long is 64-bit"); - unsigned long long int return_val = - atomicExch(reinterpret_cast(dest), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange(T* const dest, T value, MemoryOrderRelease, MemoryScope) { - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange(T* const dest, T value, MemoryOrderAcquire, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange(T* const dest, T value, MemoryOrderAcqRel, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return reinterpret_cast(return_val); -} -} // namespace desul -#endif - -// Including CUDA ptx based exchange atomics -// When building with clang we need to include the device functions always -// since clang must see a consistent overload set in both device and host compilation -// but that means we need to know on the host what to make visible, i.e. we need -// a host side compile knowledge of architecture. -// We simply can say DESUL proper doesn't support clang CUDA build pre Volta, -// Kokkos has that knowledge and so I use it here, allowing in Kokkos to use -// clang with pre Volta as CUDA compiler -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)) || \ - (!defined(__NVCC__) && !defined(DESUL_CUDA_ARCH_IS_PRE_VOLTA)) -#include -#endif - -// SeqCst is not directly supported by PTX, need the additional fences: - -#if defined(__CUDA_ARCH__) || !defined(__NVCC__) -namespace desul { -template -__device__ typename std::enable_if::type atomic_exchange( - T* const dest, T value, MemoryOrderSeqCst, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} -template -__device__ typename std::enable_if::type atomic_exchange( - T* const dest, T value, MemoryOrderSeqCst, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} -template -__device__ typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderSeqCst, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} -template -__device__ typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderSeqCst, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} -} // namespace desul -#endif - -#if defined(__CUDA_ARCH__) || !defined(__NVCC__) -namespace desul { -template -__device__ typename std::enable_if<(sizeof(T) != 8) && (sizeof(T) != 4), T>::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrder, MemoryScope scope) { - // This is a way to avoid dead lock in a warp or wave front - T return_val; - int done = 0; - unsigned int mask = DESUL_IMPL_ACTIVEMASK; - unsigned int active = DESUL_IMPL_BALLOT_MASK(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda((void*)dest, scope)) { - if (std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), scope); - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = *dest; - if (return_val == compare) { - *dest = value; - atomic_thread_fence(MemoryOrderRelease(), scope); - } - Impl::unlock_address_cuda((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(mask, done); - } - return return_val; -} -template -__device__ typename std::enable_if<(sizeof(T) != 8) && (sizeof(T) != 4), T>::type -atomic_exchange(T* const dest, T value, MemoryOrder, MemoryScope scope) { - // This is a way to avoid dead lock in a warp or wave front - T return_val; - int done = 0; - unsigned int mask = DESUL_IMPL_ACTIVEMASK; - unsigned int active = DESUL_IMPL_BALLOT_MASK(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda((void*)dest, scope)) { - if (std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), scope); - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = *dest; - *dest = value; - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address_cuda((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(mask, done); - } - return return_val; -} -} // namespace desul -#endif - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_GCC.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_GCC.hpp deleted file mode 100644 index fad3c43b04..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_GCC.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_GCC_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_GCC_HPP_ -#include "desul/atomics/Common.hpp" - -#ifdef DESUL_HAVE_GCC_ATOMICS -#if !defined(DESUL_HAVE_16BYTE_COMPARE_AND_SWAP) && !defined(__CUDACC__) -// This doesn't work in WSL?? -//#define DESUL_HAVE_16BYTE_COMPARE_AND_SWAP -#endif -namespace desul { - -namespace Impl { -template -struct atomic_exchange_available_gcc { - constexpr static bool value = -#ifndef DESUL_HAVE_LIBATOMIC - ((sizeof(T) == 4 && alignof(T) == 4) || -#ifdef DESUL_HAVE_16BYTE_COMPARE_AND_SWAP - (sizeof(T) == 16 && alignof(T) == 16) || -#endif - (sizeof(T) == 8 && alignof(T) == 8)) && -#endif - std::is_trivially_copyable::value; -}; -} // namespace Impl - -#if defined(__clang__) && (__clang_major__ >= 7) && !defined(__APPLE__) -// clang-format off -// Disable warning for large atomics on clang 7 and up (checked with godbolt) -// error: large atomic operation may incur significant performance penalty [-Werror,-Watomic-alignment] -// https://godbolt.org/z/G7YhqhbG6 -// clang-format on -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Watomic-alignment" -#endif -template -void atomic_thread_fence(MemoryOrder, MemoryScope) { - __atomic_thread_fence(GCCMemoryOrder::value); -} - -template -std::enable_if_t::value, T> atomic_exchange( - T* dest, T value, MemoryOrder, MemoryScope) { - T return_val; - __atomic_exchange(dest, &value, &return_val, GCCMemoryOrder::value); - return return_val; -} - -// Failure mode for atomic_compare_exchange_n cannot be RELEASE nor ACQREL so -// Those two get handled separatly. -template -std::enable_if_t::value, T> -atomic_compare_exchange(T* dest, T compare, T value, MemoryOrder, MemoryScope) { - (void)__atomic_compare_exchange(dest, - &compare, - &value, - false, - GCCMemoryOrder::value, - GCCMemoryOrder::value); - return compare; -} - -template -std::enable_if_t::value, T> -atomic_compare_exchange(T* dest, T compare, T value, MemoryOrderRelease, MemoryScope) { - (void)__atomic_compare_exchange( - dest, &compare, &value, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); - return compare; -} - -template -std::enable_if_t::value, T> -atomic_compare_exchange(T* dest, T compare, T value, MemoryOrderAcqRel, MemoryScope) { - (void)__atomic_compare_exchange( - dest, &compare, &value, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE); - return compare; -} - -#if defined(__clang__) && (__clang_major__ >= 7) && !defined(__APPLE__) -#pragma GCC diagnostic pop -#endif -} // namespace desul -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_HIP.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_HIP.hpp deleted file mode 100644 index 96739bc1fc..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_HIP.hpp +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_HIP_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_HIP_HPP_ -#include "desul/atomics/Common.hpp" -#include "desul/atomics/Lock_Array_HIP.hpp" - -#ifdef DESUL_HAVE_HIP_ATOMICS -namespace desul { -inline __device__ void atomic_thread_fence(MemoryOrderRelease, MemoryScopeDevice) { - __threadfence(); -} -inline __device__ void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeDevice) { - __threadfence(); -} -inline __device__ void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeDevice) { - __threadfence(); -} -inline __device__ void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeDevice) { - __threadfence(); -} -inline __device__ void atomic_thread_fence(MemoryOrderRelease, MemoryScopeCore) { - __threadfence_block(); -} -inline __device__ void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeCore) { - __threadfence_block(); -} -inline __device__ void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeCore) { - __threadfence_block(); -} -inline __device__ void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeCore) { - __threadfence_block(); -} -inline __device__ void atomic_thread_fence(MemoryOrderRelease, MemoryScopeNode) { - __threadfence_system(); -} -inline __device__ void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeNode) { - __threadfence_system(); -} -inline __device__ void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeNode) { - __threadfence_system(); -} -inline __device__ void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeNode) { - __threadfence_system(); -} - -template -__device__ typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned int) == 4, - "this function assumes an unsigned int is 32-bit"); - unsigned int return_val = atomicCAS(reinterpret_cast(dest), - reinterpret_cast(compare), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} -template -__device__ typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned long long int) == 8, - "this function assumes an unsigned long long is 64-bit"); - unsigned long long int return_val = - atomicCAS(reinterpret_cast(dest), - reinterpret_cast(compare), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelease, MemoryScope) { - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderAcquire, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderAcqRel, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if::type atomic_exchange( - T* const dest, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned int) == 4, - "this function assumes an unsigned int is 32-bit"); - unsigned int return_val = atomicExch(reinterpret_cast(dest), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} -template -__device__ typename std::enable_if::type atomic_exchange( - T* const dest, T value, MemoryOrderRelaxed, MemoryScope) { - static_assert(sizeof(unsigned long long int) == 8, - "this function assumes an unsigned long long is 64-bit"); - unsigned long long int return_val = - atomicExch(reinterpret_cast(dest), - reinterpret_cast(value)); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange(T* const dest, T compare, T value, MemoryOrderRelease, MemoryScope) { - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange( - T* const dest, T /*compare*/, T value, MemoryOrderAcquire, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange(T* const dest, T value, MemoryOrderAcqRel, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_exchange(T* const dest, T value, MemoryOrderSeqCst, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_exchange(dest, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return reinterpret_cast(return_val); -} - -template -__device__ typename std::enable_if::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderSeqCst, MemoryScope) { - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T return_val = atomic_compare_exchange( - dest, compare, value, MemoryOrderRelaxed(), MemoryScope()); - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} - -template -__device__ typename std::enable_if<(sizeof(T) != 8) && (sizeof(T) != 4), T>::type -atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrder, MemoryScope scope) { - // This is a way to avoid dead lock in a warp or wave front - T return_val; - int done = 0; - unsigned long long int active = DESUL_IMPL_BALLOT_MASK(1); - unsigned long long int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip((void*)dest, scope)) { - if (std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), scope); - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = *dest; - if (return_val == compare) { - *dest = value; - atomic_thread_fence(MemoryOrderRelease(), scope); - } - Impl::unlock_address_hip((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(done); - } - return return_val; -} - -template -__device__ typename std::enable_if<(sizeof(T) != 8) && (sizeof(T) != 4), T>::type -atomic_exchange(T* const dest, T value, MemoryOrder, MemoryScope scope) { - // This is a way to avoid dead lock in a warp or wave front - T return_val; - int done = 0; - unsigned long long int active = DESUL_IMPL_BALLOT_MASK(1); - unsigned long long int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip((void*)dest, scope)) { - if (std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), scope); - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = *dest; - *dest = value; - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address_hip((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(done); - } - return return_val; -} -} // namespace desul -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_MSVC.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_MSVC.hpp deleted file mode 100644 index edf72f13db..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_MSVC.hpp +++ /dev/null @@ -1,237 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_MSVC_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_MSVC_HPP_ -#include - -#include "desul/atomics/Common.hpp" -#ifdef DESUL_HAVE_MSVC_ATOMICS - -#ifndef DESUL_HAVE_16BYTE_COMPARE_AND_SWAP -#define DESUL_HAVE_16BYTE_COMPARE_AND_SWAP -#endif - -namespace desul { - -// Forward declare these functions. They use compare_exchange themselves -// so the actual header file with them comes after this file is included. -namespace Impl { -template -inline bool lock_address(void* ptr, MemoryScope ms); - -template -void unlock_address(void* ptr, MemoryScope ms); -} // namespace Impl - -template -void atomic_thread_fence(MemoryOrder, MemoryScope) { - std::atomic_thread_fence(CXXMemoryOrder::value); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderRelaxed, - MemoryScope) { - char return_val = _InterlockedExchange8((char*)dest, *((char*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderRelaxed, - MemoryScope) { - short return_val = _InterlockedExchange16((short*)dest, *((short*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderRelaxed, - MemoryScope) { - long return_val = _InterlockedExchange((long*)dest, *((long*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderRelaxed, - MemoryScope) { - __int64 return_val = _InterlockedExchange64((__int64*)dest, *((__int64*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderSeqCst, - MemoryScope) { - char return_val = _InterlockedExchange8((char*)dest, *((char*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderSeqCst, - MemoryScope) { - short return_val = _InterlockedExchange16((short*)dest, *((short*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderSeqCst, - MemoryScope) { - long return_val = _InterlockedExchange((long*)dest, *((long*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T val, - MemoryOrderSeqCst, - MemoryScope) { - __int64 return_val = _InterlockedExchange64((__int64*)dest, *((__int64*)&val)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if<(sizeof(T) != 1 && sizeof(T) != 2 && sizeof(T) != 4 && - sizeof(T) != 8), - T>::type -atomic_exchange(T* const dest, T val, MemoryOrder, MemoryScope scope) { - while (!Impl::lock_address((void*)dest, scope)) { - } - if (std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), scope); - atomic_thread_fence(MemoryOrderAcquire(), scope); - T return_val = *dest; - *dest = val; - atomic_thread_fence(MemoryOrderRelease(), scope); - - Impl::unlock_address((void*)dest, scope); - return return_val; -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderRelaxed, MemoryScope) { - char return_val = - _InterlockedCompareExchange8((char*)dest, *((char*)&val), *((char*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderRelaxed, MemoryScope) { - short return_val = - _InterlockedCompareExchange16((short*)dest, *((short*)&val), *((short*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderRelaxed, MemoryScope) { - long return_val = - _InterlockedCompareExchange((long*)dest, *((long*)&val), *((long*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderRelaxed, MemoryScope) { - __int64 return_val = _InterlockedCompareExchange64( - (__int64*)dest, *((__int64*)&val), *((__int64*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderRelaxed, MemoryScope) { - Dummy16ByteValue* val16 = reinterpret_cast(&val); - (void)_InterlockedCompareExchange128(reinterpret_cast<__int64*>(dest), - val16->value2, - val16->value1, - (reinterpret_cast<__int64*>(&compare))); - return compare; -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderSeqCst, MemoryScope) { - char return_val = - _InterlockedCompareExchange8((char*)dest, *((char*)&val), *((char*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderSeqCst, MemoryScope) { - short return_val = - _InterlockedCompareExchange16((short*)dest, *((short*)&val), *((short*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderSeqCst, MemoryScope) { - long return_val = - _InterlockedCompareExchange((long*)dest, *((long*)&val), *((long*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderSeqCst, MemoryScope) { - __int64 return_val = _InterlockedCompareExchange64( - (__int64*)dest, *((__int64*)&val), *((__int64*)&compare)); - return *(reinterpret_cast(&return_val)); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrderSeqCst, MemoryScope) { - Dummy16ByteValue* val16 = reinterpret_cast(&val); - (void)_InterlockedCompareExchange128(reinterpret_cast<__int64*>(dest), - val16->value2, - val16->value1, - (reinterpret_cast<__int64*>(&compare))); - return compare; -} - -template -typename std::enable_if<(sizeof(T) != 1 && sizeof(T) != 2 && sizeof(T) != 4 && - sizeof(T) != 8 && sizeof(T) != 16), - T>::type -atomic_compare_exchange( - T* const dest, T compare, T val, MemoryOrder, MemoryScope scope) { - while (!Impl::lock_address((void*)dest, scope)) { - } - if (std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), scope); - atomic_thread_fence(MemoryOrderAcquire(), scope); - T return_val = *dest; - if (return_val == compare) { - *dest = val; - atomic_thread_fence(MemoryOrderRelease(), scope); - } - - Impl::unlock_address((void*)dest, scope); - return return_val; -} - -} // namespace desul - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_OpenMP.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_OpenMP.hpp deleted file mode 100644 index dfea81a4d6..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_OpenMP.hpp +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_OPENMP_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_OPENMP_HPP_ -#include - -#include "desul/atomics/Common.hpp" - -#ifdef DESUL_HAVE_OPENMP_ATOMICS -namespace desul { - -#if _OPENMP > 201800 -// atomic_thread_fence for Core Scope -inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeCore) { -// There is no seq_cst flush in OpenMP, isn't it the same anyway for fence? -#pragma omp flush acq_rel -} -inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeCore) { -#pragma omp flush acq_rel -} -inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeCore) { -#pragma omp flush release -} -inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeCore) { -#pragma omp flush acquire -} -// atomic_thread_fence for Device Scope -inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeDevice) { -// There is no seq_cst flush in OpenMP, isn't it the same anyway for fence? -#pragma omp flush acq_rel -} -inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeDevice) { -#pragma omp flush acq_rel -} -inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeDevice) { -#pragma omp flush release -} -inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeDevice) { -#pragma omp flush acquire -} -#else -// atomic_thread_fence for Core Scope -inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeCore) { -#pragma omp flush -} -inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeCore) { -#pragma omp flush -} -inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeCore) { -#pragma omp flush -} -inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeCore) { -#pragma omp flush -} -// atomic_thread_fence for Device Scope -inline void atomic_thread_fence(MemoryOrderSeqCst, MemoryScopeDevice) { -#pragma omp flush -} -inline void atomic_thread_fence(MemoryOrderAcqRel, MemoryScopeDevice) { -#pragma omp flush -} -inline void atomic_thread_fence(MemoryOrderRelease, MemoryScopeDevice) { -#pragma omp flush -} -inline void atomic_thread_fence(MemoryOrderAcquire, MemoryScopeDevice) { -#pragma omp flush -} -#endif - -template -T atomic_exchange(T* dest, T value, MemoryOrder, MemoryScope) { - T return_val; - if (!std::is_same::value) - atomic_thread_fence(MemoryOrderAcquire(), MemoryScope()); - T& x = *dest; -#pragma omp atomic capture - { - return_val = x; - x = value; - } - if (!std::is_same::value) - atomic_thread_fence(MemoryOrderRelease(), MemoryScope()); - return return_val; -} - -// OpenMP doesn't have compare exchange, so we use build-ins and rely on testing that -// this works Note that means we test this in OpenMPTarget offload regions! -template -std::enable_if_t atomic_compare_exchange( - T* dest, T compare, T value, MemoryOrder, MemoryScope) { - using cas_t = typename Impl::atomic_compare_exchange_type::type; - cas_t retval = __sync_val_compare_and_swap(reinterpret_cast(dest), - reinterpret_cast(compare), - reinterpret_cast(value)); - return reinterpret_cast(retval); -} - -#if defined(__clang__) && (__clang_major__ >= 7) -// Disable warning for large atomics on clang 7 and up (checked with godbolt) -// clang-format off -// error: large atomic operation may incur significant performance penalty [-Werror,-Watomic-alignment] -// clang-format on -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Watomic-alignment" -#endif - -// Make 16 byte cas work on host at least -#pragma omp begin declare variant match(device = {kind(host)}) -template -std::enable_if_t -atomic_compare_exchange(T* dest, T compare, T value, MemoryOrder, MemoryScope) { - (void)__atomic_compare_exchange(dest, - &compare, - &value, - false, - GCCMemoryOrder::value, - GCCMemoryOrder::value); - return compare; -} -#pragma omp end declare variant - -#pragma omp begin declare variant match(device = {kind(nohost)}) -template -std::enable_if_t -atomic_compare_exchange(T* /*dest*/, T /*compare*/, T value, MemoryOrder, MemoryScope) { - // FIXME make sure this never gets called - return value; -} -#pragma omp end declare variant - -#if defined(__clang__) && (__clang_major__ >= 7) -#pragma GCC diagnostic pop -#endif - -} // namespace desul -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_SYCL.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_SYCL.hpp deleted file mode 100644 index 6c8c685110..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_SYCL.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_SYCL_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_SYCL_HPP_ - -// clang-format off -#include "desul/atomics/SYCLConversions.hpp" -#include "desul/atomics/Common.hpp" - -#include -// clang-format on - -#ifdef DESUL_HAVE_SYCL_ATOMICS - -namespace desul { - -template -inline void atomic_thread_fence(MemoryOrder, MemoryScope) { - sycl::atomic_fence( - Impl::DesulToSYCLMemoryOrder::value, - Impl::DesulToSYCLMemoryScope::value); -} - -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrder, MemoryScope) { - static_assert(sizeof(unsigned int) == 4, - "this function assumes an unsigned int is 32-bit"); - Impl::sycl_atomic_ref dest_ref( - *reinterpret_cast(dest)); - dest_ref.compare_exchange_strong(*reinterpret_cast(&compare), - *reinterpret_cast(&value)); - return compare; -} -template -typename std::enable_if::type atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrder, MemoryScope) { - static_assert(sizeof(unsigned long long int) == 8, - "this function assumes an unsigned long long is 64-bit"); - Impl::sycl_atomic_ref dest_ref( - *reinterpret_cast(dest)); - dest_ref.compare_exchange_strong(*reinterpret_cast(&compare), - *reinterpret_cast(&value)); - return compare; -} - -template -typename std::enable_if::type atomic_exchange(T* const dest, - T value, - MemoryOrder, - MemoryScope) { - static_assert(sizeof(unsigned int) == 4, - "this function assumes an unsigned int is 32-bit"); - Impl::sycl_atomic_ref dest_ref( - *reinterpret_cast(dest)); - unsigned int return_val = dest_ref.exchange(*reinterpret_cast(&value)); - return reinterpret_cast(return_val); -} -template -typename std::enable_if::type atomic_exchange(T* const dest, - T value, - MemoryOrder, - MemoryScope) { - static_assert(sizeof(unsigned long long int) == 8, - "this function assumes an unsigned long long is 64-bit"); - Impl::sycl_atomic_ref dest_ref( - *reinterpret_cast(dest)); - unsigned long long int return_val = - dest_ref.exchange(reinterpret_cast(value)); - return reinterpret_cast(return_val); -} - -template -typename std::enable_if<(sizeof(T) != 8) && (sizeof(T) != 4), T>::type -atomic_compare_exchange( - T* const /*dest*/, T compare, T /*value*/, MemoryOrder, MemoryScope) { - // FIXME_SYCL not implemented - assert(false); - return compare; -} - -template -typename std::enable_if<(sizeof(T) != 8) && (sizeof(T) != 4), T>::type atomic_exchange( - T* const /*dest*/, T value, MemoryOrder, MemoryScope) { - // FIXME_SYCL not implemented - assert(false); - return value; -} - -} // namespace desul - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_Serial.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_Serial.hpp deleted file mode 100644 index 9d0db5c9e1..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Compare_Exchange_Serial.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ -#ifndef DESUL_ATOMICS_COMPARE_EXCHANGE_SERIAL_HPP_ -#define DESUL_ATOMICS_COMPARE_EXCHANGE_SERIAL_HPP_ - -#ifdef DESUL_HAVE_SERIAL_ATOMICS -namespace desul { -template -void atomic_thread_fence(MemoryOrderAcquire, MemoryScope) {} - -template -void atomic_thread_fence(MemoryOrderRelease, MemoryScope) {} - -template -T atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderRelaxed, MemoryScope) { - T old = *dest; - if (old == compare) { - *dest = value; - } else { - old = compare; - } - return compare; -} -template -T atomic_compare_exchange( - T* const dest, T compare, T value, MemoryOrderSeqCst, MemoryScope) { - T old = *dest; - if (old == compare) { - *dest = value; - } else { - old = compare; - } - return compare; -} -} // namespace desul -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/GCC.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/GCC.hpp deleted file mode 100644 index 239c84fd30..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/GCC.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ -#ifndef DESUL_ATOMICS_GCC_HPP_ -#define DESUL_ATOMICS_GCC_HPP_ - -#ifdef DESUL_HAVE_GCC_ATOMICS - -#include -/* -Built - in Function : type __atomic_add_fetch(type * ptr, type val, int memorder) -Built - in Function : type __atomic_sub_fetch(type * ptr, type val, int memorder) -Built - in Function : type __atomic_and_fetch(type * ptr, type val, int memorder) -Built - in Function : type __atomic_xor_fetch(type * ptr, type val, int memorder) -Built - in Function : type __atomic_or_fetch(type * ptr, type val, int memorder) -Built - in Function : type __atomic_nand_fetch(type * ptr, type val, int memorder) -*/ - -#define DESUL_GCC_INTEGRAL_OP_ATOMICS(MEMORY_ORDER, MEMORY_SCOPE) \ - template \ - typename std::enable_if::value, T>::type atomic_fetch_add( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_fetch_add(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_fetch_sub( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_fetch_sub(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_fetch_and( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_fetch_and(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_fetch_or( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_fetch_or(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_fetch_xor( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_fetch_xor(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_fetch_nand( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_fetch_nand(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_add_fetch( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_add_fetch(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_sub_fetch( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_sub_fetch(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_and_fetch( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_and_fetch(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_or_fetch( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_or_fetch(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_xor_fetch( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_xor_fetch(dest, value, GCCMemoryOrder::value); \ - } \ - template \ - typename std::enable_if::value, T>::type atomic_nand_fetch( \ - T* const dest, T value, MEMORY_ORDER, MEMORY_SCOPE) { \ - return __atomic_nand_fetch(dest, value, GCCMemoryOrder::value); \ - } - -namespace desul { -DESUL_GCC_INTEGRAL_OP_ATOMICS(MemoryOrderRelaxed, MemoryScopeNode) -DESUL_GCC_INTEGRAL_OP_ATOMICS(MemoryOrderRelaxed, MemoryScopeDevice) -DESUL_GCC_INTEGRAL_OP_ATOMICS(MemoryOrderRelaxed, MemoryScopeCore) -DESUL_GCC_INTEGRAL_OP_ATOMICS(MemoryOrderSeqCst, MemoryScopeNode) -DESUL_GCC_INTEGRAL_OP_ATOMICS(MemoryOrderSeqCst, MemoryScopeDevice) -DESUL_GCC_INTEGRAL_OP_ATOMICS(MemoryOrderSeqCst, MemoryScopeCore) - -template -std::enable_if_t::value, T> atomic_exchange( - T* const dest, - Impl::dont_deduce_this_parameter_t val, - MemoryOrder /*order*/, - MemoryScope scope) { - // Acquire a lock for the address - // clang-format off - while (!Impl::lock_address((void*)dest, scope)) {} - // clang-format on - - atomic_thread_fence(MemoryOrderAcquire(), scope); - T return_val = *dest; - *dest = val; - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address((void*)dest, scope); - return return_val; -} - -template -std::enable_if_t::value, T> -atomic_compare_exchange(T* const dest, - Impl::dont_deduce_this_parameter_t compare, - Impl::dont_deduce_this_parameter_t val, - MemoryOrder /*order*/, - MemoryScope scope) { - // Acquire a lock for the address - // clang-format off - while (!Impl::lock_address((void*)dest, scope)) {} - // clang-format on - - atomic_thread_fence(MemoryOrderAcquire(), scope); - T return_val = *dest; - if (return_val == compare) { - *dest = val; - atomic_thread_fence(MemoryOrderRelease(), scope); - } - Impl::unlock_address((void*)dest, scope); - return return_val; -} -} // namespace desul -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Generic.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Generic.hpp deleted file mode 100644 index 1fffd3b293..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Generic.hpp +++ /dev/null @@ -1,766 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_GENERIC_HPP_ -#define DESUL_ATOMICS_GENERIC_HPP_ - -#include -#if defined(__GNUC__) && (!defined(__clang__)) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif -#include "desul/atomics/Common.hpp" -#include "desul/atomics/Compare_Exchange.hpp" -#include "desul/atomics/Lock_Array.hpp" -#include "desul/atomics/Macros.hpp" -// Combination operands to be used in an Compare and Exchange based atomic -// operation -namespace desul { -namespace Impl { - -template -struct MaxOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return (val1 > val2 ? val1 : val2); - } - DESUL_FORCEINLINE_FUNCTION - static constexpr bool check_early_exit(Scalar1 const& val1, Scalar2 const& val2) { - return val1 > val2; - } -}; - -template -struct MinOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return (val1 < val2 ? val1 : val2); - } - DESUL_FORCEINLINE_FUNCTION - static constexpr bool check_early_exit(Scalar1 const& val1, Scalar2 const& val2) { - return val1 < val2; - } -}; - -template -struct may_exit_early : std::false_type {}; - -// This exit early optimization causes weird compiler errors with MSVC 2019 -#ifndef DESUL_HAVE_MSVC_ATOMICS -template -struct may_exit_early(), - std::declval()))> - : std::true_type {}; -#endif - -template -constexpr DESUL_FUNCTION - typename std::enable_if::value, bool>::type - check_early_exit(Op const&, Scalar1 const& val1, Scalar2 const& val2) { - return Op::check_early_exit(val1, val2); -} - -template -constexpr DESUL_FUNCTION - typename std::enable_if::value, bool>::type - check_early_exit(Op const&, Scalar1 const&, Scalar2 const&) { - return false; -} - -template -struct AddOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 + val2; } -}; - -template -struct SubOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 - val2; } -}; - -template -struct MulOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 * val2; } -}; - -template -struct DivOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 / val2; } -}; - -template -struct ModOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 % val2; } -}; - -template -struct AndOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 & val2; } -}; - -template -struct OrOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 | val2; } -}; - -template -struct XorOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { return val1 ^ val2; } -}; - -template -struct NandOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return ~(val1 & val2); - } -}; - -template -struct LShiftOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 << val2; - } -}; - -template -struct RShiftOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return val1 >> val2; - } -}; - -template -struct IncModOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return ((val1 >= val2) ? Scalar1(0) : val1 + Scalar1(1)); - } -}; - -template -struct DecModOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { - return (((val1 == Scalar1(0)) | (val1 > val2)) ? val2 : (val1 - Scalar1(1))); - } -}; - -template -struct StoreOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1&, const Scalar2& val2) { return val2; } -}; - -template -struct LoadOper { - DESUL_FORCEINLINE_FUNCTION - static Scalar1 apply(const Scalar1& val1, const Scalar2&) { return val1; } -}; - -template = 0> -DESUL_INLINE_FUNCTION T atomic_fetch_oper(const Oper& op, - T* const dest, - dont_deduce_this_parameter_t val, - MemoryOrder order, - MemoryScope scope) { - using cas_t = typename atomic_compare_exchange_type::type; - cas_t oldval = reinterpret_cast(*dest); - cas_t assume = oldval; - - do { - if (Impl::check_early_exit(op, reinterpret_cast(oldval), val)) - return reinterpret_cast(oldval); - assume = oldval; - T newval = op.apply(reinterpret_cast(assume), val); - oldval = desul::atomic_compare_exchange(reinterpret_cast(dest), - assume, - reinterpret_cast(newval), - order, - scope); - } while (assume != oldval); - - return reinterpret_cast(oldval); -} - -template = 0> -DESUL_INLINE_FUNCTION T atomic_oper_fetch(const Oper& op, - T* const dest, - dont_deduce_this_parameter_t val, - MemoryOrder order, - MemoryScope scope) { - using cas_t = typename atomic_compare_exchange_type::type; - cas_t oldval = reinterpret_cast(*dest); - T newval = val; - cas_t assume = oldval; - do { - if (Impl::check_early_exit(op, reinterpret_cast(oldval), val)) - return reinterpret_cast(oldval); - assume = oldval; - newval = op.apply(reinterpret_cast(assume), val); - oldval = desul::atomic_compare_exchange(reinterpret_cast(dest), - assume, - reinterpret_cast(newval), - order, - scope); - } while (assume != oldval); - - return newval; -} - -template = 0> -DESUL_INLINE_FUNCTION T atomic_fetch_oper(const Oper& op, - T* const dest, - dont_deduce_this_parameter_t val, - MemoryOrder /*order*/, - MemoryScope scope) { -#if defined(DESUL_HAVE_FORWARD_PROGRESS) - // Acquire a lock for the address - while (!Impl::lock_address((void*)dest, scope)) { - } - - atomic_thread_fence(MemoryOrderAcquire(), scope); - T return_val = *dest; - *dest = op.apply(return_val, val); - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address((void*)dest, scope); - return return_val; -#elif defined(DESUL_HAVE_GPU_LIKE_PROGRESS) - // This is a way to avoid dead lock in a warp or wave front - T return_val; - int done = 0; -#ifdef __HIPCC__ - unsigned long long int active = DESUL_IMPL_BALLOT_MASK(1); - unsigned long long int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip((void*)dest, scope)) { - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = *dest; - *dest = op.apply(return_val, val); - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address_hip((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(done); - } - return return_val; -// FIXME_SYCL not implemented -#elif defined(__SYCL_DEVICE_ONLY__) - (void)op; - (void)dest; - (void)scope; - (void)return_val; - (void)done; - - assert(false); - return val; -#else - unsigned int mask = DESUL_IMPL_ACTIVEMASK; - unsigned int active = DESUL_IMPL_BALLOT_MASK(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda((void*)dest, scope)) { - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = *dest; - *dest = op.apply(return_val, val); - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address_cuda((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(mask, done); - } - return return_val; -#endif -#else - static_assert(false, "Unimplemented lock based atomic\n"); - return val; -#endif -} - -template = 0> -DESUL_INLINE_FUNCTION T atomic_oper_fetch(const Oper& op, - T* const dest, - dont_deduce_this_parameter_t val, - MemoryOrder /*order*/, - MemoryScope scope) { -#if defined(DESUL_HAVE_FORWARD_PROGRESS) - // Acquire a lock for the address - while (!Impl::lock_address((void*)dest, scope)) { - } - - atomic_thread_fence(MemoryOrderAcquire(), scope); - T return_val = op.apply(*dest, val); - *dest = return_val; - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address((void*)dest, scope); - return return_val; -#elif defined(DESUL_HAVE_GPU_LIKE_PROGRESS) - // This is a way to avoid dead lock in a warp or wave front - T return_val; - int done = 0; -#ifdef __HIPCC__ - unsigned long long int active = DESUL_IMPL_BALLOT_MASK(1); - unsigned long long int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_hip((void*)dest, scope)) { - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = op.apply(*dest, val); - *dest = return_val; - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address_hip((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(done); - } - return return_val; - // FIXME_SYCL not implemented -#elif defined(__SYCL_DEVICE_ONLY__) - (void)op; - (void)dest; - (void)scope; - (void)done; - - assert(false); - return val; -#else - unsigned int mask = DESUL_IMPL_ACTIVEMASK; - unsigned int active = DESUL_IMPL_BALLOT_MASK(mask, 1); - unsigned int done_active = 0; - while (active != done_active) { - if (!done) { - if (Impl::lock_address_cuda((void*)dest, scope)) { - atomic_thread_fence(MemoryOrderAcquire(), scope); - return_val = op.apply(*dest, val); - *dest = return_val; - atomic_thread_fence(MemoryOrderRelease(), scope); - Impl::unlock_address_cuda((void*)dest, scope); - done = 1; - } - } - done_active = DESUL_IMPL_BALLOT_MASK(mask, done); - } - return return_val; -#endif -#else - static_assert(false, "Unimplemented lock based atomic\n"); - return val; -#endif -} - -template -DESUL_INLINE_FUNCTION T atomic_fetch_oper(const Oper& op, - T* const dest, - dont_deduce_this_parameter_t val, - MemoryOrder /*order*/, - MemoryScopeCaller /*scope*/) { - T oldval = *dest; - *dest = op.apply(oldval, val); - return oldval; -} - -template -DESUL_INLINE_FUNCTION T atomic_oper_fetch(const Oper& op, - T* const dest, - dont_deduce_this_parameter_t val, - MemoryOrder /*order*/, - MemoryScopeCaller /*scope*/) { - T oldval = *dest; - T newval = op.apply(oldval, val); - *dest = newval; - return newval; -} - -} // namespace Impl -} // namespace desul - -namespace desul { - -// Fetch_Oper atomics: return value before operation -template -DESUL_INLINE_FUNCTION T -atomic_fetch_add(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::AddOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_sub(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::SubOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_max(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::MaxOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_min(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::MinOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_mul(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::MulOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_div(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::DivOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_mod(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::ModOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_and(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::AndOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_or(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::OrOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_xor(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::XorOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_nand(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_fetch_oper(Impl::NandOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_fetch_lshift(T* const dest, - const unsigned int val, - MemoryOrder order, - MemoryScope scope) { - return Impl::atomic_fetch_oper( - Impl::LShiftOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_fetch_rshift(T* const dest, - const unsigned int val, - MemoryOrder order, - MemoryScope scope) { - return Impl::atomic_fetch_oper( - Impl::RShiftOper(), dest, val, order, scope); -} - -// Oper Fetch atomics: return value after operation -template -DESUL_INLINE_FUNCTION T -atomic_add_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::AddOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_sub_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::SubOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_max_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::MaxOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_min_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::MinOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_mul_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::MulOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_div_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::DivOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_mod_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::ModOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_and_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::AndOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_or_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::OrOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_xor_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::XorOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_nand_fetch(T* const dest, const T val, MemoryOrder order, MemoryScope scope) { - return Impl::atomic_oper_fetch(Impl::NandOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_lshift_fetch(T* const dest, - const unsigned int val, - MemoryOrder order, - MemoryScope scope) { - return Impl::atomic_oper_fetch( - Impl::LShiftOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_rshift_fetch(T* const dest, - const unsigned int val, - MemoryOrder order, - MemoryScope scope) { - return Impl::atomic_oper_fetch( - Impl::RShiftOper(), dest, val, order, scope); -} - -// Other atomics - -template -DESUL_INLINE_FUNCTION T atomic_load(const T* const dest, - MemoryOrder order, - MemoryScope scope) { - return Impl::atomic_fetch_oper( - Impl::LoadOper(), const_cast(dest), T(), order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_store(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)Impl::atomic_fetch_oper(Impl::StoreOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_add(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)atomic_fetch_add(dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_sub(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)atomic_fetch_sub(dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_mul(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)atomic_fetch_mul(dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_div(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)atomic_fetch_div(dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_min(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)atomic_fetch_min(dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_max(T* const dest, - const T val, - MemoryOrder order, - MemoryScope scope) { - (void)atomic_fetch_max(dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_inc_fetch(T* const dest, - MemoryOrder order, - MemoryScope scope) { - return atomic_add_fetch(dest, T(1), order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_dec_fetch(T* const dest, - MemoryOrder order, - MemoryScope scope) { - return atomic_sub_fetch(dest, T(1), order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_fetch_inc(T* const dest, - MemoryOrder order, - MemoryScope scope) { - return atomic_fetch_add(dest, T(1), order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_inc_mod(T* const dest, T val, MemoryOrder order, MemoryScope scope) { - static_assert(std::is_unsigned::value, - "Signed types not supported by atomic_fetch_inc_mod."); - return Impl::atomic_fetch_oper( - Impl::IncModOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION T atomic_fetch_dec(T* const dest, - MemoryOrder order, - MemoryScope scope) { - return atomic_fetch_sub(dest, T(1), order, scope); -} - -template -DESUL_INLINE_FUNCTION T -atomic_fetch_dec_mod(T* const dest, T val, MemoryOrder order, MemoryScope scope) { - static_assert(std::is_unsigned::value, - "Signed types not supported by atomic_fetch_dec_mod."); - return Impl::atomic_fetch_oper( - Impl::DecModOper(), dest, val, order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_inc(T* const dest, - MemoryOrder order, - MemoryScope scope) { - return atomic_add(dest, T(1), order, scope); -} - -template -DESUL_INLINE_FUNCTION void atomic_dec(T* const dest, - MemoryOrder order, - MemoryScope scope) { - return atomic_sub(dest, T(1), order, scope); -} - -// FIXME -template -DESUL_INLINE_FUNCTION bool atomic_compare_exchange_strong( - T* const dest, - T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder /*failure*/, - MemoryScope scope) { - T const old = atomic_compare_exchange(dest, expected, desired, success, scope); - if (old != expected) { - expected = old; - return false; - } else { - return true; - } -} - -template -DESUL_INLINE_FUNCTION bool atomic_compare_exchange_weak(T* const dest, - T& expected, - T desired, - SuccessMemoryOrder success, - FailureMemoryOrder failure, - MemoryScope scope) { - return atomic_compare_exchange_strong( - dest, expected, desired, success, failure, scope); -} - -} // namespace desul - -#include -#include -#include -#include -#include -#if defined(__GNUC__) && (!defined(__clang__)) -#pragma GCC diagnostic pop -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/HIP.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/HIP.hpp deleted file mode 100644 index e51406e54d..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/HIP.hpp +++ /dev/null @@ -1,222 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ -#ifndef DESUL_ATOMICS_HIP_HPP_ -#define DESUL_ATOMICS_HIP_HPP_ - -#ifdef __HIP_DEVICE_COMPILE__ -namespace desul { - -// header file is organized as follows: -// 1/ device-side overload set from atomic functions provided by HIP -// 2/ fallback implementation on host-side for atomic functions defined in 1/ that are -// not included in the GCC overload set -// 3/ fallback implementation on device-side for atomic functions from the GCC -// overload set that are not defined in 1/ - -// clang-format off -inline __device__ int atomic_fetch_add( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, val); } -inline __device__ unsigned int atomic_fetch_add( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, val); } -inline __device__ unsigned long long atomic_fetch_add(unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, val); } -inline __device__ float atomic_fetch_add( float* ptr, float val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, val); } -inline __device__ double atomic_fetch_add( double* ptr, double val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, val); } - -inline __device__ int atomic_fetch_sub( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicSub(ptr, val); } -inline __device__ unsigned int atomic_fetch_sub( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicSub(ptr, val); } -inline __device__ unsigned long long atomic_fetch_sub(unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, -val); } -inline __device__ float atomic_fetch_sub( float* ptr, float val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, -val); } -inline __device__ double atomic_fetch_sub( double* ptr, double val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, -val); } - -inline __device__ int atomic_fetch_min( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicMin(ptr, val); } -inline __device__ unsigned int atomic_fetch_min( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicMin(ptr, val); } -inline __device__ unsigned long long atomic_fetch_min(unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicMin(ptr, val); } - -inline __device__ int atomic_fetch_max( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicMax(ptr, val); } -inline __device__ unsigned int atomic_fetch_max( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicMax(ptr, val); } -inline __device__ unsigned long long atomic_fetch_max(unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicMax(ptr, val); } - -inline __device__ int atomic_fetch_and( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAnd(ptr, val); } -inline __device__ unsigned int atomic_fetch_and( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAnd(ptr, val); } -inline __device__ unsigned long long atomic_fetch_and(unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAnd(ptr, val); } - -inline __device__ int atomic_fetch_or ( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicOr (ptr, val); } -inline __device__ unsigned int atomic_fetch_or ( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicOr (ptr, val); } -inline __device__ unsigned long long atomic_fetch_or (unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicOr (ptr, val); } - -inline __device__ int atomic_fetch_xor( int* ptr, int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicXor(ptr, val); } -inline __device__ unsigned int atomic_fetch_xor( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicXor(ptr, val); } -inline __device__ unsigned long long atomic_fetch_xor(unsigned long long* ptr, unsigned long long val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicXor(ptr, val); } - -inline __device__ int atomic_fetch_inc( int* ptr, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, 1 ); } -inline __device__ unsigned int atomic_fetch_inc( unsigned int* ptr, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, 1u ); } -inline __device__ unsigned long long atomic_fetch_inc(unsigned long long* ptr, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, 1ull); } - -inline __device__ int atomic_fetch_dec( int* ptr, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicSub(ptr, 1 ); } -inline __device__ unsigned int atomic_fetch_dec( unsigned int* ptr, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicSub(ptr, 1u ); } -inline __device__ unsigned long long atomic_fetch_dec(unsigned long long* ptr, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicAdd(ptr, -1 ); } - -inline __device__ unsigned int atomic_fetch_inc_mod( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicInc(ptr, val); } -inline __device__ unsigned int atomic_fetch_dec_mod( unsigned int* ptr, unsigned int val, MemoryOrderRelaxed, MemoryScopeDevice) { return atomicDec(ptr, val); } -// clang-format on - -#define DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(OP, TYPE) \ - template \ - inline __device__ TYPE atomic_fetch_##OP( \ - TYPE* ptr, TYPE val, MemoryOrder, MemoryScopeDevice) { \ - __threadfence(); \ - TYPE return_val = \ - atomic_fetch_##OP(ptr, val, MemoryOrderRelaxed(), MemoryScopeDevice()); \ - __threadfence(); \ - return return_val; \ - } \ - template \ - inline __device__ TYPE atomic_fetch_##OP( \ - TYPE* ptr, TYPE val, MemoryOrder, MemoryScopeCore) { \ - return atomic_fetch_##OP(ptr, val, MemoryOrder(), MemoryScopeDevice()); \ - } - -#define DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(OP) \ - DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(OP, int) \ - DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(OP, unsigned int) \ - DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(OP, unsigned long long) - -#define DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_FLOATING_POINT(OP) \ - DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(OP, float) \ - DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(OP, double) - -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(min) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(max) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(and) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(or) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(xor) - -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_FLOATING_POINT(add) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(add) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_FLOATING_POINT(sub) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(sub) - -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(inc) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL(dec) - -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(inc_mod, unsigned int) -DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP(dec_mod, unsigned int) - -#undef DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_FLOATING_POINT -#undef DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP_INTEGRAL -#undef DESUL_IMPL_HIP_DEVICE_ATOMIC_FETCH_OP - -// 2/ host-side fallback implementation for atomic functions not provided by GCC - -#define DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(OP_LOWERCASE, OP_PASCAL_CASE, TYPE) \ - template \ - inline __host__ TYPE atomic_fetch_##OP_LOWERCASE( \ - TYPE* ptr, TYPE val, MemoryOrder order, MemoryScopeDevice scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::OP_PASCAL_CASE##Oper(), ptr, val, order, scope); \ - } \ - template \ - inline __host__ TYPE atomic_fetch_##OP_LOWERCASE( \ - TYPE* ptr, TYPE val, MemoryOrder order, MemoryScopeCore scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::OP_PASCAL_CASE##Oper(), ptr, val, order, scope); \ - } - -#define DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_INTEGRAL(OP_LOWERCASE, OP_PASCAL_CASE) \ - DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(OP_LOWERCASE, OP_PASCAL_CASE, int) \ - DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(OP_LOWERCASE, OP_PASCAL_CASE, unsigned int) \ - DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN( \ - OP_LOWERCASE, OP_PASCAL_CASE, unsigned long long) - -#define DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_FLOATING_POINT(OP_LOWERCASE, \ - OP_PASCAL_CASE) \ - DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(OP_LOWERCASE, OP_PASCAL_CASE, float) \ - DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(OP_LOWERCASE, OP_PASCAL_CASE, double) - -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_INTEGRAL(min, Min) -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_INTEGRAL(max, Max) -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_FLOATING_POINT(add, Add) -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_FLOATING_POINT(sub, Sub) - -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(inc_mod, IncMod, unsigned int) -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN(dec_mod, DecMod, unsigned int) - -#undef DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_FLOATING_POINT -#undef DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN_INTEGRAL -#undef DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_FUN - -#define DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_INCREMENT_DECREMENT(TYPE) \ - template \ - inline __host__ TYPE atomic_fetch_inc( \ - TYPE* ptr, MemoryOrder order, MemoryScopeDevice scope) { \ - return atomic_fetch_add(ptr, static_cast(1), order, scope); \ - } \ - template \ - inline __host__ TYPE atomic_fetch_inc( \ - TYPE* ptr, MemoryOrder order, MemoryScopeCore scope) { \ - return atomic_fetch_add(ptr, static_cast(1), order, scope); \ - } \ - template \ - inline __host__ TYPE atomic_fetch_dec( \ - TYPE* ptr, MemoryOrder order, MemoryScopeDevice scope) { \ - return atomic_fetch_sub(ptr, static_cast(1), order, scope); \ - } \ - template \ - inline __host__ TYPE atomic_fetch_dec( \ - TYPE* ptr, MemoryOrder order, MemoryScopeCore scope) { \ - return atomic_fetch_sub(ptr, static_cast(1), order, scope); \ - } - -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_INCREMENT_DECREMENT(int) -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_INCREMENT_DECREMENT(unsigned int) -DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_INCREMENT_DECREMENT(unsigned long long) - -#undef DESUL_IMPL_HIP_HOST_FALLBACK_ATOMIC_INCREMENT_DECREMENT - -// 3/ device-side fallback implementation for atomic functions defined in GCC overload -// set - -#define DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE( \ - OP_LOWERCASE, OP_PASCAL_CASE, MEMORY_ORDER, MEMORY_SCOPE) \ - template \ - inline __device__ std::enable_if_t::value, T> \ - atomic_##OP_LOWERCASE##_fetch( \ - T* ptr, T val, MEMORY_ORDER order, MEMORY_SCOPE scope) { \ - return Impl::atomic_oper_fetch( \ - Impl::OP_PASCAL_CASE##Oper(), ptr, val, order, scope); \ - } \ - template \ - inline __device__ std::enable_if_t::value, T> \ - atomic_fetch_##OP_LOWERCASE( \ - T* ptr, T val, MEMORY_ORDER order, MEMORY_SCOPE scope) { \ - return Impl::atomic_fetch_oper( \ - Impl::OP_PASCAL_CASE##Oper(), ptr, val, order, scope); \ - } - -// clang-format off -#define DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(OP_LOWERCASE, OP_PASCAL_CASE) \ - DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE(OP_LOWERCASE, OP_PASCAL_CASE, MemoryOrderRelaxed, MemoryScopeNode) \ - DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE(OP_LOWERCASE, OP_PASCAL_CASE, MemoryOrderRelaxed, MemoryScopeDevice) \ - DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE(OP_LOWERCASE, OP_PASCAL_CASE, MemoryOrderRelaxed, MemoryScopeCore) \ - DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE(OP_LOWERCASE, OP_PASCAL_CASE, MemoryOrderSeqCst, MemoryScopeNode) \ - DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE(OP_LOWERCASE, OP_PASCAL_CASE, MemoryOrderSeqCst, MemoryScopeDevice) \ - DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE(OP_LOWERCASE, OP_PASCAL_CASE, MemoryOrderSeqCst, MemoryScopeCore) -// clang-format on - -DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(add, Add) -DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(sub, Sub) -DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(and, And) -DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(or, Or) -DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(xor, Xor) -DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN(nand, Nand) - -#undef DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN -#undef DESUL_IMPL_HIP_DEVICE_FALLBACK_ATOMIC_FUN_ORDER_SCOPE - -} // namespace desul - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Macros.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Macros.hpp deleted file mode 100644 index 0890b2dbc5..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/Macros.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_MACROS_HPP_ -#define DESUL_ATOMICS_MACROS_HPP_ - -// Macros - -#if (!defined(__CUDA_ARCH__) || !defined(__NVCC__)) && \ - (!defined(__HIP_DEVICE_COMPILE) || !defined(__HIP_PLATFORM_HCC__)) && \ - !defined(__SYCL_DEVICE_ONLY__) && !defined(DESUL_HAVE_OPENMP_ATOMICS) && \ - !defined(DESUL_HAVE_SERIAL_ATOMICS) -#define DESUL_IMPL_HAVE_GCC_OR_MSVC_ATOMICS -#endif - -// ONLY use GNUC atomics if not compiling for the device -// and we didn't explicitly say to use OPENMP or SERIAL atomics -#if defined(__GNUC__) && defined(DESUL_IMPL_HAVE_GCC_OR_MSVC_ATOMICS) -#define DESUL_HAVE_GCC_ATOMICS -#endif - -// Equivalent to above: if we are compiling for the device we -// need to use CUDA/HIP/SYCL atomics instead of MSVC atomics -#if defined(_MSC_VER) && defined(DESUL_IMPL_HAVE_GCC_OR_MSVC_ATOMICS) -#define DESUL_HAVE_MSVC_ATOMICS -#endif - -#undef DESUL_IMPL_HAVE_GCC_OR_MSVC_ATOMICS - -#ifdef __CUDACC__ -#define DESUL_HAVE_CUDA_ATOMICS -#endif - -#ifdef __HIPCC__ -#define DESUL_HAVE_HIP_ATOMICS -#endif - -#ifdef __SYCL_DEVICE_ONLY__ -#define DESUL_HAVE_SYCL_ATOMICS -#endif - -#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) || \ - defined(__SYCL_DEVICE_ONLY__) -#define DESUL_HAVE_GPU_LIKE_PROGRESS -#endif - -#if defined(DESUL_HAVE_CUDA_ATOMICS) || defined(DESUL_HAVE_HIP_ATOMICS) -#define DESUL_FORCEINLINE_FUNCTION inline __host__ __device__ -#define DESUL_INLINE_FUNCTION inline __host__ __device__ -#define DESUL_FUNCTION __host__ __device__ -#else -#define DESUL_FORCEINLINE_FUNCTION inline -#define DESUL_INLINE_FUNCTION inline -#define DESUL_FUNCTION -#endif - -#if !defined(DESUL_HAVE_GPU_LIKE_PROGRESS) -#define DESUL_HAVE_FORWARD_PROGRESS -#endif - -#endif // DESUL_ATOMICS_MACROS_HPP_ diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/OpenMP.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/OpenMP.hpp deleted file mode 100644 index bc6fb26c2f..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/OpenMP.hpp +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ -#ifndef DESUL_ATOMICS_OPENMP_HPP_ -#define DESUL_ATOMICS_OPENMP_HPP_ - -#ifdef DESUL_HAVE_OPENMP_ATOMICS - -#include -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/SYCLConversions.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/SYCLConversions.hpp deleted file mode 100644 index 7debf91d6d..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/SYCLConversions.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_SYCL_CONVERSIONS_HPP_ -#define DESUL_ATOMICS_SYCL_CONVERSIONS_HPP_ -#ifdef DESUL_HAVE_SYCL_ATOMICS - -// clang-format off -#include "desul/atomics/Common.hpp" - -#include -// clang-format on - -namespace desul { -namespace Impl { - -#ifdef __clang__ -namespace sycl_sync_and_atomics = ::sycl::ext::oneapi; -#else -namespace sycl_sync_and_atomics = ::sycl; -#endif - -template -using sycl_memory_order = std::conditional_t; -template -using sycl_memory_scope = std::conditional_t; - -template -struct DesulToSYCLMemoryOrder; -template -struct DesulToSYCLMemoryOrder { - static constexpr sycl_memory_order value = - sycl_memory_order::seq_cst; -}; -template -struct DesulToSYCLMemoryOrder { - static constexpr sycl_memory_order value = - sycl_memory_order::acquire; -}; -template -struct DesulToSYCLMemoryOrder { - static constexpr sycl_memory_order value = - sycl_memory_order::release; -}; -template -struct DesulToSYCLMemoryOrder { - static constexpr sycl_memory_order value = - sycl_memory_order::acq_rel; -}; -template -struct DesulToSYCLMemoryOrder { - static constexpr sycl_memory_order value = - sycl_memory_order::relaxed; -}; - -template -struct DesulToSYCLMemoryScope; -template -struct DesulToSYCLMemoryScope { - static constexpr sycl_memory_scope value = - sycl_memory_scope::work_group; -}; -template -struct DesulToSYCLMemoryScope { - static constexpr sycl_memory_scope value = - sycl_memory_scope::device; -}; -template -struct DesulToSYCLMemoryScope { - static constexpr sycl_memory_scope value = - sycl_memory_scope::system; -}; - -// FIXME_SYCL generic_space isn't available yet for CUDA. -#ifdef __NVPTX__ -template -using sycl_atomic_ref = sycl::atomic_ref::value, - DesulToSYCLMemoryScope::value, - sycl::access::address_space::global_space>; -#else -template -using sycl_atomic_ref = sycl::atomic_ref::value, - DesulToSYCLMemoryScope::value, - sycl::access::address_space::generic_space>; -#endif -} // namespace Impl -} // namespace desul - -#endif -#endif diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/CUDA_asm.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/CUDA_asm.hpp deleted file mode 100644 index 461d3e0928..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/CUDA_asm.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -namespace desul { -#if defined(__CUDA_ARCH__) || (defined(__clang__) && !defined(__NVCC__)) -// Choose the variant of atomics we are using later -#if !defined(DESUL_IMPL_ATOMIC_CUDA_PTX_GENERIC) && \ - !defined(DESUL_IMPL_ATOMIC_CUDA_PTX_PREDICATE) && \ - !defined(DESUL_IMPL_ATOMIC_CUDA_PTX_ISGLOBAL) && \ - !defined(DESUL_IMPL_ATOMIC_CUDA_PTX_FORCEGLOBAL) -#if (__CUDACC_VER_MAJOR__ > 11) || ((__CUDACC_VER_MAJOR__==11) && (__CUDACC_VER_MINOR__>1)) -#define DESUL_IMPL_ATOMIC_CUDA_PTX_ISGLOBAL -#else -#define DESUL_IMPL_ATOMIC_CUDA_PTX_PREDICATE -#endif -#endif -#include - -#endif -} diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/CUDA_asm_exchange.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/CUDA_asm_exchange.hpp deleted file mode 100644 index 0ab95e6a00..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/CUDA_asm_exchange.hpp +++ /dev/null @@ -1,8 +0,0 @@ -#include -namespace desul { -#if defined(__CUDA_ARCH__) || (defined(__clang__) && !defined(__NVCC__)) - -#include - -#endif -} diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc deleted file mode 100644 index 6de590a952..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc +++ /dev/null @@ -1,18 +0,0 @@ -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_GENERIC -#include "cuda_cc7_asm_atomic_fetch_op.inc_generic" -#endif - -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_ISGLOBAL -#include "cuda_cc7_asm_atomic_fetch_op.inc_isglobal" -#endif - -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_PREDICATE -#include "cuda_cc7_asm_atomic_fetch_op.inc_predicate" -#endif - -// This version is not generally safe -// Only here for performance comparison purposes -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_FORCEGLOBAL -#include "cuda_cc7_asm_atomic_fetch_op.inc_forceglobal" -#endif - diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc_forceglobal b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc_forceglobal deleted file mode 100644 index b235163820..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc_forceglobal +++ /dev/null @@ -1,153 +0,0 @@ - -// Inline PTX: h u16 , r u32, l u64, f f32, d f64 -// Ops: - -// binary operations - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND() \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_and(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint32_t asm_value = reinterpret_cast(value); \ - uint32_t asm_result = 0u; \ - asm volatile("atom.and.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;" : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_and(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint64_t asm_value = reinterpret_cast(value); \ - uint64_t asm_result = 0u; \ - asm volatile("atom.and.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;" : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_OR() \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_or(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint32_t asm_value = reinterpret_cast(value); \ - uint32_t asm_result = 0u; \ - asm volatile("atom.or.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;" : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_or(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint64_t asm_value = reinterpret_cast(value); \ - uint64_t asm_result = 0u; \ - asm volatile("atom.or.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;" : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_XOR() \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_xor(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint32_t asm_value = reinterpret_cast(value); \ - uint32_t asm_result = 0u; \ - asm volatile("atom.xor.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;" : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_xor(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint64_t asm_value = reinterpret_cast(value); \ - uint64_t asm_result = 0u; \ - asm volatile("atom.xor.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;" : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} - -// Fetch atomics -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_add(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - asm volatile("atom.add.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_sub(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - ctype neg_value = -value; \ - asm volatile("atom.add.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(neg_value) : "memory"); \ - return result; \ -} - - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_min(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - asm volatile("atom.min.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_max(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - asm volatile("atom.max.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_inc(ctype* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - ctype limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("atom.inc.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} \ -inline __device__ ctype atomic_fetch_inc_mod(ctype* dest, ctype limit, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - asm volatile("atom.inc.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_dec(ctype* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - ctype limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("atom.dec.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} \ -inline __device__ ctype atomic_fetch_dec_mod(ctype* dest, ctype limit, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - asm volatile("atom.dec.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} - -// Group ops for integer ctypes -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_BIN_OP() \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND() \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_OR() \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_XOR() - - -// Instantiate Functions -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(float,".f32","f","=f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(float,".f32","f","=f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(double,".f64","d","=d") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(double,".f64","d","=d") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(uint32_t,".u32","r","=r") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(uint64_t,".u64","l","=l") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(int32_t,".s32","r","=r") -//__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(int64_t,".s64","l","=l") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC(uint32_t,".u32","r","=r") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC(uint32_t,".u32","r","=r") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_BIN_OP() - -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND - diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc_generic b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc_generic deleted file mode 100644 index 0484d109c3..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_fetch_op.inc_generic +++ /dev/null @@ -1,151 +0,0 @@ - -// Inline PTX: h u16 , r u32, l u64, f f32, d f64 -// Ops: - -// binary operations - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND() \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_and(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint32_t asm_value = reinterpret_cast(value); \ - uint32_t asm_result = 0u; \ - asm volatile("atom.and" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;" : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_and(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint64_t asm_value = reinterpret_cast(value); \ - uint64_t asm_result = 0u; \ - asm volatile("atom.and" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;" : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_OR() \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_or(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint32_t asm_value = reinterpret_cast(value); \ - uint32_t asm_result = 0u; \ - asm volatile("atom.or" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;" : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_or(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint64_t asm_value = reinterpret_cast(value); \ - uint64_t asm_result = 0u; \ - asm volatile("atom.or" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;" : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_XOR() \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_xor(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint32_t asm_value = reinterpret_cast(value); \ - uint32_t asm_result = 0u; \ - asm volatile("atom.xor" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;" : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} \ -template \ -inline __device__ typename std::enable_if::type atomic_fetch_xor(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - uint64_t asm_value = reinterpret_cast(value); \ - uint64_t asm_result = 0u; \ - asm volatile("atom.xor" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;" : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \ - return reinterpret_cast(asm_result); \ -} - -// Fetch atomics -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_add(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - asm volatile("atom.add" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_sub(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - ctype neg_value = -value; \ - asm volatile("atom.add" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(neg_value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_min(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - asm volatile("atom.min" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_max(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result=0; \ - asm volatile("atom.max" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_inc(ctype* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - ctype limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("atom.inc" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} \ -inline __device__ ctype atomic_fetch_inc_mod(ctype* dest, ctype limit, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - asm volatile("atom.inc" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -inline __device__ ctype atomic_fetch_dec(ctype* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - ctype limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("atom.dec" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} \ -inline __device__ ctype atomic_fetch_dec_mod(ctype* dest, ctype limit, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - ctype result = 0; \ - asm volatile("atom.dec" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;" : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \ - return result; \ -} -// Group ops for integer ctypes -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_BIN_OP() \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND() \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_OR() \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_XOR() - - -// Instantiate Functions -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(float,".f32","f","=f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(float,".f32","f","=f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(double,".f64","d","=d") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(double,".f64","d","=d") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(uint32_t,".u32","r","=r") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(uint64_t,".u64","l","=l") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(int32_t,".s32","r","=r") -//__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(int64_t,".s64","l","=l") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC(uint32_t,".u32","r","=r") -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC(uint32_t,".u32","r","=r") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_BIN_OP() - -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC -#undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND - diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc deleted file mode 100644 index ca02410515..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc +++ /dev/null @@ -1,18 +0,0 @@ -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_GENERIC -#include "cuda_cc7_asm_atomic_op.inc_generic" -#endif - -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_ISGLOBAL -#include "cuda_cc7_asm_atomic_op.inc_isglobal" -#endif - -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_PREDICATE -#include "cuda_cc7_asm_atomic_op.inc_predicate" -#endif - -// This version is not generally safe -// Only here for performance comparison purposes -#ifdef DESUL_IMPL_ATOMIC_CUDA_PTX_FORCEGLOBAL -#include "cuda_cc7_asm_atomic_op.inc_forceglobal" -#endif - diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc_forceglobal b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc_forceglobal deleted file mode 100644 index 3767b2ab49..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc_forceglobal +++ /dev/null @@ -1,64 +0,0 @@ - -// Inline PTX: h u16 , r u32, l u64, f f32, d f64 -// Ops: - -// Non Returning Atomic Operations -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(type,asm_type,reg_type) \ -inline __device__ void atomic_add(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - asm volatile("red.add.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(type,asm_type,reg_type) \ -inline __device__ void atomic_sub(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - type neg_value = -value; \ - asm volatile("red.add.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(neg_value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_MIN(type,asm_type,reg_type) \ -inline __device__ void atomic_min(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - asm volatile("red.min.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_MAX(type,asm_type,reg_type) \ -inline __device__ void atomic_max(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - asm volatile("red.max.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_INC(type,asm_type,reg_type) \ -inline __device__ void atomic_inc(type* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - type limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("red.inc.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(limit) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_DEC(type,asm_type,reg_type) \ -inline __device__ void atomic_dec(type* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - type limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("red.dec.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(limit) : "memory"); \ -} - -// Group ops for integer types -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MIN(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MAX(type,asm_type,reg_type) - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_UNSIGNED_OP(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MIN(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MAX(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_INC(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_DEC(type,asm_type,reg_type) - -// Instantiate Functions -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(float,".f32","f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(float,".f32","f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(double,".f64","d") -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(double,".f64","d") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_UNSIGNED_OP(uint32_t,".u32","r") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(int64_t,".u64","l") -__DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(int32_t,".s32","r") -//__DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(int64_t,".s64","l") diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc_generic b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc_generic deleted file mode 100644 index 5de36a3e0a..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/cuda/cuda_cc7_asm_atomic_op.inc_generic +++ /dev/null @@ -1,64 +0,0 @@ - -// Inline PTX: h u16 , r u32, l u64, f f32, d f64 -// Ops: - -// Non Returning Atomic Operations -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(type,asm_type,reg_type) \ -inline __device__ void atomic_add(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - asm volatile("red.add" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(type,asm_type,reg_type) \ -inline __device__ void atomic_sub(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - type neg_value = -value; \ - asm volatile("red.add" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(neg_value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_MIN(type,asm_type,reg_type) \ -inline __device__ void atomic_min(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - asm volatile("red.min" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_MAX(type,asm_type,reg_type) \ -inline __device__ void atomic_max(type* dest, type value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - asm volatile("red.max" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(value) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_INC(type,asm_type,reg_type) \ -inline __device__ void atomic_inc(type* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - type limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("red.inc" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(limit) : "memory"); \ -} - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_DEC(type,asm_type,reg_type) \ -inline __device__ void atomic_dec(type* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \ - type limit = desul::Impl::numeric_limits_max::value; \ - asm volatile("red.dec" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_type " [%0],%1;" :: "l"(dest),reg_type(limit) : "memory"); \ -} - -// Group ops for integer types -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MIN(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MAX(type,asm_type,reg_type) - -#define __DESUL_IMPL_CUDA_ASM_ATOMIC_UNSIGNED_OP(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MIN(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_MAX(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_INC(type,asm_type,reg_type) \ -__DESUL_IMPL_CUDA_ASM_ATOMIC_DEC(type,asm_type,reg_type) - -// Instantiate Functions -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(float,".f32","f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(float,".f32","f") -__DESUL_IMPL_CUDA_ASM_ATOMIC_ADD(double,".f64","d") -__DESUL_IMPL_CUDA_ASM_ATOMIC_SUB(double,".f64","d") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_UNSIGNED_OP(uint32_t,".u32","r") - -__DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(int64_t,".u64","l") -__DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(int32_t,".s32","r") -//__DESUL_IMPL_CUDA_ASM_ATOMIC_INTEGER_OP(int64_t,".s64","l") diff --git a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/openmp/OpenMP_40.hpp b/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/openmp/OpenMP_40.hpp deleted file mode 100644 index f4f1bbd96e..0000000000 --- a/bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics/openmp/OpenMP_40.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright (c) 2019, Lawrence Livermore National Security, LLC -and DESUL project contributors. See the COPYRIGHT file for details. -Source: https://github.com/desul/desul - -SPDX-License-Identifier: (BSD-3-Clause) -*/ - -#ifndef DESUL_ATOMICS_OPENMP40_HPP_ -#define DESUL_ATOMICS_OPENMP40_HPP_ -#include - -namespace desul { -namespace Impl { - template - void openmp_maybe_call_pre_capture_flush(MEMORY_ORDER_TMP, MEMORY_SCOPE_TMP) {} - template - void openmp_maybe_call_pre_capture_flush(MemoryOrderAcquire, MEMORY_SCOPE_TMP) { - atomic_thread_fence(MemoryOrderAcquire(), MEMORY_SCOPE_TMP()); - } - template - void openmp_maybe_call_pre_capture_flush(MemoryOrderAcqRel, MEMORY_SCOPE_TMP) { - atomic_thread_fence(MemoryOrderAcqRel(), MEMORY_SCOPE_TMP()); - } - template - void openmp_maybe_call_pre_capture_flush(MemoryOrderSeqCst, MEMORY_SCOPE_TMP) { - atomic_thread_fence(MemoryOrderSeqCst(), MEMORY_SCOPE_TMP()); - } - - template - void openmp_maybe_call_post_capture_flush(MEMORY_ORDER_TMP, MEMORY_SCOPE_TMP) {} - template - void openmp_maybe_call_post_capture_flush(MemoryOrderRelease, MEMORY_SCOPE_TMP) { - atomic_thread_fence(MemoryOrderRelease(), MEMORY_SCOPE_TMP()); - } - template - void openmp_maybe_call_post_capture_flush(MemoryOrderAcqRel, MEMORY_SCOPE_TMP) { - atomic_thread_fence(MemoryOrderAcqRel(), MEMORY_SCOPE_TMP()); - } - template - void openmp_maybe_call_post_capture_flush(MemoryOrderSeqCst, MEMORY_SCOPE_TMP) { - atomic_thread_fence(MemoryOrderSeqCst(), MEMORY_SCOPE_TMP()); - } - - template - struct is_openmp_atomic_type_t { - static constexpr bool value = std::is_arithmetic::value; - }; - template - constexpr bool is_openmp_atomic_type_v = is_openmp_atomic_type_t::value; -} -} - -namespace desul { -// Can't use a macro approach to get all definitions since the ops include #pragma omp -// So gonna use multiple inclusion of the same code snippet here. - -// Can't do Node level atomics this way with OpenMP Target, but we could -// have a define which says whether or not Device level IS node level (e.g. for pure CPU node) - -#define MEMORY_ORDER MemoryOrderRelaxed -// #define MEMORY_SCOPE MemoryScopeNode -// #include -// #undef MEMORY_SCOPE -#define MEMORY_SCOPE MemoryScopeDevice -#include -#undef MEMORY_SCOPE -#define MEMORY_SCOPE MemoryScopeCore -#include -#undef MEMORY_SCOPE -#undef MEMORY_ORDER - -#define MEMORY_ORDER MemoryOrderAcqRel -// #define MEMORY_SCOPE MemoryScopeNode -// #include -// #undef MEMORY_SCOPE -#define MEMORY_SCOPE MemoryScopeDevice -#include -#undef MEMORY_SCOPE -#define MEMORY_SCOPE MemoryScopeCore -#include -#undef MEMORY_SCOPE -#undef MEMORY_ORDER - -#define MEMORY_ORDER MemoryOrderSeqCst -// #define MEMORY_SCOPE MemoryScopeNode -// #include -// #undef MEMORY_SCOPE -#define MEMORY_SCOPE MemoryScopeDevice -#include -#undef MEMORY_SCOPE -#define MEMORY_SCOPE MemoryScopeCore -#include -#undef MEMORY_SCOPE -#undef MEMORY_ORDER -} // namespace desul -#endif diff --git a/bundled/kokkos-4.5.01/CHANGELOG.md b/bundled/kokkos-4.5.01/CHANGELOG.md new file mode 100644 index 0000000000..84bbd03585 --- /dev/null +++ b/bundled/kokkos-4.5.01/CHANGELOG.md @@ -0,0 +1,2370 @@ +# CHANGELOG + +## 4.5.01 + +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.5.00...4.5.01) + +### Bug Fixes + +* Fix re-builds after cleaning the binary tree when doing `add_subdirectory` on the Kokkos source [\#7557](https://github.com/kokkos/kokkos/pull/7557) +* Update mdspan to include fix for submdspan and bracket operator with clang 15&16 [\#7559](https://github.com/kokkos/kokkos/pull/7559) +* Fix DynRankView performance regression by re-introducing shortcut operator() impls [\#7606](https://github.com/kokkos/kokkos/pull/7606) +* Add missing MI300A (`GFX942_APU`) option to Makefile build-system + +## 4.5.00 + +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.4.01...4.5.00) + +### Features + +* SYCL backend graduated to production ready +* Introduce new `SequentialHostInit` view allocation property [\#7229](https://github.com/kokkos/kokkos/pull/7229) (backported in 4.4.01) +* Support building with Run-Time Type Information (RTTI) disabled +* Add new `KOKKOS_RELOCATABLE_FUNCTION` function annotation macro [\#5993](https://github.com/kokkos/kokkos/pull/5993) + +### Backend and Architecture Enhancements + +#### CUDA + +* Adding occupancy tuning for CUDA architectures [\#6788](https://github.com/kokkos/kokkos/pull/6788) +* By default disable `cudaMallocAsync` (i.e., revert the change made in version 4.2) [\#7353](https://github.com/kokkos/kokkos/pull/7353) + +#### HIP + +* Add support for AMD Phoenix APUs with Radeon 740M/760M/780M/880M/890M [\#7162](https://github.com/kokkos/kokkos/pull/7162) +* Update maximum waves per CU values for consumer card [\#7347](https://github.com/kokkos/kokkos/pull/7347) +* Check that Kokkos is running on the architecture it was compiled for [\#7379](https://github.com/kokkos/kokkos/pull/7379) +* Add opt-in option to use `hipMallocAsync` instead of `hipMalloc` [\#7324](https://github.com/kokkos/kokkos/pull/7324) +* Introduce new architecture option `AMD_GFX942_APU` for MI300A [\#7462](https://github.com/kokkos/kokkos/pull/7462) + +#### SYCL + +* Move the `SYCL` backend out of the `Experimental` namespace [\#7171](https://github.com/kokkos/kokkos/pull/7171) +* Introduce `KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE` as CMake option [\#5993](https://github.com/kokkos/kokkos/pull/5993) + +#### OpenACC + +* Add support for building with the Clacc compiler [\#7198](https://github.com/kokkos/kokkos/pull/7198) +* Workaround NVHPC collapse clause bug for `MDRangePolicy` [\#7425](https://github.com/kokkos/kokkos/pull/7425) + +#### HPX + +* Implement `Experimental::partition_space` to produce truly independent execution spaces [\#7287](https://github.com/kokkos/kokkos/pull/7287) + +#### Threads + +* Fix compilation for `parallel_reduce` `MDRange` with `Dynamic` scheduling [\#7478](https://github.com/kokkos/kokkos/pull/7478) +* Fix race conditions on ARM architectures [\#7498](https://github.com/kokkos/kokkos/pull/7498) + +#### OpenMP + +* Fix run time behavior when compiling with `-fvisibility-hidden` [\#7284](https://github.com/kokkos/kokkos/pull/7284) (backported in 4.4.01) +* Fix linking with Cray Clang compiler [\#7341](https://github.com/kokkos/kokkos/pull/7341) + +#### Serial + +* Allow `Kokkos_ENABLE_ATOMICS_BYPASS` to skip mutexes to remediate performance regression in 4.4 [\#7369](https://github.com/kokkos/kokkos/pull/7369) + +### General Enhancements + +* Improve `View` initialization/destruction for non-scalar trivial and trivially-destructible types [\#7219](https://github.com/kokkos/kokkos/pull/7219) [\#7225](https://github.com/kokkos/kokkos/pull/7225) +* Add getters for default tile sizes used in `MDRangePolicy` [\#6839](https://github.com/kokkos/kokkos/pull/6839) +* Improve performance of `Kokkos::sort` when `std::sort` is used [\#7264](https://github.com/kokkos/kokkos/pull/7264) +* Add range-based for loop support for `Array` [\#7293](https://github.com/kokkos/kokkos/pull/7293) +* Allow functors as reducers for nested team parallel reduce [\#6921](https://github.com/kokkos/kokkos/pull/6921) +* Avoid making copies of string rvalue reference arguments to `view_alloc()` [\#7364](https://github.com/kokkos/kokkos/pull/7364) +* Add `atomic_{mod,xor,nand,lshift,rshift}` [\#7458](https://github.com/kokkos/kokkos/pull/7458) +* Allow using `SequentialHostInit` with `Kokkos::DualView` [\#7456](https://github.com/kokkos/kokkos/pull/7456) +* Add `Graph::instantiate()` [\#7240](https://github.com/kokkos/kokkos/pull/7240) +* Allow an arbitrary execution space instance to be used in `Kokkos::Graph::submit()` [\#7249](https://github.com/kokkos/kokkos/pull/7249) +* Enable compile-time diagnostic of illegal reduction target for graphs [\#7460](https://github.com/kokkos/kokkos/pull/7460) + +### Build System Changes + +* Make sure backend-specific options such as `IMPL_CUDA_MALLOC_ASYNC` only show when that backend is actually enabled [\#7228](https://github.com/kokkos/kokkos/pull/7228) +* Major refactoring removing `TriBITS` paths [\#6164](https://github.com/kokkos/kokkos/pull/6164) +* Add support for SpacemiT K60 (RISC-V) [\#7160](https://github.com/kokkos/kokkos/pull/7160) + +### Deprecations + +* Deprecate Tasking interface [\#7393](https://github.com/kokkos/kokkos/pull/7393) +* Deprecate `atomic_query_version`, `atomic_assign`, `atomic_compare_exchange_strong`, `atomic_{inc, dec}rement` [\#7458](https://github.com/kokkos/kokkos/pull/7458) +* Deprecate `{OpenMP,HPX}::is_asynchronous()` [\#7322](https://github.com/kokkos/kokkos/pull/7322) + +### Bug Fixes + +* Fix undefined behavior in `BinSort` when sorting within bins on host [\#7223](https://github.com/kokkos/kokkos/pull/7223) +* Using CUDA limits to set extents for blocks, grids [\#7235](https://github.com/kokkos/kokkos/pull/7235) +* Fix `deep_copy (serial_exec, dst, src)` with multiple host backends [\#7245](https://github.com/kokkos/kokkos/pull/7245) +* Skip `RangePolicy` bounds conversion checks if roundtrip convertibility is not provided [\#7172](https://github.com/kokkos/kokkos/pull/7172) +* Allow extracting host and device views from `DualView` with `const` value type [\#7242](https://github.com/kokkos/kokkos/pull/7242) +* Fix `TeamPolicy` array reduction for CUDA and HIP [\#6296](https://github.com/kokkos/kokkos/pull/6296) +* Fix implicit copy assignment operators in few AVX2 masks being deleted [\#7296](https://github.com/kokkos/kokkos/pull/7296) +* Fix configuring without architecture flags for SYCL [\#7303](https://github.com/kokkos/kokkos/pull/7303) +* Set an initial value index during join of `MinLoc`, `MaxLoc` or `MinMaxLoc` [\#7330](https://github.com/kokkos/kokkos/pull/7330) +* Fix storage lifetime of driver for global launch of graph nodes for CUDA and HIP [\#7365](https://github.com/kokkos/kokkos/pull/7365) +* Make `value_type` for `RandomAccessIterator` non-`const` [\#7485](https://github.com/kokkos/kokkos/pull/7485) + +## [4.4.01](https://github.com/kokkos/kokkos/tree/4.4.01) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.4.00...4.4.01) + +### Features: +* Introduce new SequentialHostInit view allocation property [\#7229](https://github.com/kokkos/kokkos/pull/7229) + +### Backend and Architecture Enhancements: + +#### CUDA: +* Experimental support for unified memory mode (intended for Grace-Hopper etc.) [\#6823](https://github.com/kokkos/kokkos/pull/6823) + +### Bug Fixes +* OpenMP: Fix issue related to the visibility of an internal symbol with shared libraries that affected `ScatterView` in particular [\#7284](https://github.com/kokkos/kokkos/pull/7284) +* Fix implicit copy assignment operators in few AVX2 masks being deleted [\#7296](https://github.com/kokkos/kokkos/pull/7296) + +## [4.4.00](https://github.com/kokkos/kokkos/tree/4.4.00) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.3.01...4.4.00) + +### Features: +* Add `Kokkos::View` conversions from and to [`std::mdspan`](https://en.cppreference.com/w/cpp/container/mdspan) [\#6830](https://github.com/kokkos/kokkos/pull/6830) [\#7069](https://github.com/kokkos/kokkos/pull/7069) + +### Backend and Architecture Enhancements: + +#### CUDA: +* `nvcc_wrapper`: Adding ability to process `--disable-warnings` flag [\#6936](https://github.com/kokkos/kokkos/issues/6936) +* Use recommended/max team size functions in Cuda ParallelFor and Reduce constructors [\#6891](https://github.com/kokkos/kokkos/issues/6891) +* Improve compile-times when building with `Kokkos_ENABLE_DEBUG_BOUNDS_CHECK` in Cuda [\#7013](https://github.com/kokkos/kokkos/pull/7013) + +#### HIP: +* Use HIP builtin atomics [\#6882](https://github.com/kokkos/kokkos/pull/6882) [\#7000](https://github.com/kokkos/kokkos/pull/7000) +* Enable user-specified compiler and linker flags for AMD GPUs [\#7127](https://github.com/kokkos/kokkos/pull/7127) + +#### SYCL: +* Add support for Graphs [\#6912](https://github.com/kokkos/kokkos/pull/6912) +* Fix multi-GPU support [\#6887](https://github.com/kokkos/kokkos/pull/6887) +* Improve performance of reduction and scan operations [\#6562](https://github.com/kokkos/kokkos/pull/6562), [\#6750](https://github.com/kokkos/kokkos/pull/6750) +* Fix lock for guarding scratch space in `TeamPolicy` `parallel_reduce` [\#6988](https://github.com/kokkos/kokkos/pull/6988) +* Include submission command queue property information into `SYCL::print_configuration()` [\#7004](https://github.com/kokkos/kokkos/pull/7004) + +#### OpenACC: +* Make `TeamPolicy` `parallel_for` execute on the correct async queue [\#7012](https://github.com/kokkos/kokkos/pull/7012) + +#### OpenMPTarget: +* Honor user requested loop ordering in `MDRange` policy [\#6925](https://github.com/kokkos/kokkos/pull/6925) +* Prevent data races by guarding the scratch space used in `parallel_scan` [\#6998](https://github.com/kokkos/kokkos/pull/6998) + +#### HPX: +* Workaround issue with template argument deduction to support compilation with NVCC [\#7015](https://github.com/kokkos/kokkos/pull/7015) + +### General Enhancements +* Improve performance of view copies in host parallel regions [\#6730](https://github.com/kokkos/kokkos/pull/6730) +* Harmonize convertibility rules of `Kokkos::RandomAccessIterator` with `View`s [\#6929](https://github.com/kokkos/kokkos/pull/6929) +* Add a check precondition non-overlapping ranges for the `adjacent_difference` algorithm in debug mode [\#6922](https://github.com/kokkos/kokkos/pull/6922) +* Add deduction guides for `TeamPolicy` [\#7030](https://github.com/kokkos/kokkos/pull/7030) +* SIMD: Allow flexible vector width for 32 bit types [\#6802](https://github.com/kokkos/kokkos/pull/6802) +* Updates for `Kokkos::Array`: add `kokkos_swap(Array)` specialization [\#6943](https://github.com/kokkos/kokkos/pull/6943), add `Kokkos::to_array` [\#6375](https://github.com/kokkos/kokkos/pull/6375), make `Kokkos::Array` equality-comparable [\#7148](https://github.com/kokkos/kokkos/pull/7148) +* Structured binding support for `Kokkos::complex` [\#7040](https://github.com/kokkos/kokkos/pull/7040) +* Introduce `KOKKOS_DEDUCTION_GUIDE` macro to allow for portable user-defined deduction guides [\#6954](https://github.com/kokkos/kokkos/pull/6954) + +### Build System Changes +* Do not require OpenMP support for languages other than CXX [\#6965](https://github.com/kokkos/kokkos/pull/6965) +* Update Intel GPU architectures in Makefile [\#6895](https://github.com/kokkos/kokkos/pull/6895) +* Fix use of OpenMP with Cuda or HIP as compile language [\#6972](https://github.com/kokkos/kokkos/pull/6972) +* Define and enforce new minimum compiler versions for C++20 support [\#7128](https://github.com/kokkos/kokkos/pull/7128), [\#7123](https://github.com/kokkos/kokkos/pull/7123) +* Add nvidia Grace CPU architecture: `Kokkos_ARCH_ARMV9_GRACE` [\#7158](https://github.com/kokkos/kokkos/pull/7158) +* Fix Makefile.kokkos for Threads [\#6896](https://github.com/kokkos/kokkos/pull/6896) +* Remove support for NVHPC as CUDA device compiler [\#6987](https://github.com/kokkos/kokkos/pull/6987) +* Fix using CUDAToolkit for CMake 3.28.4 and higher [\#7062](https://github.com/kokkos/kokkos/pull/7062) + +### Incompatibilities (i.e. breaking changes) +* Drop `Kokkos::Array` special treatment in `View`s [\#6906](https://github.com/kokkos/kokkos/pull/6906) +* Drop `Experimental::RawMemoryAllocationFailure` [\#7145](https://github.com/kokkos/kokkos/pull/7145) + +### Deprecations +* Remove `Experimental::LayoutTiled` class template and deprecate `is_layouttiled` trait [\#6907](https://github.com/kokkos/kokkos/pull/6907) +* Deprecate `Kokkos::layout_iterate_type_selector` [\#7076](https://github.com/kokkos/kokkos/pull/7076) +* Deprecate specialization of `Kokkos::pair` for a single element [\#6947](https://github.com/kokkos/kokkos/pull/6947) +* Deprecate `deep_copy` of `UnorderedMap` of different size [\#6812](https://github.com/kokkos/kokkos/pull/6812) +* Deprecate trailing `Proxy` template argument of `Kokkos::Array` [\#6934](https://github.com/kokkos/kokkos/pull/6934) +* Deprecate implicit conversions of integers to `ChunkSize` [\#7151](https://github.com/kokkos/kokkos/pull/7151) +* Deprecate implicit conversions to execution spaces [\#7156](https://github.com/kokkos/kokkos/pull/7156) + +### Bug Fixes +* Do not return a copy of the input functor in `Experimental::for_each` [\#6910](https://github.com/kokkos/kokkos/pull/6910) +* Fix `realloc` on views of non-default constructible element types [\#6993](https://github.com/kokkos/kokkos/pull/6993) +* Fix undefined behavior in `View` initialization or fill with zeros [\#7014](https://github.com/kokkos/kokkos/pull/7014) +* Fix `sort_by_key` on host execution spaces when building with NVCC [\#7059](https://github.com/kokkos/kokkos/pull/7059) +* Fix using shared libraries and -fvisibility=hidden [\#7065](https://github.com/kokkos/kokkos/pull/7065) +* Fix view reference counting when functor copy constructor throws in parallel dispatch [\#6289](https://github.com/kokkos/kokkos/pull/6289) +* Fix `initialize(InitializationSetting)` for handling `print_configuration` setting [\#7098](https://github.com/kokkos/kokkos/pull/7098) +* Thread safety fixes for the Serial and OpenMP backend [\#7080](https://github.com/kokkos/kokkos/pull/7080), [\#6151](https://github.com/kokkos/kokkos/pull/6151) + +## [4.3.01](https://github.com/kokkos/kokkos/tree/4.3.01) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.3.00...4.3.01) + +### Backend and Architecture Enhancements: + +#### HIP: +* MI300 support unified memory [\#6877](https://github.com/kokkos/kokkos/pull/6877) + +### Bug Fixes +* Serial: Use the provided execution space instance in TeamPolicy [\#6951](https://github.com/kokkos/kokkos/pull/6951) +* `nvcc_wrapper`: bring back support for `--fmad` option [\#6931](https://github.com/kokkos/kokkos/pull/6931) +* Fix CUDA reduction overflow for `RangePolicy` [\#6578](https://github.com/kokkos/kokkos/pull/6578) + +## [4.3.00](https://github.com/kokkos/kokkos/tree/4.3.00) (2024-03-19) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.2.01...4.3.00) + +### Features: +* Add `Experimental::sort_by_key(exec, keys, values)` algorithm [\#6801](https://github.com/kokkos/kokkos/pull/6801) + +### Backend and Architecture Enhancements: + +#### CUDA: +* Experimental multi-GPU support (from the same process) [\#6782](https://github.com/kokkos/kokkos/pull/6782) +* Link against CUDA libraries even with KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE [\#6701](https://github.com/kokkos/kokkos/pull/6701) +* Don't use the compiler launcher script if the CMake compile language is CUDA. [\#6704](https://github.com/kokkos/kokkos/pull/6704) +* nvcc(wrapper): adding "long" and "short" versions for all flags [\#6615](https://github.com/kokkos/kokkos/pull/6615) + +#### HIP: + * Fix compilation when using amdclang (with ROCm >= 5.7) and RDC [\#6857](https://github.com/kokkos/kokkos/pull/6857) + * Use rocthrust for sorting, when available [\#6793](https://github.com/kokkos/kokkos/pull/6793) + +#### SYCL: +* We only support OneAPI SYCL implementation: add check during initialization + * Error out on initialization if the backend is different from `ext_oneapi_*` [\#6784](https://github.com/kokkos/kokkos/pull/6784) + * Filter GPU devices for `ext_onapi_*` GPU devices [\#6758](https://github.com/kokkos/kokkos/pull/6784) +* Performance Improvements + * Avoid unnecessary zero-memset of the scratch flags in SYCL [\#6739](https://github.com/kokkos/kokkos/pull/6739) + * Use host-pinned memory to copy reduction/scan result [\#6500](https://github.com/kokkos/kokkos/pull/6500) +* Address deprecations after oneAPI 2023.2.0 [\#6577](https://github.com/kokkos/kokkos/pull/6739) +* Make sure to call find_dependency for oneDPL if necessary [\#6870](https://github.com/kokkos/kokkos/pull/6870) + +#### OpenMPTarget: +* Use LLVM extensions for dynamic shared memory [\#6380](https://github.com/kokkos/kokkos/pull/6380) +* Guard scratch memory usage in ParallelReduce [\#6585 ](https://github.com/kokkos/kokkos/pull/6585) +* Update linker flags for Intel GPUs update [\#6735](https://github.com/kokkos/kokkos/pull/6735) +* Improve handling of printf on Intel GPUs [\#6652](https://github.com/kokkos/kokkos/pull/6652) + +#### OpenACC: +* Add atomics support [\#6446](https://github.com/kokkos/kokkos/pull/6446) +* Make the OpenACC backend asynchronous [\#6772](https://github.com/kokkos/kokkos/pull/6772) + +#### Threads: +* Add missing broadcast to TeamThreadRange parallel_scan [\#6601](https://github.com/kokkos/kokkos/pull/6601) + +#### OpenMP: +* Improve performance of view initializations and filling with zeros [\#6573](https://github.com/kokkos/kokkos/pull/6573) + +### General Enhancements + +* Improve performance of random number generation when using a normal distribution on GPUs [\#6556](https://github.com/kokkos/kokkos/pull/6556) +* Allocate temporary view with the user-provided execution space instance and do not initialize in `unique` algorithm [\#6598](https://github.com/kokkos/kokkos/pull/6598) +* Add deduction guide for `Kokkos::Array` [\#6373](https://github.com/kokkos/kokkos/pull/6373) +* Provide new public headers `` and `` [\#6687](https://github.com/kokkos/kokkos/pull/6687) +* Fix/improvement to `remove_if` parallel algorithm: use the provided execution space instance for temporary allocations and drop unnecessaryinitialization + avoid evaluating twice the predicate during final pass [\#6747](https://github.com/kokkos/kokkos/pull/6747) +* Add runtime function to query the number of devices and make device ID consistent with `KOKKOS_VISIBLE_DEVICES` [\#6713](https://github.com/kokkos/kokkos/pull/6713) +* simd: support `vector_aligned_tag` [\#6243](https://github.com/kokkos/kokkos/pull/6243) +* Avoid unnecessary allocation when default constructing Bitset [\#6524](https://github.com/kokkos/kokkos/pull/6524) +* Fix constness for views in std algorithms [\#6813](https://github.com/kokkos/kokkos/pull/6813) +* Improve error message on unsafe implicit conversion in MDRangePolicy [\#6855](https://github.com/kokkos/kokkos/pull/6855) +* CTAD (deduction guides) for RangePolicy [\#6850](https://github.com/kokkos/kokkos/pull/6850) +* CTAD (deduction guides) for MDRangePolicy [\#5516](https://github.com/kokkos/kokkos/pull/5516) + +### Build System Changes +* Require `Kokkos_ENABLE_ATOMICS_BYPASS` option to bypass atomic operation for Serial backend only builds [\#6692](https://github.com/kokkos/kokkos/pull/6692) +* Add support for RISCV and the Milk-V's Pioneer [\#6773](https://github.com/kokkos/kokkos/pull/6773) +* Add C++26 standard to CMake setup [\#6733](https://github.com/kokkos/kokkos/pull/6733) +* Fix Makefile when using gnu_generate_makefile.sh and make >= 4.3 [\#6606](https://github.com/kokkos/kokkos/pull/6606) +* Cuda: Fix configuring with CMake >= 3.28.4 - temporary fallback to internal CudaToolkit.cmake [\#6898](https://github.com/kokkos/kokkos/pull/6898) + +### Incompatibilities (i.e. breaking changes) +* Remove all `DEPRECATED_CODE_3` option and all code that was guarded by it [\#6523](https://github.com/kokkos/kokkos/pull/6523) +* Drop guards to accommodate external code defining `KOKKOS_ASSERT` [\#6665](https://github.com/kokkos/kokkos/pull/6665) +* `Profiling::ProfilingSection(std::string)` constructor marked explicit and nodiscard [\#6690](https://github.com/kokkos/kokkos/pull/6690) +* Add bound check preconditions for `RangePolicy` and `MDRangePolicy` [\#6617](https://github.com/kokkos/kokkos/pull/6617) [\#6726](https://github.com/kokkos/kokkos/pull/6726) +* Add checks for unsafe implicit conversions in RangePolicy [\#6754](https://github.com/kokkos/kokkos/pull/6754) +* Remove Kokkos::[b]half_t volatile overloads [\#6579](https://github.com/kokkos/kokkos/pull/6579) +* Remove KOKKOS_IMPL_DO_NOT_USE_PRINTF [\#6593](https://github.com/kokkos/kokkos/pull/6593) +* Check matching static extents in View constructor [\#5190 ](https://github.com/kokkos/kokkos/pull/5190) +* Tools(profiling): fix typo Kokkos_Tools_Optim[i]zationGoal [\#6642](https://github.com/kokkos/kokkos/pull/6642) +* Remove variadic range policy constructor (disallow passing multiple trailing chunk size arguments) [\#6845](https://github.com/kokkos/kokkos/pull/6845) +* Improve message on view out of bounds access and always abort [\#6861](https://github.com/kokkos/kokkos/pull/6861) +* Drop `KOKKOS_ENABLE_INTEL_MM_ALLOC` macro [\#6797](https://github.com/kokkos/kokkos/pull/6797) +* Remove `Kokkos::Experimental::LogicalMemorySpace` (without going through deprecation) [\#6557](https://github.com/kokkos/kokkos/pull/6557) +* Remove `Experimental::HBWSpace` and support for linking against memkind [\#6791](https://github.com/kokkos/kokkos/pull/6791) +* Drop librt TPL and associated `KOKKOS_ENABLE_LIBRT` macro [\#6798](https://github.com/kokkos/kokkos/pull/6798) +* Drop support for old CPU architectures (`ARCH_BGQ`, `ARCH_POWER7`, `ARCH_WSM` and associated `ARCH_SSE4` macro) [\#6806](https://github.com/kokkos/kokkos/pull/6806) +* Drop support for deprecated command-line arguments and environment variables [\#6744](https://github.com/kokkos/kokkos/pull/6744) + +### Deprecations +* Provide kokkos_swap as part of Core and deprecate Experimental::swap in Algorithms [\#6697](https://github.com/kokkos/kokkos/pull/6697) +* Deprecate {Cuda,HIP}::detect_device_count() and Cuda::[detect_]device_arch() [\#6710](https://github.com/kokkos/kokkos/pull/6710) +* Deprecate `ExecutionSpace::in_parallel()` [\#6582](https://github.com/kokkos/kokkos/pull/6582) + +### Bug Fixes +* Fix team-level MDRange reductions: [\#6511](https://github.com/kokkos/kokkos/pull/6511) +* Fix CUDA and SYCL small value type (16-bit) team reductions [\#5334](https://github.com/kokkos/kokkos/pull/5334) +* Enable `{transform_}exclusive_scan` in place [\#6667](https://github.com/kokkos/kokkos/pull/6667) +* `fill_random` overload that do not take an execution space instance argument should fence [\#6658](https://github.com/kokkos/kokkos/pull/6658) +* HIP,Cuda,OpenMPTarget: Fixup use provided execution space when copying host inaccessible reduction result [\#6777](https://github.com/kokkos/kokkos/pull/6777) +* Fix typo in `cuda_func_set_attribute[s]_wrapper` preventing proper setting of desired occupancy [\#6786](https://github.com/kokkos/kokkos/pull/6786) +* Avoid undefined behavior due to conversion between signed and unsigned integers in shift_{right, left}_team_impl [\#6821](https://github.com/kokkos/kokkos/pull/6821) +* Fix a bug in Makefile.kokkos when using AMD GPU architectures as `AMD_GFXYYY` [\#6892](https://github.com/kokkos/kokkos/pull/6892) + +## [4.2.01](https://github.com/kokkos/kokkos/tree/4.2.01) (2023-12-07) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.2.00...4.2.01) + +### Backend and Architecture Enhancements: + +#### CUDA: +- Add warp sync for `parallel_reduce` to avoid race condition [\#6630](https://github.com/kokkos/kokkos/pull/6630), [\#6746](https://github.com/kokkos/kokkos/pull/6746) + +#### HIP: +- Fix Graph "multiple definition of" linking error (missing `inline` specifier) [\#6624](https://github.com/kokkos/kokkos/pull/6624) +- Add support for gfx940 (AMD Instinct MI300 GPU) [\#6671](https://github.com/kokkos/kokkos/pull/6671) + +### Build System +- CMake: Don't let Kokkos set `CMAKE_CXX_FLAGS` for Trilinos builds [\#6742](https://github.com/kokkos/kokkos/pull/6742) + +### Bug Fixes +- Remove deprecation warning for `AllocationMechanism` for GCC <11.0 [\#6653](https://github.com/kokkos/kokkos/pull/6653) +- Fix bug early tools finalize with non-default host execution instances [\#6635](https://github.com/kokkos/kokkos/pull/6635) +- Fix various issues for MSVC CUDA builds [\#6659](https://github.com/kokkos/kokkos/pull/6659) +- Fix "extra `;`" warning with `-pedantic` flag in `` [\#6510](https://github.com/kokkos/kokkos/pull/6510) + +## [4.2.00](https://github.com/kokkos/kokkos/tree/4.2.00) (2023-11-06) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.1.00...4.2.00) + +### Features: +- SIMD: significant improvements to SIMD support and alignment with C++26 SIMD + - add `Kokkos::abs` overload for SIMD types [\#6069](https://github.com/kokkos/kokkos/pull/6069) + - add generator constructors [\#6347](https://github.com/kokkos/kokkos/pull/6347) + - convert binary operators to hidden friends [\#6320](https://github.com/kokkos/kokkos/pull/6320) + - add shift operators [\#6109](https://github.com/kokkos/kokkos/pull/6109) + - add `float` support [\#6177](https://github.com/kokkos/kokkos/pull/6177) + - add remaining `gather_from` and `scatter_to` overloads [\#6220](https://github.com/kokkos/kokkos/pull/6220) + - define simd math function overloads in the Kokkos namespace [\#6465](https://github.com/kokkos/kokkos/pull/6465), [\#6487](https://github.com/kokkos/kokkos/pull/6487) + - `Kokkos_ENABLE_NATIVE=ON` autodetects SIMD types supported [\#6188](https://github.com/kokkos/kokkos/pull/6188) + - fix AVX2 SIMD support for ZEN2 AMD CPU [\#6238](https://github.com/kokkos/kokkos/pull/6238) +- `Kokkos::printf` [\#6083](https://github.com/kokkos/kokkos/pull/6083) +- `Kokkos::sort`: support custom comparator [\#6253](https://github.com/kokkos/kokkos/pull/6253) +- `half_t` and `bhalf_t` numeric traits [\#5778](https://github.com/kokkos/kokkos/pull/5778) +- `half_t` and `bhalf_t` mixed comparisons [\#6407](https://github.com/kokkos/kokkos/pull/6407) +- `half_t` and `bhalf_t` mathematical functions [\#6124](https://github.com/kokkos/kokkos/pull/6124) +- `TeamThreadRange` `parallel_scan` with return value [\#6090](https://github.com/kokkos/kokkos/pull/6090), [\#6301](https://github.com/kokkos/kokkos/pull/6301), [\#6302](https://github.com/kokkos/kokkos/pull/6302), [\#6303](https://github.com/kokkos/kokkos/pull/6303), [\#6307](https://github.com/kokkos/kokkos/pull/6307) +- `ThreadVectorRange` `parallel_scan` with return value [\#6235](https://github.com/kokkos/kokkos/pull/6235), [\#6242](https://github.com/kokkos/kokkos/pull/6242), [\#6308](https://github.com/kokkos/kokkos/pull/6308), [\#6305](https://github.com/kokkos/kokkos/pull/6305), [\#6292](https://github.com/kokkos/kokkos/pull/6292) +- Add team-level std algorithms [\#6200](https://github.com/kokkos/kokkos/pull/6200), [\#6205](https://github.com/kokkos/kokkos/pull/6205), [\#6207](https://github.com/kokkos/kokkos/pull/6207), [\#6208](https://github.com/kokkos/kokkos/pull/6208), [\#6209](https://github.com/kokkos/kokkos/pull/6209), [\#6210](https://github.com/kokkos/kokkos/pull/6210), [\#6211](https://github.com/kokkos/kokkos/pull/6211), [\#6212](https://github.com/kokkos/kokkos/pull/6212), [\#6213](https://github.com/kokkos/kokkos/pull/6213), [\#6256](https://github.com/kokkos/kokkos/pull/6256), [\#6258](https://github.com/kokkos/kokkos/pull/6258), [\#6350](https://github.com/kokkos/kokkos/pull/6350), [\#6351](https://github.com/kokkos/kokkos/pull/6351) +- Serial: Allow for distinct execution space instances [\#6441](https://github.com/kokkos/kokkos/pull/6441) + +### Backend and Architecture Enhancements: + +#### CUDA: +- Fixed potential data race in Cuda `parallel_reduce` [\#6236](https://github.com/kokkos/kokkos/pull/6236) +- Use `cudaMallocAsync` by default [\#6402](https://github.com/kokkos/kokkos/pull/6402) +- Bugfix for using Kokkos from a thread of execution [\#6299](https://github.com/kokkos/kokkos/pull/6299) + +#### HIP: +- New naming convention for AMD GPU: VEGA906, VEGA908, VEGA90A, NAVI1030 to AMD_GFX906, AMD_GFX908, AMD_GFX90A, AMD_GFX1030 [\#6266](https://github.com/kokkos/kokkos/pull/6266) +- Add initial support for gfx942: [\#6358](https://github.com/kokkos/kokkos/pull/6358) +- Improve reduction performance [\#6229](https://github.com/kokkos/kokkos/pull/6229) +- Deprecate `HIP(hipStream_t,bool)` constructor [\#6401](https://github.com/kokkos/kokkos/pull/6401) +- Add support for Graph [\#6370](https://github.com/kokkos/kokkos/pull/6370) +- Improve reduction performance when using Teams [\#6284](https://github.com/kokkos/kokkos/pull/6284) +- Fix concurrency calculation [\#6479](https://github.com/kokkos/kokkos/pull/6479) +- Fix potential data race in HIP `parallel_reduce` [\#6429](https://github.com/kokkos/kokkos/pull/6429) + +#### SYCL: +- Enforce external `sycl::queues` to be in-order [\#6246](https://github.com/kokkos/kokkos/pull/6246) +- Improve reduction performance: [\#6272](https://github.com/kokkos/kokkos/pull/6272) [\#6271](https://github.com/kokkos/kokkos/pull/6271) [\#6270](https://github.com/kokkos/kokkos/pull/6270) [\#6264](https://github.com/kokkos/kokkos/pull/6264) +- Allow using the SYCL execution space on AMD GPUs [\#6321](https://github.com/kokkos/kokkos/pull/6321) +- Allow sorting via native oneDPL to support Views with stride=1 [\#6322](https://github.com/kokkos/kokkos/pull/6322) +- Make in-order queues the default via macro [\#6189](https://github.com/kokkos/kokkos/pull/6189) + +#### OpenACC: +- Support Clacc compiler [\#6250](https://github.com/kokkos/kokkos/pull/6250) + +### General Enhancements +- Add missing `is_*_view` traits and `is_*_view_v` helper variable templates for `DynRankView`, `DynamicView`, `OffsetView`, `ScatterView` containers [\#6195](https://github.com/kokkos/kokkos/pull/6195) +- Make `nvcc_wrapper` and `compiler_launcher` scripts more portable by switching to a `#!/usr/bin/env` shebang [\#6357](https://github.com/kokkos/kokkos/pull/6357) +- Add an improved `Kokkos::malloc` / `Kokkos::free` performance test [\#6377](https://github.com/kokkos/kokkos/pull/6377) +- Ensure `Views` with `size==0` can be used with `deep_copy` [\#6273](https://github.com/kokkos/kokkos/pull/6273) +- `Kokkos::abort` is moved to header `Kokkos_Abort.hpp` [\#6445](https://github.com/kokkos/kokkos/pull/6445) +- `KOKKOS_ASSERT`, `KOKKOS_EXPECTS`, `KOKKOS_ENSURES` are moved to header `Kokkos_Assert.hpp` [\#6445](https://github.com/kokkos/kokkos/pull/6445) +- Add a permuted-index mode to the gups benchmark [\#6378](https://github.com/kokkos/kokkos/pull/6378) +- Check for overflow during backend initialization [\#6159](https://github.com/kokkos/kokkos/pull/6159) +- Make constraints on `Kokkos::sort` more visible [\#6234](https://github.com/kokkos/kokkos/pull/6234) and cleanup API [\#6239](https://github.com/kokkos/kokkos/pull/6239) +- Add converting assignment to `DualView`: [\#6474](https://github.com/kokkos/kokkos/pull/6474) + + +### Build System Changes + +- Export `Kokkos_CXX_COMPILER_VERSION` [\#6282](https://github.com/kokkos/kokkos/pull/6282) +- Disable default oneDPL support in Trilinos [\#6342](https://github.com/kokkos/kokkos/pull/6342) + +### Incompatibilities (i.e. breaking changes) + - Ensure that `Kokkos::complex` only gets instantiated for cv-unqualified floating-point types [\#6251](https://github.com/kokkos/kokkos/pull/6251) + - Removed (deprecated-3) support for volatile join operators in reductions [\#6385](https://github.com/kokkos/kokkos/pull/6385) + - Enforce `ViewCtorArgs` restrictions for `create_mirror_view` [\#6304](https://github.com/kokkos/kokkos/pull/6304) + - SIMD types for ARM NEON are not autodetected anymore but need `Kokkos_ARCH_ARM_NEON` or `Kokkos_ARCH_NATIVE=ON` [\#6394](https://github.com/kokkos/kokkos/pull/6394) + - Remove `#include ` from headers where possible [\#6482](https://github.com/kokkos/kokkos/pull/6482) + +### Deprecations +- Deprecated `Kokkos::vector` [\#6252](https://github.com/kokkos/kokkos/pull/6252) +- All host allocation mechanisms except for `STD_MALLOC` have been deprecated [\#6341](https://github.com/kokkos/kokkos/pull/6341) + +### Bug Fixes + - Missing memory fence in `RandomPool::free_state` functions [\#6290](https://github.com/kokkos/kokkos/pull/6290) + - Fix for corner case in `Kokkos::Experimental::is_partitioned` algorithm [\#6257](https://github.com/kokkos/kokkos/pull/6257) + - Fix initialization of scratch lock variables in the `Cuda` backend [\#6433](https://github.com/kokkos/kokkos/pull/6433) + - Fixes for `Kokkos::Array` [\#6372](https://github.com/kokkos/kokkos/pull/6372) + - Fixed symlink configure issue for Windows [\#6241](https://github.com/kokkos/kokkos/pull/6241) + - OpenMPTarget init-join fix [\#6444](https://github.com/kokkos/kokkos/pull/6444) + - Fix atomic operations bug for Min and Max [\#6435](https://github.com/kokkos/kokkos/pull/6435) + - Fix implementation for `cyl_bessel_i0` [\#6484](https://github.com/kokkos/kokkos/pull/6484) + - Fix various NVCC warnings in `BinSort`, `Array`, and bit manipulation function templates [\#6483](https://github.com/kokkos/kokkos/pull/6483) + +## [4.1.00](https://github.com/kokkos/kokkos/tree/4.1.00) (2023-06-16) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.0.01...4.1.00) + +### Features: +* Add `` header [\#4577](https://github.com/kokkos/kokkos/pull/4577) [\#5907](https://github.com/kokkos/kokkos/pull/5907) [\#5967](https://github.com/kokkos/kokkos/pull/5967) [\#6101](https://github.com/kokkos/kokkos/pull/6101) +* Add `UnorderedMapInsertOpTypes` [\#5877](https://github.com/kokkos/kokkos/pull/5877) and documentation [\#350](https://github.com/kokkos/kokkos-core-wiki/pull/350) +* Add multiple reducers support for team-level parallel reduce [\#5727](https://github.com/kokkos/kokkos/pull/5727) + +### Backend and Architecture Enhancements: + +#### CUDA: + +* Allow NVCC 12 to compile using C++20 flag [\#5977](https://github.com/kokkos/kokkos/pull/5977) +* Remove ability to disable CMake option `Kokkos_ENABLE_CUDA_LAMBDA` and unconditionally enable CUDA extended lambda support. [\#5964](https://github.com/kokkos/kokkos/pull/5964) +* Drop unnecessary fences around the memory allocation when using `CudaUVMSpace` in views [\#6008](https://github.com/kokkos/kokkos/pull/6008) + +#### HIP: +* Improve performance for `parallel_reduce`. Use different parameters for `LightWeight` kernels [\#6029](https://github.com/kokkos/kokkos/pull/6029) and [\#6160](https://github.com/kokkos/kokkos/pull/6160) + +#### SYCL: +* Only pass one wrapper object in SYCL reductions [\#6047](https://github.com/kokkos/kokkos/pull/6047) +* Improve and simplify parallel_scan implementation [\#6064](https://github.com/kokkos/kokkos/pull/6064) +* Remove workaround for submit_barrier not being enqueued properly [\#5504](https://github.com/kokkos/kokkos/pull/5504) +* Fix guards for using scratch space with SYCL [\#6003](https://github.com/kokkos/kokkos/pull/6003) +* Fix compiling SYCL with KOKKOS_IMPL_DO_NOT_USE_PRINTF_USAGE [\#6219](https://github.com/kokkos/kokkos/pull/6219) + +#### OpenMPTarget: +* Improve hierarchical parallelism for Intel architectures [\#6043](https://github.com/kokkos/kokkos/pull/6043) +* Enable Cray compiler for the OpenMPTarget backend. [\#5889](https://github.com/kokkos/kokkos/pull/5889) + +#### HPX: +* Update HPX backend to use HPX's sender/receiver functionality [\#5628](https://github.com/kokkos/kokkos/pull/5628) +* Increase minimum required HPX version to 1.8.0 [\#6132](https://github.com/kokkos/kokkos/pull/6132) +* Implement HPX::in_parallel [\#6143](https://github.com/kokkos/kokkos/pull/6143) + +### General Enhancements +* Export CMake `Kokkos_{CUDA,HIP}_ARCHITECTURES` variables [\#5919](https://github.com/kokkos/kokkos/pull/5919) [\#5925](https://github.com/kokkos/kokkos/pull/5925) +* Add `Kokkos::Profiling::ScopedRegion` [\#5959](https://github.com/kokkos/kokkos/pull/5959) [\#5972](https://github.com/kokkos/kokkos/pull/5972) +* Add support for `View::rank[_dynamic]()`[\#5870](https://github.com/kokkos/kokkos/pull/5870) +* Detect incompatible relocatable device code mode to prevent ODR violations [\#5991](https://github.com/kokkos/kokkos/pull/5991) +* Add (experimental) support for 32-bit Darwin and PPC [\#5916](https://github.com/kokkos/kokkos/pull/5916) +* Add missing half and bhalf specialization of the infinity numeric trait [\#6055](https://github.com/kokkos/kokkos/pull/6055) +* Add `is_dual_view` trait and align further with regular view [\#6120](https://github.com/kokkos/kokkos/pull/6120) +* Allow templated functors in parallel_for, parallel_reduce and parallel_scan [\#5976](https://github.com/kokkos/kokkos/pull/5976) +* Define KOKKOS_COMPILER_INTEL_LLVM and only define at most one KOKKOS_COMPILER* macro [\#5906](https://github.com/kokkos/kokkos/pull/5906) +* Allow linking against build tree [\#6078](https://github.com/kokkos/kokkos/pull/6078) +* Allow passing a temporary std::vector to partition_space [\#6167](https://github.com/kokkos/kokkos/pull/6167) +* `Kokkos` can be used as an external dependency in `Trilinos` [\#6142](https://github.com/kokkos/kokkos/pull/6142), [\#6157](https://github.com/kokkos/kokkos/pull/6157) [\#6163](https://github.com/kokkos/kokkos/pull/6163) +* Left align demangled stacktrace output [\#6191](https://github.com/kokkos/kokkos/pull/6191) +* Improve OpenMP affinity warning to include MPI concerns [\#6185](https://github.com/kokkos/kokkos/pull/6185) + +### Build System Changes +* Drop `Kokkos_ENABLE_LAUNCH_COMPILER` option which had no effect [\#6148](https://github.com/kokkos/kokkos/pull/6148) +* Export variables for relevant Kokkos options with cmake[\#6142](https://github.com/kokkos/kokkos/pull/6142) + +### Incompatibilities (i.e. breaking changes) +* Desul atomics always enabled [\#5801](https://github.com/kokkos/kokkos/pull/5801) +* Drop `KOKKOS_ENABLE_CUDA_ASM*` and `KOKKOS_ENABLE_*_ATOMICS` macros [\#5940](https://github.com/kokkos/kokkos/pull/5940) +* Drop `KOKKOS_ENABLE_RFO_PREFETCH` macro [\#5944](https://github.com/kokkos/kokkos/pull/5944) +* Deprecate `Kokkos_ENABLE_CUDA_LAMBDA` configuration option and force it to `ON` [\#5964](https://github.com/kokkos/kokkos/pull/5964) +* Remove TriBITS Kokkos subpackages [\#6104](https://github.com/kokkos/kokkos/pull/6104) +* Cuda: Remove unused attach_texture_object [\#6129](https://github.com/kokkos/kokkos/pull/6129) +* Drop Kokkos_ENABLE_PROFILING_LOAD_PRINT configuration option [\#6150](https://github.com/kokkos/kokkos/pull/6150) +* Drop pointless Kokkos{Algorithms,Containers}_config.h files [\#6108](https://github.com/kokkos/kokkos/pull/6108) + +### Deprecations +* Deprecate `BinSort`, `BinOp1D`, and `BinOp3D` default constructors [\#6131](https://github.com/kokkos/kokkos/pull/6131) + +### Bug Fixes +* Fix `SYCLTeamMember` to take arguments for scratch sizes as `std::size_t` [\#5981](https://github.com/kokkos/kokkos/pull/5981) +* Fix Kokkos_SIMD with AVX2 on 64-bit architectures [\#6075](https://github.com/kokkos/kokkos/pull/6075) +* Fix an incorrectly returning size for SIMD uint64_t in AVX2 [\#6004](https://github.com/kokkos/kokkos/pull/6004) +* Fix missing avx512 header file with gcc versions before 10 [\#6183](https://github.com/kokkos/kokkos/pull/6183) +* Fix incorrect results of `parallel_reduce` of types smaller than `int` on CUDA and HIP: [\#5745](https://github.com/kokkos/kokkos/pull/5745) +* CMake: update package compatibility mode when building within Trilinos [\#6012](https://github.com/kokkos/kokkos/pull/6012) +* Fix warnings generated from internal uses of `ALL_t` rather than `Kokkos::ALL_t` [\#6028](https://github.com/kokkos/kokkos/pull/6028) +* Fix bug in `hpcbind` script: check for correct Slurm variable [\#6116](https://github.com/kokkos/kokkos/pull/6116) +* KokkosTools: Don't call callbacks before backends are initialized [\#6114](https://github.com/kokkos/kokkos/pull/6114) +* Fix global fence in Kokkos::resize(DynRankView) [\#6184](https://github.com/kokkos/kokkos/pull/6184) +* Fix `BinSort` support for strided views [\#6081](https://github.com/kokkos/kokkos/pull/6184) +* Fix missing `is_*_view` traits in containers [\#6195](https://github.com/kokkos/kokkos/pull/6195) +* Fix broken OpenMP target on NVHPC [\#6171](https://github.com/kokkos/kokkos/pull/6171) +* Sorting an empty view should exit early and not fail [\#6130](https://github.com/kokkos/kokkos/pull/6130) + +## [4.0.01](https://github.com/kokkos/kokkos/tree/4.0.01) (2023-04-14) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.0.00...4.0.01) + +### Backend and Architecture Enhancements: + +#### CUDA: + +- Allow NVCC 12 to compile using C++20 flag [\#6020](https://github.com/kokkos/kokkos/pull/6020) +- Add CUDA Ada architecture support [\#6022](https://github.com/kokkos/kokkos/pull/6022) + +#### HIP: + +- Add support for AMDGPU target NAVI31 / RX 7900 XT(X): gfx1100 [\#6021](https://github.com/kokkos/kokkos/pull/6021) +- HIP: Fix warning from `std::memcpy` [\#6019](https://github.com/kokkos/kokkos/pull/6019) + +#### SYCL: +- Fix `SYCLTeamMember` to take arguments for scratch sizes as `std::size_t` [\#5986](https://github.com/kokkos/kokkos/pull/5986) + +### General Enhancements +- Fixup 4.0 change log [\#6023](https://github.com/kokkos/kokkos/pull/6023) + +### Build System Changes +- Cherry-pick TriBITS update from Trilinos [\#6037](https://github.com/kokkos/kokkos/pull/6037) +- CMake: update package compatibility mode when building within Trilinos [\#6013](https://github.com/kokkos/kokkos/pull/6013) + +### Bug Fixes +- Fix an incorrectly returning size for SIMD uint64_t in AVX2 [\#6011](https://github.com/kokkos/kokkos/pull/6011) +- Desul atomics: wrong value for `desul::Impl::numeric_limits_max` [\#6018](https://github.com/kokkos/kokkos/pull/6018) +- Fix warning in some user code when using std::memcpy [\#6000](https://github.com/kokkos/kokkos/pull/6000) +- Fix excessive build times using Makefile.kokkos [\#6068](https://github.com/kokkos/kokkos/pull/6068) + +## [4.0.0](https://github.com/kokkos/kokkos/tree/4.0.00) (2023-02-21) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.7.01...4.0.00) + +### Features: +- Allow value types without default constructor in `Kokkos::View` with `Kokkos::WithoutInitializing` [\#5307](https://github.com/kokkos/kokkos/pull/5307) +- `parallel_scan` with `View` as result type. [\#5146](https://github.com/kokkos/kokkos/pull/5146) +- Introduced `SharedSpace`, an alias for a `MemorySpace` that is accessible by every `ExecutionSpace`. The memory is moved and then accessed locally. [\#5289](https://github.com/kokkos/kokkos/pull/5289) +- Introduced `SharedHostPinnedSpace`, an alias for a `MemorySpace` that is accessible by every `ExecutionSpace`. The memory is pinned to the host and accessed via zero-copy access. [\#5405](https://github.com/kokkos/kokkos/pull/5405) +- Add team- and thread-level `sort`, `sort_by_key` algorithms. [\#5317](https://github.com/kokkos/kokkos/pull/5317) +- Groundwork for `MDSpan` integration. [\#4973](https://github.com/kokkos/kokkos/pull/4973) and [\#5304](https://github.com/kokkos/kokkos/pull/5304) +- Introduced MD version of hierarchical parallelism: `TeamThreadMDRange`, `ThreadVectorMDRange` and `TeamVectorMDRange`. [\#5238](https://github.com/kokkos/kokkos/pull/5238) + +### Backend and Architecture Enhancements: + +#### CUDA: +- Allow CUDA PTX forward compatibility [\#3612](https://github.com/kokkos/kokkos/pull/3612) [\#5536](https://github.com/kokkos/kokkos/pull/5536) [\#5527](https://github.com/kokkos/kokkos/pull/5527) +- Add support for NVIDIA Hopper GPU architecture [\#5538](https://github.com/kokkos/kokkos/pull/5538) +- Don't rely on synchronization behavior of default stream in CUDA and HIP [\#5391](https://github.com/kokkos/kokkos/pull/5391) +- Improve CUDA cache config settings [\#5706](https://github.com/kokkos/kokkos/pull/5706) + +#### HIP: + - Move `HIP`, `HIPSpace`, `HIPHostPinnedSpace`, and `HIPManagedSpace` out of the `Experimental` namespace [\#5383](https://github.com/kokkos/kokkos/pull/5383) + - Don't rely on synchronization behavior of default stream in CUDA and HIP [\#5391](https://github.com/kokkos/kokkos/pull/5391) + - Export AMD architecture flag when using Trilinos [\#5528](https://github.com/kokkos/kokkos/pull/5528) + - Fix linking error (see [OLCF issue](https://docs.olcf.ornl.gov/systems/crusher_quick_start_guide.html#olcfdev-1167-kokkos-build-failures-with-prgenv-amd)) when using `amdclang`: [\#5539](https://github.com/kokkos/kokkos/pull/5539) + - Remove support for MI25 and added support for Navi 1030 [\#5522](https://github.com/kokkos/kokkos/pull/5522) + - Fix race condition when using `HSA_XNACK=1` [\#5755](https://github.com/kokkos/kokkos/pull/5755) + - Add parameter to force using GlobalMemory launch mechanism. This can be used when encountering compiler bugs with ROCm 5.3 and 5.4 [\#5796](https://github.com/kokkos/kokkos/pull/5796) + +#### SYCL: +- Delegate choice of workgroup size for `parallel_reduce` with `RangePolicy` to the compiler. [\#5227](https://github.com/kokkos/kokkos/pull/5227) +- SYCL `RangePolicy`: manually specify workgroup size through chunk size [\#4875](https://github.com/kokkos/kokkos/pull/4875) + +#### OpenMPTarget: +- Select the right device [\#5492](https://github.com/kokkos/kokkos/pull/5492) + +#### OpenMP: + - Add `partition_space` [\#5105](https://github.com/kokkos/kokkos/pull/5105) + +### General Enhancements +- Implement `OffsetView` constructor taking `pair`s and `ViewCtorProp` [\#5303](https://github.com/kokkos/kokkos/pull/5303) +- Promote math constants to `Kokkos::numbers` namespace [\#5434](https://github.com/kokkos/kokkos/pull/5434) +- Add overloads of `hypot` math function that take 3 arguments [\#5341](https://github.com/kokkos/kokkos/pull/5341) +- Add `fma` fused multiply-add math function [\#5428](https://github.com/kokkos/kokkos/pull/5428) +- Views using `MemoryTraits::Atomic` don't need `volatile` overloads for the value type anymore. [\#5455](https://github.com/kokkos/kokkos/pull/5455) +- Added `is_team_handle` trait [\#5375](https://github.com/kokkos/kokkos/pull/5375) +- Refactor desul atomics to support compiling CUDA with NVC++ [\#5431](https://github.com/kokkos/kokkos/pull/5431) [\#5497](https://github.com/kokkos/kokkos/pull/5497) [\#5498](https://github.com/kokkos/kokkos/pull/5498) +- Support finding `libquadmath` with native compiler support [\#5286](https://github.com/kokkos/kokkos/pull/5286) +- Add architecture flags for MSVC [\#5673](https://github.com/kokkos/kokkos/pull/5673) +- SIMD backend for ARM NEON [\#5829](https://github.com/kokkos/kokkos/pull/5829) + +### Build System Changes +- Let CMake determine OpenMP flags. [\#4105](https://github.com/kokkos/kokkos/pull/4105) +- Update minimum compiler versions. [\#5323](https://github.com/kokkos/kokkos/pull/5323) +- Makefile and CMake support for C++23 [\#5283](https://github.com/kokkos/kokkos/pull/5283) +- Do not add `-cuda` to the link line with NVHPC compiler when the CUDA backend is not actually enabled [\#5485](https://github.com/kokkos/kokkos/pull/5485) +- Only add `-latomic` in generated GNU makefiles when OpenMPTarget backend is enabled [\#5501](https://github.com/kokkos/kokkos/pull/5501) [\#5537](https://github.com/kokkos/kokkos/pull/5537) (3.7 patch release candidate) +- `Kokkos_ENABLE_CUDA_LAMBDA` now `ON` by default with NVCC [\#5580](https://github.com/kokkos/kokkos/pull/5580) +- Fix enabling of relocatable device code when using CUDA as CMake language [\#5564](https://github.com/kokkos/kokkos/pull/5564) +- Fix cmake configuration with CUDA 12 [\#5691](https://github.com/kokkos/kokkos/pull/5691) + +### Incompatibilities (i.e. breaking changes) +- ***Require C++17*** [\#5277](https://github.com/kokkos/kokkos/pull/5277) +- Turn setting `Kokkos_CXX_STANDARD` into an error [\#5293](https://github.com/kokkos/kokkos/pull/5293) +- Remove all deprecations in Kokkos 3 [\#5297](https://github.com/kokkos/kokkos/pull/5297) +- Remove `KOKKOS_COMPILER_CUDA_VERSION` [\#5430](https://github.com/kokkos/kokkos/pull/5430) +- Drop `reciprocal_overflow_threshold` numeric trait [\#5326](https://github.com/kokkos/kokkos/pull/5326) +- Move `reduction_identity` out of `` into a new `` header [\#5450](https://github.com/kokkos/kokkos/pull/5450) +- Reduction and scan routines will report an error if the `join()` operator they would use takes `volatile`-qualified parameters [\#5409](https://github.com/kokkos/kokkos/pull/5409) +- `ENABLE_CUDA_UVM` is dropped in favor of using `SharedSpace` as `MemorySpace` explicitly [\#5608](https://github.com/kokkos/kokkos/pull/5608) +- Remove Kokkos_ENABLE_CUDA_LDG_INTRINSIC option [\#5623](https://github.com/kokkos/kokkos/pull/5623) +- Don't rely on synchronization behavior of default stream in CUDA and HIP - this potentially will break unintended implicit synchronization with other libraries such as MPI [\#5391](https://github.com/kokkos/kokkos/pull/5391) +- Make ExecutionSpace::concurrency() a non-static member function [\#5655](https://github.com/kokkos/kokkos/pull/5655) and related PRs +- Remove code guarded by `KOKKOS_ENABLE_DEPRECATED_CODE_3` + +### Deprecations +- Deprecate `CudaUVMSpace::available()` which always returned `true` [\#5614](https://github.com/kokkos/kokkos/pull/5614) +- Deprecate `volatile`-qualified members from `Kokkos::pair` and `Kokkos::complex` [\#5412](https://github.com/kokkos/kokkos/pull/5412) +- Deprecate `KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_*` macros [\#5824](https://github.com/kokkos/kokkos/pull/5824) (oversight in 3.6) + +### Bug Fixes +- Avoid allocating memory for `UniqueToken` [\#5300](https://github.com/kokkos/kokkos/pull/5300) +- Fix `pragma ivdep` in `Kokkos_OpenMP_Parallel.hpp` [\#5356](https://github.com/kokkos/kokkos/pull/5356) +- Fix configuring with Threads support when rerunning CMake [\#5486](https://github.com/kokkos/kokkos/pull/5486) +- Fix View assignment between `LayoutLeft` and `LayoutRight` with static extents [\#5535](https://github.com/kokkos/kokkos/pull/5535) (3.7 patch release candidate) +- Add `fence()` calls to sorting routine overloads that don't take an execution space parameter [\#5389](https://github.com/kokkos/kokkos/pull/5389) +- `ClockTic` changed to 64 bit to fix overflow on Power [\#5577](https://github.com/kokkos/kokkos/pull/5577) (incl. in 3.7.01 patch release) +- Fix incorrect offset in CUDA and HIP `parallel_scan` for < 4 byte types [\#5555](https://github.com/kokkos/kokkos/pull/5555) (3.7 patch release candidate) +- Fix incorrect alignment behavior of scratch allocations in some corner cases (e.g. very small allocations) [\#5687](https://github.com/kokkos/kokkos/pull/5687) (3.7 patch release candidate) +- Add missing `ReductionIdentity` specialization [\#5798](https://github.com/kokkos/kokkos/pull/5798) +- Don't install standard algorithms headers multiple times [\#5670](https://github.com/kokkos/kokkos/pull/5670) +- Fix max scratch size calculation for level 0 scratch in CUDA and HIP [\#5718](https://github.com/kokkos/kokkos/pull/5718) + +## [3.7.02](https://github.com/kokkos/kokkos/tree/3.7.02) (2023-05-17) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.7.01...3.7.02) + +### Backends and Archs Enhancements: +#### CUDA +- Add Hopper support and update nvcc_wrapper to work with CUDA-12 [\#5693](https://github.com/kokkos/kokkos/pull/5693) +### General Enhancements: +- sprintf -> snprintf [\#5787](https://github.com/kokkos/kokkos/pull/5787) +### Build System: +- Add error message when not using `hipcc` and when `CMAKE_CXX_STANDARD` is not set [\#5945](https://github.com/kokkos/kokkos/pull/5945) +### Bug Fixes: +- Fix Scratch allocation alignment issues [\#5692](https://github.com/kokkos/kokkos/pull/5692) +- Fix Intel Classic Compiler ICE [\#5710](https://github.com/kokkos/kokkos/pull/5710) +- Don't install std algorithm headers multiple times [\#5711](https://github.com/kokkos/kokkos/pull/5711) +- Fix static init order issue in InitalizationSettings [\#5721](https://github.com/kokkos/kokkos/pull/5721) +- Fix src/dst Properties in deep_copy(DynamicView,View) [\#5732](https://github.com/kokkos/kokkos/pull/5732) +- Fix build on Fedora Rawhide [\#5782](https://github.com/kokkos/kokkos/pull/5782) +- Finalize HIP lock arrays [\#5694](https://github.com/kokkos/kokkos/pull/5694) +- Fix CUDA lock arrays for current Desul [\#5812](https://github.com/kokkos/kokkos/pull/5812) +- Set the correct device/context in InterOp tests [\#5701](https://github.com/kokkos/kokkos/pull/5701) + +## [3.7.01](https://github.com/kokkos/kokkos/tree/3.7.01) (2022-12-01) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.7.00...3.7.01) + +### Bug Fixes: +- Add fences to all sorting routines not taking an execution space instance argument [\#5547](https://github.com/kokkos/kokkos/pull/5547) +- Fix repeated `team_reduce` without barrier [\#5552](https://github.com/kokkos/kokkos/pull/5552) +- Fix memory spaces in `create_mirror_view` overloads using `view_alloc` [\#5521](https://github.com/kokkos/kokkos/pull/5521) +- Allow `as_view_of_rank_n()` to be overloaded for "special" scalar types [\#5553](https://github.com/kokkos/kokkos/pull/5553) +- Fix warning calling a `__host__` function from a `__host__ __device__` from `View:: as_view_of_rank_n` [\#5591](https://github.com/kokkos/kokkos/pull/5591) +- OpenMPTarget: adding implementation to set device id. [\#5557](https://github.com/kokkos/kokkos/pull/5557) +- Use `Kokkos::atomic_load` to Correct Race Condition Giving Rise to Seg Faulting Error in OpenMP tests [\#5559](https://github.com/kokkos/kokkos/pull/5559) +- cmake: define `KOKKOS_ARCH_A64FX` [\#5561](https://github.com/kokkos/kokkos/pull/5561) +- Only link against libatomic in gnu-make OpenMPTarget build [\#5565](https://github.com/kokkos/kokkos/pull/5565) +- Fix static extents assignment for LayoutLeft/LayoutRight assignment [\#5566](https://github.com/kokkos/kokkos/pull/5566) +- Do not add -cuda to the link line with NVHPC compiler when the CUDA backend is not actually enabled [\#5569](https://github.com/kokkos/kokkos/pull/5569) +- Export the flags in `KOKKOS_AMDGPU_OPTIONS` when using Trilinos [\#5571](https://github.com/kokkos/kokkos/pull/5571) +- Add support for detecting MPI local rank with MPICH and PMI [\#5570](https://github.com/kokkos/kokkos/pull/5570) [\#5582](https://github.com/kokkos/kokkos/pull/5582) +- Remove listing of undefined TPL dependencies [\#5573](https://github.com/kokkos/kokkos/pull/5573) +- ClockTic changed to 64 bit to fix overflow on Power [\#5592](https://github.com/kokkos/kokkos/pull/5592) +- Fix incorrect offset in CUDA and HIP parallel scan for < 4 byte types [\#5607](https://github.com/kokkos/kokkos/pull/5607) +- Fix initialization of Cuda lock arrays [\#5622](https://github.com/kokkos/kokkos/pull/5622) + +## [3.7.00](https://github.com/kokkos/kokkos/tree/3.7.00) (2022-08-22) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.6.01...3.7.00) + +### Features: +- Use non-volatile `join()` member functions and `operator+=` in `parallel_reduce/scan` [\#4931](https://github.com/kokkos/kokkos/pull/4931) [\#4954](https://github.com/kokkos/kokkos/pull/4954) [\#4951](https://github.com/kokkos/kokkos/pull/4951) +- Add `SIMD` sub package (requires C++17) [\#5016](https://github.com/kokkos/kokkos/pull/5016) +- Add `is_finalized()` [\#5247](https://github.com/kokkos/kokkos/pull/5247) +- Promote mathematical functions from `namespace Kokkos::Experimental` to `namespace Kokkos` [\#4791](https://github.com/kokkos/kokkos/pull/4791) +- Promote `min`, `max`, `clamp`, `minmax` functions from `namespace Kokkos::Experimental` to `namespace Kokkos` [\#5170](https://github.com/kokkos/kokkos/pull/5170) +- Add `round`, `logb`, `nextafter`, `copysign`, and `signbit` math functions [\#4768](https://github.com/kokkos/kokkos/pull/4768) +- Add `HIPManagedSpace`, similar to `CudaUVMSpace` [\#5112](https://github.com/kokkos/kokkos/pull/5112) +- Accept view construction allocation properties in `create_mirror[_view,_view_and_copy]` and `resize/realloc` [\#5125](https://github.com/kokkos/kokkos/pull/5125) [\#5095](https://github.com/kokkos/kokkos/pull/5095) [\#5035](https://github.com/kokkos/kokkos/pull/5035) [\#4805](https://github.com/kokkos/kokkos/pull/4805) [\#4844](https://github.com/kokkos/kokkos/pull/4844) +- Allow `MemorySpace::allocate()` to be called with execution space [\#4826](https://github.com/kokkos/kokkos/pull/4826) +- Experimental: Compile time view subscriber [\#4197](https://github.com/kokkos/kokkos/pull/4197) + +### Backends and Archs Enhancements: +- Add support for Sapphire Rapids Intel architecture [\#5015](https://github.com/kokkos/kokkos/pull/5015) +- Add support for ICX, SKL and ICL Intel architectures [\#5013](https://github.com/kokkos/kokkos/pull/5013) [\#4929](https://github.com/kokkos/kokkos/pull/4929) +- Add arch flags for Intel GPU Ponte Vecchio [\#4932](https://github.com/kokkos/kokkos/pull/4932) +- SYCL: require GPU if GPU architecture was set at configuration time (i.e. do not allow fallback to CPU device) [\#5264](https://github.com/kokkos/kokkos/pull/5264) [\#5222](https://github.com/kokkos/kokkos/pull/5222) +- SYCL: Add `SYCL::sycl_queue()` for interoperability [\#5241](https://github.com/kokkos/kokkos/pull/5241) +- SYCL: Loosen restriction for using built-in `sycl::group_broadcast` [\#4552](https://github.com/kokkos/kokkos/pull/4552) +- SYCL: preserve address space [\#4396](https://github.com/kokkos/kokkos/pull/4396) +- OpenMPTarget: Adding a workaound for team scan [\#5219](https://github.com/kokkos/kokkos/pull/5219) +- OpenMPTarget: Adding logic to skip the kernel launch if `league_size=0` [\#5067](https://github.com/kokkos/kokkos/pull/5067) +- OpenMPTarget: Make sure `Kokkos::abort()` causes abnormal program termination when called on the host-side [\#4808](https://github.com/kokkos/kokkos/pull/4808) +- HIP: Make HIPHostPinnedSpace coarse-grained [\#5152](https://github.com/kokkos/kokkos/pull/5152) +- Refactor OpenMP `parallel_for` implementation to use more native OpenMP constructs [\#4664](https://github.com/kokkos/kokkos/pull/4664) +- Add option to optimize for local CPU architecture `Kokkos_ARCH_NATIVE` [\#4930](https://github.com/kokkos/kokkos/pull/4930) + + +### Implemented enhancements +- Add command line argument/environment variable to print the configuration [\#5233](https://github.com/kokkos/kokkos/pull/5233) +- Improve error message in view memory access violations [\#4950](https://github.com/kokkos/kokkos/pull/4950) +- Remove unnecessary fences in View initialization [\#4823](https://github.com/kokkos/kokkos/pull/4823) +- Make `View::shmem_size()` device-callable [\#4936](https://github.com/kokkos/kokkos/pull/4936) +- Update numerics support for `__float128` [\#5081](https://github.com/kokkos/kokkos/pull/5081) +- Add `log10` overload for `Kokkos::complex` [\#5009](https://github.com/kokkos/kokkos/pull/5009) +- Add `[[nodiscard]]` to `ScopeGuard` [\#5224](https://github.com/kokkos/kokkos/pull/5224) +- Add structured binding support for `Kokkos::Array` [\#4962](https://github.com/kokkos/kokkos/pull/4962) +- Enable accessing `Kokkos::Array` elements in constant expressions [\#4916](https://github.com/kokkos/kokkos/pull/4916) +- Mark `as_view_of_rank_n` as KOKKOS_FUNCTION [\#5248](https://github.com/kokkos/kokkos/pull/5248) +- Cleanup/rework fence overloads [\#5148](https://github.com/kokkos/kokkos/pull/5148) +- Assert that `Layout` construction from extents is valid in functions taking integer extents [\#5209](https://github.com/kokkos/kokkos/pull/5209) +- Add `fill_random` overload that takes an execution space as first argument [\#5181](https://github.com/kokkos/kokkos/pull/5181) +- Avoid some unnecessary fences in `parallel_reduce/scan` [\#5154](https://github.com/kokkos/kokkos/pull/5154) +- Include `KOKKOS_ENABLE_LIBDL` in options when printing configuration [\#5086](https://github.com/kokkos/kokkos/pull/5086) +- DynRankView: make `layout()` return the same as a corresponding static View [\#5026](https://github.com/kokkos/kokkos/pull/5026) +- Use `_mm_malloc` for icpx [\#5012](https://github.com/kokkos/kokkos/pull/5012) +- Avoid forcing matching execution spaces in `BinSort` constructor and `sort()` [\#4919](https://github.com/kokkos/kokkos/pull/4919) +- Check number of bins in `BinSort` [\#4890](https://github.com/kokkos/kokkos/pull/4890) +- Improve performance in parallel STL-like algorithms [\#4887](https://github.com/kokkos/kokkos/pull/4887) [\#4886](https://github.com/kokkos/kokkos/pull/4886) +- Disable `memset` on A64FX and launch `parallel_for` instead (performance) [\#4884](https://github.com/kokkos/kokkos/pull/4884) +- Allow non-power-of-two team sizes for team reductions and scans [\#4809](https://github.com/kokkos/kokkos/pull/4809) + +#### Harmonization of Kokkos execution environment initialization: +- Warn when unable to detect local MPI rank and user explicitly asked for it [\#5263](https://github.com/kokkos/kokkos/pull/5263) +- Refactor parsing of command line arguments and environment variables [\#5221](https://github.com/kokkos/kokkos/pull/5221) +- Refactor device selection at initialization [\#5211](https://github.com/kokkos/kokkos/pull/5211) +- Rename tools settings for consistency [\#5201](https://github.com/kokkos/kokkos/pull/5201) +- Print help only once [\#5128](https://github.com/kokkos/kokkos/pull/5128) +- Update precedence rule in initialization [\#5130](https://github.com/kokkos/kokkos/pull/5130) +- Warn instead of just ignoring user settings when kokkos-tools is disabled [\#5088](https://github.com/kokkos/kokkos/pull/5088) +- Drop numa args in threads backend initialization [\#5127](https://github.com/kokkos/kokkos/pull/5127) +- Warn users when a flag prefixed with -[-]kokkos is not recognized and do not remove it [\#5256](https://github.com/kokkos/kokkos/pull/5256) +- Give back to Core what belongs to Core (aka moving tune_internals option from Tools back to Core) [\#5202](https://github.com/kokkos/kokkos/pull/5202) + +#### Build system updates: +- `nvcc_wrapper`: filter out -pedantic-errors from nvcc options [\#5235](https://github.com/kokkos/kokkos/pull/5235) +- `nvcc_wrapper`: add known nvcc option --source-in-ptx [\#5052](https://github.com/kokkos/kokkos/pull/5052) +- Link libdl as interface library [\#5179](https://github.com/kokkos/kokkos/pull/5179) +- Only show GPU architectures with enabled corresponding backend [\#5119](https://github.com/kokkos/kokkos/pull/5119) +- Enable optional external desul build [\#5021](https://github.com/kokkos/kokkos/pull/5021) [\#5132](https://github.com/kokkos/kokkos/pull/5132) +- Export `Kokkos_CXX_STANDARD` variable with CMake [\#5068](https://github.com/kokkos/kokkos/pull/5068) +- Suppress warnings with nvc++ [\#5031](https://github.com/kokkos/kokkos/pull/5031) +- Disallow multiple host architectures in CMake [\#4996](https://github.com/kokkos/kokkos/pull/4996) +- Do not include compiler warning flags in the compile option of the cmake target [\#4989](https://github.com/kokkos/kokkos/pull/4989) +- AOT flags for OpenMPTarget targeting Intel GPUs [\#4915](https://github.com/kokkos/kokkos/pull/4915) +- Repurpose `Kokkos_ARCH_INTEL_GEN` for SYCL to mean JIT to be conforming with OMPT [\#4894](https://github.com/kokkos/kokkos/pull/4894) +- Replace amdgpu-target with offload-arch [\#4874](https://github.com/kokkos/kokkos/pull/4874) +- Do not enable `kokkos_launch_compiler` when `CMAKE_CXX_COMPILER_LAUNCHER` is set [\#4870](https://github.com/kokkos/kokkos/pull/4870) +- Move CMake version check up [\#4797](https://github.com/kokkos/kokkos/pull/4797) + +### Incompatibilities: +- Remove `KOKKOS_THREAD_LOCAL` [\#5064](https://github.com/kokkos/kokkos/pull/5064) +- Remove `KOKKOS_ENABLE_POSIX_MEMALIGN` [\#5011](https://github.com/kokkos/kokkos/pull/5011) +- Remove unused `KOKKOS_ENABLE_TM` [\#4995](https://github.com/kokkos/kokkos/pull/4995) +- Remove unused cmakedefine `KOKKOS_ENABLE_COMPILER_WARNINGS` [\#4883](https://github.com/kokkos/kokkos/pull/4883) +- Remove unused `KOKKOS_ENABLE_DUALVIEW_MODIFY_CHECK` [\#4882](https://github.com/kokkos/kokkos/pull/4882) +- Drop Instruction Set Architecture (ISA) macros [\#4981](https://github.com/kokkos/kokkos/pull/4981) +- Warn in `ScopeGuard` about illegal usage [\#5250](https://github.com/kokkos/kokkos/pull/5250) + +### Deprecations: +- Guard against non-public header inclusion [\#5178](https://github.com/kokkos/kokkos/pull/5178) +- Raise deprecation warnings if non empty WorkTag class is used [\#5230](https://github.com/kokkos/kokkos/pull/5230) +- Deprecate `parallel_*` overloads taking the label as trailing argument [\#5141](https://github.com/kokkos/kokkos/pull/5141) +- Deprecate nested types in functional [\#5185](https://github.com/kokkos/kokkos/pull/5185) +- Deprecate `InitArguments` struct and replace it with `InitializationSettings` [\#5135](https://github.com/kokkos/kokkos/pull/5135) +- Deprecate `finalize_all()` [\#5134](https://github.com/kokkos/kokkos/pull/5134) +- Deprecate command line arguments (other than `--help`) that are not prefixed with `kokkos-*` [\#5120](https://github.com/kokkos/kokkos/pull/5120) +- Deprecate `--[kokkos-]numa` cmdline arg and `KOKKOS_NUMA` env var [\#5117](https://github.com/kokkos/kokkos/pull/5117) +- Deprecate `--[kokkos-]threads` command line argument in favor of `--[kokkos-]num-threads` [\#5111](https://github.com/kokkos/kokkos/pull/5111) +- Deprecate `Kokkos::is_reducer_type` [\#4957](https://github.com/kokkos/kokkos/pull/4957) +- Deprecate `OffsetView` constructors taking `index_list_type` [\#4810](https://github.com/kokkos/kokkos/pull/4810) +- Deprecate overloads of `Kokkos::sort` taking a parameter `bool always_use_kokkos_sort` [\#5382](https://github.com/kokkos/kokkos/issues/5382) +- Warn about `parallel_reduce` cases that call `join()` with volatile-qualified arguments [\#5215](https://github.com/kokkos/kokkos/pull/5215) + +### Bug Fixes: +- CUDA Reductions: Fix data races reported by Nvidia `compute-sanitizer` [\#4855](https://github.com/kokkos/kokkos/pull/4855) +- Work around Intel compiler bug [\#5301](https://github.com/kokkos/kokkos/pull/5301) +- Avoid allocating memory for UniqueToken [\#5300](https://github.com/kokkos/kokkos/pull/5300) +- DynamicView: Properly resize mirror instances after construction [\#5276](https://github.com/kokkos/kokkos/pull/5276) +- Remove Kokkos::Rank limit of 6 ranks [\#5271](https://github.com/kokkos/kokkos/pull/5271) +- Do not forget to set last element to nullptr when removing a flag in `Kokkos::initialize` [\#5272](https://github.com/kokkos/kokkos/pull/5272) +- Fix CUDA+MSVC build issue [\#5261](https://github.com/kokkos/kokkos/pull/5261) +- Fix `DynamicView::resize_serial` [\#5220](https://github.com/kokkos/kokkos/pull/5220) +- Fix cmake default compiler flags for unknown compiler [\#5217](https://github.com/kokkos/kokkos/pull/5217) +- Fix `move_backward` [\#5191](https://github.com/kokkos/kokkos/pull/5191) +- Fixing issue 5196 - missing symbol with intel compiler [\#5207](https://github.com/kokkos/kokkos/pull/5207) +- Preserve `KOKKOS_INVALID_INDEX` in ViewDimension and ArrayLayout construction [\#5188](https://github.com/kokkos/kokkos/pull/5188) +- Finalize `deep_copy_space` early avoiding printing to `std::cerr` for Cuda [\#5151](https://github.com/kokkos/kokkos/pull/5151) +- Use correct policy in Threads MDRange `parallel_reduce` [\#5123](https://github.com/kokkos/kokkos/pull/5123) +- Fix building with NVCC as the CXX compiler while the CUDA backend is not enabled [\#5115](https://github.com/kokkos/kokkos/pull/5115) +- OpenMPTarget Index range fix for MDRange. [\#5089](https://github.com/kokkos/kokkos/pull/5089) +- Fix bug with CUDA's team reduction for empty ranges [\#5079](https://github.com/kokkos/kokkos/pull/5079) +- Fix using `ZeroMemset` for Serial [\#5077](https://github.com/kokkos/kokkos/pull/5077) +- Fix `Kokkos::Vector::push_back` for default execution space [\#5047](https://github.com/kokkos/kokkos/pull/5047) +- ScatterView: Fix ScatterMin/ScatterMax to use proper atomics [\#5045](https://github.com/kokkos/kokkos/pull/5045) +- Fix calling `ZeroMemset` in `deep_copy` [\#5040](https://github.com/kokkos/kokkos/pull/5040) +- Make View self-assignment not produce double-free [\#5024](https://github.com/kokkos/kokkos/pull/5024) +- Guard against unrecognized pragma with intel compilers [\#5019](https://github.com/kokkos/kokkos/pull/5019) +- Fix racing condition in `HIPParallelLaunch` [\#5008](https://github.com/kokkos/kokkos/pull/5008) +- KokkosP: Fix `device_id` in profiling [\#4997](https://github.com/kokkos/kokkos/pull/4997) +- Fix for `Kokkos::vector::insert` into empty vector with begin and end iterators [\#4988](https://github.com/kokkos/kokkos/pull/4988) +- Fix Core header files installation [\#4984](https://github.com/kokkos/kokkos/pull/4984) +- Fix bounds errors with `Kokkos::sort` [\#4980](https://github.com/kokkos/kokkos/pull/4980) +- Fixup let `RangePolicy::set_chunk_size` return a reference to self [\#4918](https://github.com/kokkos/kokkos/pull/4918) +- Fix allocating large Views [\#4907](https://github.com/kokkos/kokkos/pull/4907) +- Fix combined reductions with `Kokkos::View` [\#4896](https://github.com/kokkos/kokkos/pull/4896) +- Fixed `_CUDA_ARCH__` to `__CUDA_ARCH__` for CUDA LDG [\#4893](https://github.com/kokkos/kokkos/pull/4893) +- Fixup `View::access()` truncate parameter pack [\#4876](https://github.com/kokkos/kokkos/pull/4876) +- Fix `abort` with HIP backend for ROCm 5.0.2 and beyond [\#4873](https://github.com/kokkos/kokkos/pull/4873) +- Fix HIP version when printing the configuration [\#4872](https://github.com/kokkos/kokkos/pull/4872) +- Fix scratch lock array when using scratch level 1 [\#4871](https://github.com/kokkos/kokkos/pull/4871) +- Fix Makefile.kokkos to work with fujitsu compiler [\#4867](https://github.com/kokkos/kokkos/pull/4867) +- cmake: Correct link THREADS link option [\#4854](https://github.com/kokkos/kokkos/pull/4854) +- UniqueToken `impl_acquire` function should be device only [\#4819](https://github.com/kokkos/kokkos/pull/4819) +- Fix example calls to non existing static `print_configuration` [\#4806](https://github.com/kokkos/kokkos/pull/4806) +- Fix requests for large team scratch sizes [\#4728](https://github.com/kokkos/kokkos/pull/4728) + + +## [3.6.01](https://github.com/kokkos/kokkos/tree/3.6.01) (2022-05-23) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.6.00...3.6.01) + +### Bug Fixes: +- Fix Threads: Fix serial resizing scratch space (3.6.01 cherry-pick) [\#5109](https://github.com/kokkos/kokkos/pull/5109) +- Fix ScatterMin/ScatterMax to use proper atomics (3.6.01 cherry-pick) [\#5046](https://github.com/kokkos/kokkos/pull/5046) +- Fix allocating large Views [\#4907](https://github.com/kokkos/kokkos/pull/4907) +- Fix bounds errors with Kokkos::sort [\#4980](https://github.com/kokkos/kokkos/pull/4980) +- Fix HIP version when printing the configuration [\#4872](https://github.com/kokkos/kokkos/pull/4872) +- Fixed `_CUDA_ARCH__` to `__CUDA_ARCH__` for CUDA LDG [\#4893](https://github.com/kokkos/kokkos/pull/4893) +- Fixed an incorrect struct initialization [\#5028](https://github.com/kokkos/kokkos/pull/5028) +- Fix racing condition in `HIPParallelLaunch` [\#5008](https://github.com/kokkos/kokkos/pull/5008) +- Avoid deprecation warnings with `OpenMPExec::validate_partition` [\#4982](https://github.com/kokkos/kokkos/pull/4982) +- Make View self-assignment not produce double-free [\#5024](https://github.com/kokkos/kokkos/pull/5024) + + +## [3.6.00](https://github.com/kokkos/kokkos/tree/3.6.00) (2022-02-18) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.5.00...3.6.00) + +### Features: +- Add C++ standard algorithms [\#4315](https://github.com/kokkos/kokkos/pull/4315) +- Implement `fill_random` for `DynRankView` [\#4763](https://github.com/kokkos/kokkos/pull/4763) +- Add `bhalf_t` [\#4543](https://github.com/kokkos/kokkos/pull/4543) [\#4653](https://github.com/kokkos/kokkos/pull/4653) +- Add mathematical constants [\#4519](https://github.com/kokkos/kokkos/pull/4519) +- Allow `Kokkos::{create_mirror*,resize,realloc}` to be used with `WithoutInitializing` [\#4486](https://github.com/kokkos/kokkos/pull/4486) [\#4337](https://github.com/kokkos/kokkos/pull/4337) +- Implement `KOKKOS_IF_ON_{HOST,DEVICE}` macros [\#4660](https://github.com/kokkos/kokkos/pull/4660) +- Allow setting the CMake language for Kokkos [\#4323](https://github.com/kokkos/kokkos/pull/4323) + +#### Perf bug fix +- Desul: Add ScopeCaller [\#4690](https://github.com/kokkos/kokkos/pull/4690) +- Enable Desul atomics by default when using Makefiles [\#4606](https://github.com/kokkos/kokkos/pull/4606) +- Unique token improvement [\#4741](https://github.com/kokkos/kokkos/pull/4741) [\#4748](https://github.com/kokkos/kokkos/pull/4748) + +#### Other improvements: +- Add math function long double overload on the host side [\#4712](https://github.com/kokkos/kokkos/pull/4712) + +### Deprecations: +- Array reductions with pointer return types [\#4756](https://github.com/kokkos/kokkos/pull/4756) +- Deprecate `partition_master`, `validate_partition` [\#4737](https://github.com/kokkos/kokkos/pull/4737) +- Deprecate `Kokkos_ENABLE_PTHREAD` in favor of `Kokkos_ENABLE_THREADS` [\#4619](https://github.com/kokkos/kokkos/pull/4619) ** pair with use std::threads ** +- Deprecate `log2(unsigned) -> int` (removing in next release) [\#4595](https://github.com/kokkos/kokkos/pull/4595) +- Deprecate `Kokkos::Impl::is_view` [\#4592](https://github.com/kokkos/kokkos/pull/4592) +- Deprecate `KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_*` macros and the `ActiveExecutionMemorySpace` alias [\#4668](https://github.com/kokkos/kokkos/issues/4668) + +### Backends and Archs Enhancements: + +#### SYCL: +- Update required SYCL compiler version [\#4749](https://github.com/kokkos/kokkos/pull/4749) +- Cap vector size to kernel maximum for SYCL [\#4704](https://github.com/kokkos/kokkos/pull/4704) +- Improve check for compatibility of vector size and subgroup size in SYCL [\#4579](https://github.com/kokkos/kokkos/pull/4579) +- Provide `chunk_size` for SYCL [\#4635](https://github.com/kokkos/kokkos/pull/4635) +- Use host-pinned memory for SYCL kernel memory [\#4627](https://github.com/kokkos/kokkos/pull/4627) +- Use shuffle-based algorithm for scalar reduction [\#4608](https://github.com/kokkos/kokkos/pull/4608) +- Implement pool of USM IndirectKernelMemory [\#4596](https://github.com/kokkos/kokkos/pull/4596) +- Provide valid default team size for SYCL [\#4481](https://github.com/kokkos/kokkos/pull/4481) + +#### CUDA: +- Add checks for shmem usage in `parallel_reduce` [\#4548](https://github.com/kokkos/kokkos/pull/4548) + +#### HIP: +- Add support for fp16 in the HIP backend [\#4688](https://github.com/kokkos/kokkos/pull/4688) +- Disable multiple kernel instantiations when using HIP (configure with `-DKokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS=ON` to use) [\#4644](https://github.com/kokkos/kokkos/pull/4644) +- Fix HIP scratch use per instance [\#4439](https://github.com/kokkos/kokkos/pull/4439) +- Change allocation header to 256B alignment for AMD VEGA architecture [\#4753](https://github.com/kokkos/kokkos/pull/4753) +- Add generic `KOKKOS_ARCH_VEGA` macro [\#4782](https://github.com/kokkos/kokkos/pull/4782) +- Require ROCm 4.5 [\#4689](https://github.com/kokkos/kokkos/pull/4689) + +### HPX: +- Adapt to HPX 1.7.0 which is now required [\#4241](https://github.com/kokkos/kokkos/pull/4241) + +#### OpenMP: +- Fix thread deduction for OpenMP for `thread_count==0` [\#4541](https://github.com/kokkos/kokkos/pull/4541) + +#### OpenMPTarget: +- Update memory space `size_type` to improve performance (`size_t -> unsigned`) [\#4779](https://github.com/kokkos/kokkos/pull/4779) + +#### Other Improvements: +- Improve NVHPC support [\#4599](https://github.com/kokkos/kokkos/pull/4599) +- Add `Kokkos::Experimental::{min,max,minmax,clamp}` [\#4629](https://github.com/kokkos/kokkos/pull/4629) [\#4506](https://github.com/kokkos/kokkos/pull/4506) +- Use device type as template argument in Containers and Algorithms [\#4724](https://github.com/kokkos/kokkos/pull/4724) [\#4675](https://github.com/kokkos/kokkos/pull/4675) +- Implement `Kokkos::sort` with execution space [\#4490](https://github.com/kokkos/kokkos/pull/4490) +- `Kokkos::resize` always error out for mismatch in runtime rank [\#4681](https://github.com/kokkos/kokkos/pull/4681) +- Print current call stack when calling `Kokkos::abort()` from the host [\#4672](https://github.com/kokkos/kokkos/pull/4672) [\#4671](https://github.com/kokkos/kokkos/pull/4671) +- Detect mismatch of execution spaces in functors [\#4655](https://github.com/kokkos/kokkos/pull/4655) +- Improve view label access on host [\#4647](https://github.com/kokkos/kokkos/pull/4647) +- Error out for `const` scalar return type in reduction [\#4645](https://github.com/kokkos/kokkos/pull/4645) +- Don't allow calling `UnorderdMap::value_at` for a set [\#4639](https://github.com/kokkos/kokkos/pull/4639) +- Add `KOKKOS_COMPILER_NVHPC` macro, disable `quiet_NaN` and `signaling_NaN` [\#4586](https://github.com/kokkos/kokkos/pull/4586) +- Improve performance of `local_deep_copy` [\#4511](https://github.com/kokkos/kokkos/pull/4511) +- Improve performance when sorting integers [\#4464](https://github.com/kokkos/kokkos/pull/4464) +- Add missing numeric traits (`denorm_min`, `reciprocal_overflow_threshold`, `{quiet,silent}_NaN}`) and make them work on cv-qualified types [\#4466](https://github.com/kokkos/kokkos/pull/4466) [\#4415](https://github.com/kokkos/kokkos/pull/4415) [\#4473](https://github.com/kokkos/kokkos/pull/4473) [\#4443](https://github.com/kokkos/kokkos/pull/4443) + +### Implemented enhancements BuildSystem +- Manually compute IntelLLVM compiler version for older CMake versions [\#4760](https://github.com/kokkos/kokkos/pull/4760) +- Add Xptxas without = to `nvcc_wrapper` [\#4646](https://github.com/kokkos/kokkos/pull/4646) +- Use external GoogleTest optionally [\#4563](https://github.com/kokkos/kokkos/pull/4563) +- Silent warnings about multiple optimization flags with `nvcc_wrapper` [\#4502](https://github.com/kokkos/kokkos/pull/4502) +- Use the same flags in Makefile.kokkos for POWER7/8/9 as for CMake [\#4483](https://github.com/kokkos/kokkos/pull/4483) +- Fix support for A64FX architecture [\#4745](https://github.com/kokkos/kokkos/pull/4745) + +### Incompatibilities: +- Drop `KOKKOS_ARCH_HIP` macro when using generated GNU makefiles [\#4786](https://github.com/kokkos/kokkos/pull/4786) +- Remove gcc-toolchain auto add for clang in Makefile.kokkos [\#4762](https://github.com/kokkos/kokkos/pull/4762) + +### Bug Fixes: +- Lock constant memory in Cuda/HIP kernel launch with a mutex (thread safety) [\#4525](https://github.com/kokkos/kokkos/pull/4525) +- Fix overflow for large requested scratch allocation [\#4551](https://github.com/kokkos/kokkos/pull/4551) +- Fix Windows build in mingw [\#4564](https://github.com/kokkos/kokkos/pull/4564) +- Fix `kokkos_launch_compiler`: escape `$` character [\#4769](https://github.com/kokkos/kokkos/pull/4769) [\#4703](https://github.com/kokkos/kokkos/pull/4703) +- Fix math functions with NVCC and GCC 5 as host compiler [\#4733](https://github.com/kokkos/kokkos/pull/4733) +- Fix shared build with Intel19 [\#4725](https://github.com/kokkos/kokkos/pull/4725) +- Do not install empty `desul/src/` directory [\#4714](https://github.com/kokkos/kokkos/pull/4714) +- Fix wrong `device_id` computation in `identifier_from_devid` (Profiling Interface) [\#4694](https://github.com/kokkos/kokkos/pull/4694) +- Fix a bug in CUDA scratch memory pool (abnormally high memory consumption) [\#4673](https://github.com/kokkos/kokkos/pull/4673) +- Remove eval of command args in `hpcbind` [\#4630](https://github.com/kokkos/kokkos/pull/4630) +- SYCL fix to run when no GPU is detected [\#4623](https://github.com/kokkos/kokkos/pull/4623) +- Fix `layout_strides::span` for rank-0 views [\#4605](https://github.com/kokkos/kokkos/pull/4605) +- Fix SYCL atomics for local memory [\#4585](https://github.com/kokkos/kokkos/pull/4585) +- Hotfix `mdrange_large_deep_copy` for SYCL [\#4581](https://github.com/kokkos/kokkos/pull/4581) +- Fix bug when sorting integer using the HIP backend [\#4570](https://github.com/kokkos/kokkos/pull/4570) +- Fix compilation error when using HIP with RDC [\#4553](https://github.com/kokkos/kokkos/pull/4553) +- `DynamicView`: Fix deallocation extent [\#4533](https://github.com/kokkos/kokkos/pull/4533) +- SYCL fix running parallel_reduce with TeamPolicy for large ranges [\#4532](https://github.com/kokkos/kokkos/pull/4532) +- Fix bash syntax error in `nvcc_wrapper` [\#4524](https://github.com/kokkos/kokkos/pull/4524) +- OpenMPTarget `team_policy` reduce fixes for `init/join` reductions [\#4521](https://github.com/kokkos/kokkos/pull/4521) +- Avoid hangs in the Threads backend [\#4499](https://github.com/kokkos/kokkos/pull/4499) +- OpenMPTarget fix reduction bug in `parallel_reduce` for `TeamPolicy` [\#4491](https://github.com/kokkos/kokkos/pull/4491) +- HIP fix scratch space per instance [\#4439](https://github.com/kokkos/kokkos/pull/4439) +- OpenMPTarget fix team scratch allocation [\#4431](https://github.com/kokkos/kokkos/pull/4431) + + +## [3.5.00](https://github.com/kokkos/kokkos/tree/3.5.00) (2021-10-19) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.4.01...3.5.00) + +### Features: + +- Add support for quad-precision math functions/traits [\#4098](https://github.com/kokkos/kokkos/pull/4098) +- Adding ExecutionSpace partitioning function [\#4096](https://github.com/kokkos/kokkos/pull/4096) +- Improve Python Interop Capabilities [\#4065](https://github.com/kokkos/kokkos/pull/4065) +- Add half_t Kokkos::rand specialization [\#3922](https://github.com/kokkos/kokkos/pull/3922) +- Add math special functions: erf, erfcx, expint1, Bessel functions, Hankel functions [\#3920](https://github.com/kokkos/kokkos/pull/3920) +- Add missing common mathematical functions [\#4043](https://github.com/kokkos/kokkos/pull/4043) [\#4036](https://github.com/kokkos/kokkos/pull/4036) [\#4034](https://github.com/kokkos/kokkos/pull/4034) +- Let the numeric traits be SFINAE-friendly [\#4038](https://github.com/kokkos/kokkos/pull/4038) +- Add Desul atomics - enabling memory-order and memory-scope parameters [\#3247](https://github.com/kokkos/kokkos/pull/3247) +- Add detection idiom from the C++ standard library extension version 2 [\#3980](https://github.com/kokkos/kokkos/pull/3980) +- Fence Profiling Support in all backends [\#3966](https://github.com/kokkos/kokkos/pull/3966) [\#4304](https://github.com/kokkos/kokkos/pull/4304) [\#4258](https://github.com/kokkos/kokkos/pull/4258) [\#4232](https://github.com/kokkos/kokkos/pull/4232) +- Significant SYCL enhancements (see below) + +### Deprecations: + +- Deprecate CUDA_SAFE_CALL and HIP_SAFE_CALL [\#4249](https://github.com/kokkos/kokkos/pull/4249) +- Deprecate Kokkos::Impl::Timer (Kokkos::Timer has been available for a long time) [\#4201](https://github.com/kokkos/kokkos/pull/4201) +- Deprecate Experimental::MasterLock [\#4094](https://github.com/kokkos/kokkos/pull/4094) +- Deprecate Kokkos_TaskPolicy.hpp (headers got reorganized, doesn't remove functionality) [\#4011](https://github.com/kokkos/kokkos/pull/4011) +- Deprecate backward compatibility features [\#3978](https://github.com/kokkos/kokkos/pull/3978) +- Update and deprecate is_space::host_memory/execution/mirror_space [\#3973](https://github.com/kokkos/kokkos/pull/3973) + + +### Backends and Archs Enhancements: + +- Enabling constbitset constructors in kernels [\#4296](https://github.com/kokkos/kokkos/pull/4296) +- Use ZeroMemset in View constructor to improve performance [\#4226](https://github.com/kokkos/kokkos/pull/4226) +- Use memset in deep_copy [\#3944](https://github.com/kokkos/kokkos/pull/3944) +- Add missing fence() calls in resize(View) that effectively do deep_copy(resized, orig) [\#4212](https://github.com/kokkos/kokkos/pull/4212) +- Avoid allocations in resize and realloc [\#4207](https://github.com/kokkos/kokkos/pull/4207) +- StaticCsrGraph: use device type instead of execution space to construct views [\#3991](https://github.com/kokkos/kokkos/pull/3991) +- Consider std::sort when view is accessible from host [\#3929](https://github.com/kokkos/kokkos/pull/3929) +- Fix CPP20 warnings except for volatile [\#4312](https://github.com/kokkos/kokkos/pull/4312) + +#### SYCL: +- Introduce SYCLHostUSMSpace [\#4268](https://github.com/kokkos/kokkos/pull/4268) +- Implement SYCL TeamPolicy for vector_size > 1 [\#4183](https://github.com/kokkos/kokkos/pull/4183) +- Enable 64bit ranges for SYCL [\#4211](https://github.com/kokkos/kokkos/pull/4211) +- Don't print SYCL device info in execution space intialization [\#4168](https://github.com/kokkos/kokkos/pull/4168) +- Improve SYCL MDRangePolicy performance [\#4161](https://github.com/kokkos/kokkos/pull/4161) +- Use sub_groups in SYCL parallel_scan [\#4147](https://github.com/kokkos/kokkos/pull/4147) +- Implement subgroup reduction for SYCL RangePolicy parallel_reduce [\#3940](https://github.com/kokkos/kokkos/pull/3940) +- Use DPC++ broadcast extension in SYCL team_broadcast [\#4103](https://github.com/kokkos/kokkos/pull/4103) +- Only fence in SYCL parallel_reduce for non-device-accessible result_ptr [\#4089](https://github.com/kokkos/kokkos/pull/4089) +- Improve fencing behavior in SYCL backend [\#4088](https://github.com/kokkos/kokkos/pull/4088) +- Fence all registered SYCL queues before deallocating memory [\#4086](https://github.com/kokkos/kokkos/pull/4086) +- Implement SYCL::print_configuration [\#3992](https://github.com/kokkos/kokkos/pull/3992) +- Reuse scratch memory in parallel_scan and TeamPolicy (decreases memory footprint) [\#3899](https://github.com/kokkos/kokkos/pull/3899) [\#3889](https://github.com/kokkos/kokkos/pull/3889) + +#### CUDA: +- Cuda improve heuristic for blocksize [\#4271](https://github.com/kokkos/kokkos/pull/4271) +- Don't use [[deprecated]] for nvcc [\#4229](https://github.com/kokkos/kokkos/pull/4229) +- Improve error message for NVHPC as host compiler [\#4227](https://github.com/kokkos/kokkos/pull/4227) +- Update support for cuda reductions to work with types < 4bytes [\#4156](https://github.com/kokkos/kokkos/pull/4156) +- Fix incompatible team size deduction in rare cases parallel_reduce [\#4142](https://github.com/kokkos/kokkos/pull/4142) +- Remove UVM usage in DynamicView [\#4129](https://github.com/kokkos/kokkos/pull/4129) +- Remove dependency between core and containers [\#4114](https://github.com/kokkos/kokkos/pull/4114) +- Adding opt-in CudaMallocSync support when using CUDA version >= 11.2 [\#4026](https://github.com/kokkos/kokkos/pull/4026) [\#4233](https://github.com/kokkos/kokkos/pull/4233) +- Fix a potential race condition in the CUDA backend [\#3999](https://github.com/kokkos/kokkos/pull/3999) + +#### HIP: +- Implement new blocksize deduction method for HIP Backend [\#3953](https://github.com/kokkos/kokkos/pull/3953) +- Add multiple LaunchMechanism [\#3820](https://github.com/kokkos/kokkos/pull/3820) +- Make HIP backend thread-safe [\#4170](https://github.com/kokkos/kokkos/pull/4170) + +#### Serial: +- Refactor Serial backend and fix thread-safety issue [\#4053](https://github.com/kokkos/kokkos/pull/4053) + +#### OpenMPTarget: +- OpenMPTarget: support array reductions in RangePolicy [\#4040](https://github.com/kokkos/kokkos/pull/4040) +- OpenMPTarget: add MDRange parallel_reduce [\#4032](https://github.com/kokkos/kokkos/pull/4032) +- OpenMPTarget: Fix bug in for the case of a reducer. [\#4044](https://github.com/kokkos/kokkos/pull/4044) +- OpenMPTarget: verify process fix [\#4041](https://github.com/kokkos/kokkos/pull/4041) + +### Implemented enhancements BuildSystem + +#### Important BuildSystem Updates: +- Use hipcc architecture autodetection when Kokkos_ARCH is not set [\#3941](https://github.com/kokkos/kokkos/pull/3941) +- Introduce Kokkos_ENABLE_DEPRECATION_WARNINGS and remove deprecated code with Kokkos_ENABLE_DEPRECATED_CODE_3 [\#4106](https://github.com/kokkos/kokkos/pull/4106) [\#3855](https://github.com/kokkos/kokkos/pull/3855) + +#### Other Improvements: +- Add allow-unsupported-compiler flag to nvcc-wrapper [\#4298](https://github.com/kokkos/kokkos/pull/4298) +- nvcc_wrapper: fix errors in argument handling [\#3993](https://github.com/kokkos/kokkos/pull/3993) +- Adds support for -time= and -time in nvcc_wrapper [\#4015](https://github.com/kokkos/kokkos/pull/4015) +- nvcc_wrapper: suppress duplicates of GPU architecture and RDC flags [\#3968](https://github.com/kokkos/kokkos/pull/3968) +- Fix TMPDIR support in nvcc_wrapper [\#3792](https://github.com/kokkos/kokkos/pull/3792) +- NVHPC: update PGI compiler arch flags [\#4133](https://github.com/kokkos/kokkos/pull/4133) +- Replace PGI with NVHPC (works for both) [\#4196](https://github.com/kokkos/kokkos/pull/4196) +- Make sure that KOKKOS_CXX_HOST_COMPILER_ID is defined [\#4235](https://github.com/kokkos/kokkos/pull/4235) +- Add options to Makefile builds for deprecated code and warnings [\#4215](https://github.com/kokkos/kokkos/pull/4215) +- Use KOKKOS_CXX_HOST_COMPILER_ID for identifying CPU arch flags [\#4199](https://github.com/kokkos/kokkos/pull/4199) +- Added support for Cray Clang to Makefile.kokkos [\#4176](https://github.com/kokkos/kokkos/pull/4176) +- Add XLClang as compiler [\#4120](https://github.com/kokkos/kokkos/pull/4120) +- Keep quoted compiler flags when passing to Trilinos [\#3987](https://github.com/kokkos/kokkos/pull/3987) +- Add support for AMD Zen3 CPU architecture [\#3972](https://github.com/kokkos/kokkos/pull/3972) +- Rename IntelClang to IntelLLVM [\#3945](https://github.com/kokkos/kokkos/pull/3945) +- Add cppcoreguidelines-pro-type-cstyle-cast to clang-tidy [\#3522](https://github.com/kokkos/kokkos/pull/3522) +- Add sve bit size definition for A64FX [\#3947](https://github.com/kokkos/kokkos/pull/3947) [\#3946](https://github.com/kokkos/kokkos/pull/3946) +- Remove KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES [\#4150](https://github.com/kokkos/kokkos/pull/4150) + +### Other Changes: + +#### Tool Enhancements: + +- Retrieve original value from a point in a MultidimensionalSparseTuningProblem [\#3977](https://github.com/kokkos/kokkos/pull/3977) +- Allow extension of built-in tuners with additional tuning axes [\#3961](https://github.com/kokkos/kokkos/pull/3961) +- Added a categorical tuner [\#3955](https://github.com/kokkos/kokkos/pull/3955) + + +#### Miscellaneous: + +- hpcbind: Use double quotes around $@ when invoking user command [\#4284](https://github.com/kokkos/kokkos/pull/4284) +- Add file and line to error message [\#3985](https://github.com/kokkos/kokkos/pull/3985) +- Fix compiler warnings when compiling with nvc++ [\#4198](https://github.com/kokkos/kokkos/pull/4198) +- Add OpenMPTarget CI build on AMD GPUs [\#4055](https://github.com/kokkos/kokkos/pull/4055) +- CI: icpx is now part of intel container [\#4002](https://github.com/kokkos/kokkos/pull/4002) + +### Incompatibilities: + +- Remove pre CUDA 9 KOKKOS_IMPL_CUDA_* macros [\#4138](https://github.com/kokkos/kokkos/pull/4138) + +### Bug Fixes: +- UnorderedMap::clear() should zero the size() [\#4130](https://github.com/kokkos/kokkos/pull/4130) +- Add memory fence for HostSharedPtr::cleanup() [\#4144](https://github.com/kokkos/kokkos/pull/4144) +- SYCL: Fix race conditions in TeamPolicy::parallel_reduce [\#4418](https://github.com/kokkos/kokkos/pull/4418) +- Adding missing memory fence to serial exec space fence. [\#4292](https://github.com/kokkos/kokkos/pull/4292) +- Fix using external SYCL queues in tests [\#4291](https://github.com/kokkos/kokkos/pull/4291) +- Fix digits10 bug [\#4281](https://github.com/kokkos/kokkos/pull/4281) +- Fixes constexpr errors with frounding-math on gcc < 10. [\#4278](https://github.com/kokkos/kokkos/pull/4278) +- Fix compiler flags for PGI/NVHPC [\#4264](https://github.com/kokkos/kokkos/pull/4264) +- Fix Zen2/3 also implying Zen Arch with Makefiles [\#4260](https://github.com/kokkos/kokkos/pull/4260) +- Kokkos_Cuda.hpp: Fix shadow warning with cuda/11.0 [\#4252](https://github.com/kokkos/kokkos/pull/4252) +- Fix issue w/ static initialization of function attributes [\#4242](https://github.com/kokkos/kokkos/pull/4242) +- Disable long double hypot test on Power systems [\#4221](https://github.com/kokkos/kokkos/pull/4221) +- Fix false sharing in random pool [\#4218](https://github.com/kokkos/kokkos/pull/4218) +- Fix a missing memory_fence for debug shared alloc code [\#4216](https://github.com/kokkos/kokkos/pull/4216) +- Fix two xl issues [\#4179](https://github.com/kokkos/kokkos/pull/4179) +- Makefile.kokkos: fix (standard_in) 1: syntax error [\#4173](https://github.com/kokkos/kokkos/pull/4173) +- Fixes for query_device example [\#4172](https://github.com/kokkos/kokkos/pull/4172) +- Fix a bug when using HIP atomic with Kokkos::Complex [\#4159](https://github.com/kokkos/kokkos/pull/4159) +- Fix mistaken logic in pthread creation [\#4157](https://github.com/kokkos/kokkos/pull/4157) +- Define KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION when requesting Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION=ON [\#4107](https://github.com/kokkos/kokkos/pull/4107) +- Fix compilation with latest MSVC version [\#4102](https://github.com/kokkos/kokkos/pull/4102) +- Fix incorrect macro definitions when compiling with Intel compiler on Windows [\#4087](https://github.com/kokkos/kokkos/pull/4087) +- Fixup global buffer overflow in hand rolled string manipulation [\#4070](https://github.com/kokkos/kokkos/pull/4070) +- Fixup heap buffer overflow in cmd line args parsing unit tests [\#4069](https://github.com/kokkos/kokkos/pull/4069) +- Only add quotes in compiler flags for Trilinos if necessary [\#4067](https://github.com/kokkos/kokkos/pull/4067) +- Fixed invocation of tools init callbacks [\#4061](https://github.com/kokkos/kokkos/pull/4061) +- Work around SYCL JIT compiler issues with static variables [\#4013](https://github.com/kokkos/kokkos/pull/4013) +- Fix TestDetectionIdiom.cpp test inclusion for Trilinos/TriBITS [\#4010](https://github.com/kokkos/kokkos/pull/4010) +- Fixup allocation headers with OpenMPTarget backend [\#4003](https://github.com/kokkos/kokkos/pull/4003) +- Add missing specialization for OMPT to Kokkos Random [\#3967](https://github.com/kokkos/kokkos/pull/3967) +- Disable hypot long double test on power arches [\#3962](https://github.com/kokkos/kokkos/pull/3962) +- Use different EBO workaround for MSVC (rebased) [\#3924](https://github.com/kokkos/kokkos/pull/3924) +- Fix SYCL Kokkos::Profiling::(de)allocateData calls [\#3928](https://github.com/kokkos/kokkos/pull/3928) + +## [3.4.01](https://github.com/kokkos/kokkos/tree/3.4.01) (2021-05-19) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.4.00...3.4.01) + +**Bug Fixes:** +- Windows: Remove atomic_compare_exchange_strong overload conflicts with Windows [\#4024](https://github.com/kokkos/kokkos/pull/4024) +- OpenMPTarget: Fixup allocation headers with OpenMPTarget backend [\#4020](https://github.com/kokkos/kokkos/pull/4020) +- OpenMPTarget: Add missing specailization for OMPT to Kokkos Random [\#4022](https://github.com/kokkos/kokkos/pull/4022) +- AMD: Add support for AMD Zen3 CPU architecture [\#4021](https://github.com/kokkos/kokkos/pull/4021) +- SYCL: Implement SYCL::print_configuration [\#4012](https://github.com/kokkos/kokkos/pull/4012) +- Containers: staticcsrgraph: use device type instead of execution space to construct views [\#3998](https://github.com/kokkos/kokkos/pull/3998) +- nvcc_wrapper: fix errors in argument handling, suppress duplicates of GPU architecture and RDC flags [\#4006](https://github.com/kokkos/kokkos/pull/4006) +- CI: Add icpx testing to intel container [\#4004](https://github.com/kokkos/kokkos/pull/4004) +- CMake/TRIBITS: Keep quoted compiler flags when passing to Trilinos [\#4007](https://github.com/kokkos/kokkos/pull/4007) +- CMake: Rename IntelClang to IntelLLVM [\#3945](https://github.com/kokkos/kokkos/pull/3945) + +## [3.4.00](https://github.com/kokkos/kokkos/tree/3.4.00) (2021-04-25) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.3.01...3.4.00) + +**Highlights:** +- SYCL Backend Almost Feature Complete +- OpenMPTarget Backend Almost Feature Complete +- Performance Improvements for HIP backend +- Require CMake 3.16 or newer +- Tool Callback Interface Enhancements +- cmath wrapper functions available now in Kokkos::Experimental + +**Features:** +- Implement parallel_scan with ThreadVectorRange and Reducer [\#3861](https://github.com/kokkos/kokkos/pull/3861) +- Implement SYCL Random [\#3849](https://github.com/kokkos/kokkos/pull/3849) +- OpenMPTarget: Adding Implementation for nested reducers [\#3845](https://github.com/kokkos/kokkos/pull/3845) +- Implement UniqueToken for SYCL [\#3833](https://github.com/kokkos/kokkos/pull/3833) +- OpenMPTarget: UniqueToken::Global implementation [\#3823](https://github.com/kokkos/kokkos/pull/3823) +- DualView sync's on ExecutionSpaces [\#3822](https://github.com/kokkos/kokkos/pull/3822) +- SYCL outer TeamPolicy parallel_reduce [\#3818](https://github.com/kokkos/kokkos/pull/3818) +- SYCL TeamPolicy::team_scan [\#3815](https://github.com/kokkos/kokkos/pull/3815) +- SYCL MDRangePolicy parallel_reduce [\#3801](https://github.com/kokkos/kokkos/pull/3801) +- Enable use of execution space instances in ScatterView [\#3786](https://github.com/kokkos/kokkos/pull/3786) +- SYCL TeamPolicy nested parallel_reduce [\#3783](https://github.com/kokkos/kokkos/pull/3783) +- OpenMPTarget: MDRange with TagType for parallel_for [\#3781](https://github.com/kokkos/kokkos/pull/3781) +- Adding OpenMPTarget parallel_scan [\#3655](https://github.com/kokkos/kokkos/pull/3655) +- SYCL basic TeamPolicy [\#3654](https://github.com/kokkos/kokkos/pull/3654) +- OpenMPTarget: scratch memory implementation [\#3611](https://github.com/kokkos/kokkos/pull/3611) + +**Implemented enhancements Backends and Archs:** +- SYCL choose a specific GPU [\#3918](https://github.com/kokkos/kokkos/pull/3918) +- [HIP] Lock access to scratch memory when using Teams [\#3916](https://github.com/kokkos/kokkos/pull/3916) +- [HIP] fix multithreaded access to get_next_driver [\#3908](https://github.com/kokkos/kokkos/pull/3908) +- Forward declare HIPHostPinnedSpace and SYCLSharedUSMSpace [\#3902](https://github.com/kokkos/kokkos/pull/3902) +- Let SYCL USMObjectMem use SharedAllocationRecord [\#3898](https://github.com/kokkos/kokkos/pull/3898) +- Implement clock_tic for SYCL [\#3893](https://github.com/kokkos/kokkos/pull/3893) +- Don't use a static variable in HIPInternal::scratch_space [\#3866](https://github.com/kokkos/kokkos/pull/3866)(https://github.com/kokkos/kokkos/pull/3866) +- Reuse memory for SYCL parallel_reduce [\#3873](https://github.com/kokkos/kokkos/pull/3873) +- Update SYCL compiler in CI [\#3826](https://github.com/kokkos/kokkos/pull/3826) +- Introduce HostSharedPtr to manage m_space_instance for Cuda/HIP/SYCL [\#3824](https://github.com/kokkos/kokkos/pull/3824) +- [HIP] Use shuffle for range reduction [\#3811](https://github.com/kokkos/kokkos/pull/3811) +- OpenMPTarget: Changes to the hierarchical parallelism [\#3808](https://github.com/kokkos/kokkos/pull/3808) +- Remove ExtendedReferenceWrapper for SYCL parallel_reduce [\#3802](https://github.com/kokkos/kokkos/pull/3802) +- Eliminate sycl_indirect_launch [\#3777](https://github.com/kokkos/kokkos/pull/3777) +- OpenMPTarget: scratch implementation for parallel_reduce [\#3776](https://github.com/kokkos/kokkos/pull/3776) +- Allow initializing SYCL execution space from sycl::queue and SYCL::impl_static_fence [\#3767](https://github.com/kokkos/kokkos/pull/3767) +- SYCL TeamPolicy scratch memory alternative [\#3763](https://github.com/kokkos/kokkos/pull/3763) +- Alternative implementation for SYCL TeamPolicy [\#3759](https://github.com/kokkos/kokkos/pull/3759) +- Unify handling of synchronous errors in SYCL [\#3754](https://github.com/kokkos/kokkos/pull/3754) +- core/Cuda: Half_t updates for cgsolve [\#3746](https://github.com/kokkos/kokkos/pull/3746) +- Unify HIPParallelLaunch structures [\#3733](https://github.com/kokkos/kokkos/pull/3733) +- Improve performance for SYCL parallel_reduce [\#3732](https://github.com/kokkos/kokkos/pull/3732) +- Use consistent types in Kokkos_OpenMPTarget_Parallel.hpp [\#3703](https://github.com/kokkos/kokkos/pull/3703) +- Implement non-blocking kernel launches for HIP backend [\#3697](https://github.com/kokkos/kokkos/pull/3697) +- Change SYCLInternal::m_queue std::unique_ptr -> std::optional [\#3677](https://github.com/kokkos/kokkos/pull/3677) +- Use alternative SYCL parallel_reduce implementation [\#3671](https://github.com/kokkos/kokkos/pull/3671) +- Use runtime values in KokkosExp_MDRangePolicy.hpp [\#3626](https://github.com/kokkos/kokkos/pull/3626) +- Clean up AnalyzePolicy [\#3564](https://github.com/kokkos/kokkos/pull/3564) +- Changes for indirect launch of SYCL parallel reduce [\#3511](https://github.com/kokkos/kokkos/pull/3511) + +**Implemented enhancements BuildSystem:** +- Also require C++14 when building gtest [\#3912](https://github.com/kokkos/kokkos/pull/3912) +- Fix compiling SYCL with OpenMP [\#3874](https://github.com/kokkos/kokkos/pull/3874) +- Require C++17 for SYCL (at configuration time) [\#3869](https://github.com/kokkos/kokkos/pull/3869) +- Add COMPILE_DEFINITIONS argument to kokkos_create_imported_tpl [\#3862](https://github.com/kokkos/kokkos/pull/3862) +- Do not pass arch flags to the linker with no rdc [\#3846](https://github.com/kokkos/kokkos/pull/3846) +- Try compiling C++14 check with C++14 support and print error message [\#3843](https://github.com/kokkos/kokkos/pull/3843) +- Enable HIP with Cray Clang [\#3842](https://github.com/kokkos/kokkos/pull/3842) +- Add an option to disable header self containment tests [\#3834](https://github.com/kokkos/kokkos/pull/3834) +- CMake check for C++14 [\#3809](https://github.com/kokkos/kokkos/pull/3809) +- Prefer -std=* over --std=* [\#3779](https://github.com/kokkos/kokkos/pull/3779) +- Kokkos launch compiler updates [\#3778](https://github.com/kokkos/kokkos/pull/3778) +- Updated comments and enabled no-op for kokkos_launch_compiler [\#3774](https://github.com/kokkos/kokkos/pull/3774) +- Apple's Clang not correctly recognised [\#3772](https://github.com/kokkos/kokkos/pull/3772) +- kokkos_launch_compiler + CUDA auto-detect arch [\#3770](https://github.com/kokkos/kokkos/pull/3770) +- Add Spack test support for Kokkos [\#3753](https://github.com/kokkos/kokkos/pull/3753) +- Split SYCL tests for aot compilation [\#3741](https://github.com/kokkos/kokkos/pull/3741) +- Use consistent OpenMP flag for IntelClang [\#3735](https://github.com/kokkos/kokkos/pull/3735) +- Add support for -Wno-deprecated-gpu-targets [\#3722](https://github.com/kokkos/kokkos/pull/3722) +- Add configuration to target CUDA compute capability 8.6 [\#3713](https://github.com/kokkos/kokkos/pull/3713) +- Added VERSION and SOVERSION to KOKKOS_INTERNAL_ADD_LIBRARY [\#3706](https://github.com/kokkos/kokkos/pull/3706) +- Add fast-math to known NVCC flags [\#3699](https://github.com/kokkos/kokkos/pull/3699) +- Add MI-100 arch string [\#3698](https://github.com/kokkos/kokkos/pull/3698) +- Require CMake >=3.16 [\#3679](https://github.com/kokkos/kokkos/pull/3679) +- KokkosCI.cmake, KokkosCTest.cmake.in, CTestConfig.cmake.in + CI updates [\#2844](https://github.com/kokkos/kokkos/pull/2844) + +**Implemented enhancements Tools:** +- Improve readability of the callback invocation in profiling [\#3860](https://github.com/kokkos/kokkos/pull/3860) +- V1.1 Tools Interface: incremental, action-based [\#3812](https://github.com/kokkos/kokkos/pull/3812) +- Enable launch latency simulations [\#3721](https://github.com/kokkos/kokkos/pull/3721) +- Added metadata callback to tools interface [\#3711](https://github.com/kokkos/kokkos/pull/3711) +- MDRange Tile Size Tuning [\#3688](https://github.com/kokkos/kokkos/pull/3688) +- Added support for command-line args for kokkos-tools [\#3627](https://github.com/kokkos/kokkos/pull/3627) +- Query max tile sizes for an MDRangePolicy, and set tile sizes on an existing policy [\#3481](https://github.com/kokkos/kokkos/pull/3481) + +**Implemented enhancements Other:** +- Try detecting ndevices in get_gpu [\#3921](https://github.com/kokkos/kokkos/pull/3921) +- Use strcmp to compare names() [\#3909](https://github.com/kokkos/kokkos/pull/3909) +- Add execution space arguments for constructor overloads that might allocate a new underlying View [\#3904](https://github.com/kokkos/kokkos/pull/3904) +- Prefix labels in internal use of kokkos_malloc [\#3891](https://github.com/kokkos/kokkos/pull/3891) +- Prefix labels for internal uses of SharedAllocationRecord [\#3890](https://github.com/kokkos/kokkos/pull/3890) +- Add missing hypot math function [\#3880](https://github.com/kokkos/kokkos/pull/3880) +- Unify algorithm unit tests to avoid code duplication [\#3851](https://github.com/kokkos/kokkos/pull/3851) +- DualView.template view() better matches for Devices in UVMSpace cases [\#3857](https://github.com/kokkos/kokkos/pull/3857) +- More extensive disentangling of Policy Traits [\#3829](https://github.com/kokkos/kokkos/pull/3829) +- Replaced nanosleep and sched_yield with STL routines [\#3825](https://github.com/kokkos/kokkos/pull/3825) +- Constructing Atomic Subviews [\#3810](https://github.com/kokkos/kokkos/pull/3810) +- Metadata Declaration in Core [\#3729](https://github.com/kokkos/kokkos/pull/3729) +- Allow using tagged final functor in parallel_reduce [\#3714](https://github.com/kokkos/kokkos/pull/3714) +- Major duplicate code removal in SharedAllocationRecord specializations [\#3658](https://github.com/kokkos/kokkos/pull/3658) + +**Fixed bugs:** +- Provide forward declarations in Kokkos_ViewLayoutTiled.hpp for XL [\#3911](https://github.com/kokkos/kokkos/pull/3911) +- Fixup absolute value of floating points in Kokkos complex [\#3882](https://github.com/kokkos/kokkos/pull/3882) +- Address intel 17 ICE [\#3881](https://github.com/kokkos/kokkos/pull/3881) +- Add missing pow(Kokkos::complex) overloads [\#3868](https://github.com/kokkos/kokkos/pull/3868) +- Fix bug {pow, log}(Kokkos::complex) [\#3866](https://github.com/kokkos/kokkos/pull/3866)(https://github.com/kokkos/kokkos/pull/3866) +- Cleanup writing to output streams in Cuda [\#3859](https://github.com/kokkos/kokkos/pull/3859) +- Fixup cache CUDA fallback execution space instance used by DualView::sync [\#3856](https://github.com/kokkos/kokkos/pull/3856) +- Fix cmake warning with pthread [\#3854](https://github.com/kokkos/kokkos/pull/3854) +- Fix typo FOUND_CUDA_{DRIVVER -> DRIVER} [\#3852](https://github.com/kokkos/kokkos/pull/3852) +- Fix bug in SYCL team_reduce [\#3848](https://github.com/kokkos/kokkos/pull/3848) +- Atrocious bug in MDRange tuning [\#3803](https://github.com/kokkos/kokkos/pull/3803) +- Fix compiling SYCL with Kokkos_ENABLE_TUNING=ON [\#3800](https://github.com/kokkos/kokkos/pull/3800) +- Fixed command line parsing bug [\#3797](https://github.com/kokkos/kokkos/pull/3797) +- Workaround race condition in SYCL parallel_reduce [\#3782](https://github.com/kokkos/kokkos/pull/3782) +- Fix Atomic{Min,Max} for Kepler30 [\#3780](https://github.com/kokkos/kokkos/pull/3780) +- Fix SYCL typo [\#3755](https://github.com/kokkos/kokkos/pull/3755) +- Fixed Kokkos_install_additional_files macro [\#3752](https://github.com/kokkos/kokkos/pull/3752) +- Fix a typo for Kokkos_ARCH_A64FX [\#3751](https://github.com/kokkos/kokkos/pull/3751) +- OpenMPTarget: fixes and workarounds to work with "Release" build type [\#3748](https://github.com/kokkos/kokkos/pull/3748) +- Fix parsing bug for number of devices command line argument [\#3724](https://github.com/kokkos/kokkos/pull/3724) +- Avoid more warnings with clang and C++20 [\#3719](https://github.com/kokkos/kokkos/pull/3719) +- Fix gcc-10.1 C++20 warnings [\#3718](https://github.com/kokkos/kokkos/pull/3718) +- Fix cuda cache config not being set correct [\#3712](https://github.com/kokkos/kokkos/pull/3712) +- Fix dualview deepcopy perftools [\#3701](https://github.com/kokkos/kokkos/pull/3701) +- use drand instead of frand in drand [\#3696](https://github.com/kokkos/kokkos/pull/3696) + +**Incompatibilities:** +- Remove unimplemented member functions of SYCLDevice [\#3919](https://github.com/kokkos/kokkos/pull/3919) +- Replace cl::sycl [\#3896](https://github.com/kokkos/kokkos/pull/3896) +- Get rid of SYCL workaround in Kokkos_Complex.hpp [\#3884](https://github.com/kokkos/kokkos/pull/3884) +- Replace most uses of if_c [\#3883](https://github.com/kokkos/kokkos/pull/3883) +- Remove Impl::enable_if_type [\#3863](https://github.com/kokkos/kokkos/pull/3863) +- Remove HostBarrier test [\#3847](https://github.com/kokkos/kokkos/pull/3847) +- Avoid (void) interface [\#3836](https://github.com/kokkos/kokkos/pull/3836) +- Remove VerifyExecutionCanAccessMemorySpace [\#3813](https://github.com/kokkos/kokkos/pull/3813) +- Avoid duplicated code in ScratchMemorySpace [\#3793](https://github.com/kokkos/kokkos/pull/3793) +- Remove superfluous FunctorFinal specialization [\#3788](https://github.com/kokkos/kokkos/pull/3788) +- Rename cl::sycl -> sycl in Kokkos_MathematicalFunctions.hpp [\#3678](https://github.com/kokkos/kokkos/pull/3678) +- Remove integer_sequence backward compatibility implementation [\#3533](https://github.com/kokkos/kokkos/pull/3533) + +**Enabled tests:** +- Fixup re-enable core performance tests [\#3903](https://github.com/kokkos/kokkos/pull/3903) +- Enable more SYCL tests [\#3900](https://github.com/kokkos/kokkos/pull/3900) +- Restrict MDRange Policy tests for Intel GPUs [\#3853](https://github.com/kokkos/kokkos/pull/3853) +- Disable death tests for rawhide [\#3844](https://github.com/kokkos/kokkos/pull/3844) +- OpenMPTarget: Block unit tests that do not pass with the nvidia compiler [\#3839](https://github.com/kokkos/kokkos/pull/3839) +- Enable Bitset container test for SYCL [\#3830](https://github.com/kokkos/kokkos/pull/3830) +- Enable some more SYCL tests [\#3744](https://github.com/kokkos/kokkos/pull/3744) +- Enable SYCL atomic tests [\#3742](https://github.com/kokkos/kokkos/pull/3742) +- Enable more SYCL perf_tests [\#3692](https://github.com/kokkos/kokkos/pull/3692) +- Enable examples for SYCL [\#3691](https://github.com/kokkos/kokkos/pull/3691) + +## [3.3.01](https://github.com/kokkos/kokkos/tree/3.3.01) (2021-01-06) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.3.00...3.3.01) + +**Bug Fixes:** +- Fix severe performance bug in DualView which added memcpys for sync and modify [\#3693](https://github.com/kokkos/kokkos/issues/#3693) +- Fix performance bug in CUDA backend, where the cuda Cache config was not set correct. + +## [3.3.00](https://github.com/kokkos/kokkos/tree/3.3.00) (2020-12-16) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.2.01...3.3.00) + +**Features:** +- Require C++14 as minimum C++ standard. C++17 and C++20 are supported too. +- HIP backend is nearly feature complete. Kokkos Dynamic Task Graphs are missing. +- Major update for OpenMPTarget: many capabilities now work. For details contact us. +- Added DPC++/SYCL backend: primary capabilites are working. +- Added Kokkos Graph API analogous to CUDA Graphs. +- Added parallel_scan support with TeamThreadRange [\#3536](https://github.com/kokkos/kokkos/pull/3536) +- Added Logical Memory Spaces [\#3546](https://github.com/kokkos/kokkos/pull/3546) +- Added initial half precision support [\#3439](https://github.com/kokkos/kokkos/pull/3439) +- Experimental feature: control cuda occupancy [\#3379](https://github.com/kokkos/kokkos/pull/3379) + +**Implemented enhancements Backends and Archs:** +- Add a64fx and fujitsu Compiler support [\#3614](https://github.com/kokkos/kokkos/pull/3614) +- Adding support for AMD gfx908 archictecture [\#3375](https://github.com/kokkos/kokkos/pull/3375) +- SYCL parallel\_for MDRangePolicy [\#3583](https://github.com/kokkos/kokkos/pull/3583) +- SYCL add parallel\_scan [\#3577](https://github.com/kokkos/kokkos/pull/3577) +- SYCL custom reductions [\#3544](https://github.com/kokkos/kokkos/pull/3544) +- SYCL Enable container unit tests [\#3550](https://github.com/kokkos/kokkos/pull/3550) +- SYCL feature level 5 [\#3480](https://github.com/kokkos/kokkos/pull/3480) +- SYCL Feature level 4 (parallel\_for) [\#3474](https://github.com/kokkos/kokkos/pull/3474) +- SYCL feature level 3 [\#3451](https://github.com/kokkos/kokkos/pull/3451) +- SYCL feature level 2 [\#3447](https://github.com/kokkos/kokkos/pull/3447) +- OpenMPTarget: Hierarchial reduction for + operator on scalars [\#3504](https://github.com/kokkos/kokkos/pull/3504) +- OpenMPTarget hierarchical [\#3411](https://github.com/kokkos/kokkos/pull/3411) +- HIP Add Impl::atomic\_[store,load] [\#3440](https://github.com/kokkos/kokkos/pull/3440) +- HIP enable global lock arrays [\#3418](https://github.com/kokkos/kokkos/pull/3418) +- HIP Implement multiple occupancy paths for various HIP kernel launchers [\#3366](https://github.com/kokkos/kokkos/pull/3366) + +**Implemented enhancements Policies:** +- MDRangePolicy: Let it be semiregular [\#3494](https://github.com/kokkos/kokkos/pull/3494) +- MDRangePolicy: Check narrowing conversion in construction [\#3527](https://github.com/kokkos/kokkos/pull/3527) +- MDRangePolicy: CombinedReducers support [\#3395](https://github.com/kokkos/kokkos/pull/3395) +- Kokkos Graph: Interface and Default Implementation [\#3362](https://github.com/kokkos/kokkos/pull/3362) +- Kokkos Graph: add Cuda Graph implementation [\#3369](https://github.com/kokkos/kokkos/pull/3369) +- TeamPolicy: implemented autotuning of team sizes and vector lengths [\#3206](https://github.com/kokkos/kokkos/pull/3206) +- RangePolicy: Initialize all data members in default constructor [\#3509](https://github.com/kokkos/kokkos/pull/3509) + +**Implemented enhancements BuildSystem:** +- Auto-generate core test files for all backends [\#3488](https://github.com/kokkos/kokkos/pull/3488) +- Avoid rewriting test files when calling cmake [\#3548](https://github.com/kokkos/kokkos/pull/3548) +- RULE\_LAUNCH\_COMPILE and RULE\_LAUNCH\_LINK system for nvcc\_wrapper [\#3136](https://github.com/kokkos/kokkos/pull/3136) +- Adding -include as a known argument to nvcc\_wrapper [\#3434](https://github.com/kokkos/kokkos/pull/3434) +- Install hpcbind script [\#3402](https://github.com/kokkos/kokkos/pull/3402) +- cmake/kokkos\_tribits.cmake: add parsing for args [\#3457](https://github.com/kokkos/kokkos/pull/3457) + +**Implemented enhancements Tools:** +- Changed namespacing of Kokkos::Tools::Impl::Impl::tune\_policy [\#3455](https://github.com/kokkos/kokkos/pull/3455) +- Delegate to an impl allocate/deallocate method to allow specifying a SpaceHandle for MemorySpaces [\#3530](https://github.com/kokkos/kokkos/pull/3530) +- Use the Kokkos Profiling interface rather than the Impl interface [\#3518](https://github.com/kokkos/kokkos/pull/3518) +- Runtime option for tuning [\#3459](https://github.com/kokkos/kokkos/pull/3459) +- Dual View Tool Events [\#3326](https://github.com/kokkos/kokkos/pull/3326) + +**Implemented enhancements Other:** +- Abort on errors instead of just printing [\#3528](https://github.com/kokkos/kokkos/pull/3528) +- Enable C++14 macros unconditionally [\#3449](https://github.com/kokkos/kokkos/pull/3449) +- Make ViewMapping trivially copyable [\#3436](https://github.com/kokkos/kokkos/pull/3436) +- Rename struct ViewMapping to class [\#3435](https://github.com/kokkos/kokkos/pull/3435) +- Replace enums in Kokkos\_ViewMapping.hpp (removes -Wextra) [\#3422](https://github.com/kokkos/kokkos/pull/3422) +- Use bool for enums representing bools [\#3416](https://github.com/kokkos/kokkos/pull/3416) +- Fence active instead of default execution space instances [\#3388](https://github.com/kokkos/kokkos/pull/3388) +- Refactor parallel\_reduce fence usage [\#3359](https://github.com/kokkos/kokkos/pull/3359) +- Moved Space EBO helpers to Kokkos\_EBO [\#3357](https://github.com/kokkos/kokkos/pull/3357) +- Add remove\_cvref type trait [\#3340](https://github.com/kokkos/kokkos/pull/3340) +- Adding identity type traits and update definition of identity\_t alias [\#3339](https://github.com/kokkos/kokkos/pull/3339) +- Add is\_specialization\_of type trait [\#3338](https://github.com/kokkos/kokkos/pull/3338) +- Make ScratchMemorySpace semi-regular [\#3309](https://github.com/kokkos/kokkos/pull/3309) +- Optimize min/max atomics with early exit on no-op case [\#3265](https://github.com/kokkos/kokkos/pull/3265) +- Refactor Backend Development [\#2941](https://github.com/kokkos/kokkos/pull/2941) + +**Fixed bugs:** +- Fixup MDRangePolicy construction from Kokkos arrays [\#3591](https://github.com/kokkos/kokkos/pull/3591) +- Add atomic functions for unsigned long long using gcc built-in [\#3588](https://github.com/kokkos/kokkos/pull/3588) +- Fixup silent pointless comparison with zero in checked\_narrow\_cast (compiler workaround) [\#3566](https://github.com/kokkos/kokkos/pull/3566) +- Fixes for ROCm 3.9 [\#3565](https://github.com/kokkos/kokkos/pull/3565) +- Fix windows build issues which crept in for the CUDA build [\#3532](https://github.com/kokkos/kokkos/pull/3532) +- HIP Fix atomics of large data types and clean up lock arrays [\#3529](https://github.com/kokkos/kokkos/pull/3529) +- Pthreads fix exception resulting from 0 grain size [\#3510](https://github.com/kokkos/kokkos/pull/3510) +- Fixup do not require atomic operation to be default constructible [\#3503](https://github.com/kokkos/kokkos/pull/3503) +- Fix race condition in HIP backend [\#3467](https://github.com/kokkos/kokkos/pull/3467) +- Replace KOKKOS\_DEBUG with KOKKOS\_ENABLE\_DEBUG [\#3458](https://github.com/kokkos/kokkos/pull/3458) +- Fix multi-stream team scratch space definition for HIP [\#3398](https://github.com/kokkos/kokkos/pull/3398) +- HIP fix template deduction [\#3393](https://github.com/kokkos/kokkos/pull/3393) +- Fix compiling with HIP and C++17 [\#3390](https://github.com/kokkos/kokkos/pull/3390) +- Fix sigFPE in HIP blocksize deduction [\#3378](https://github.com/kokkos/kokkos/pull/3378) +- Type alias change: replace CS with CTS to avoid conflicts with NVSHMEM [\#3348](https://github.com/kokkos/kokkos/pull/3348) +- Clang compilation of CUDA backend on Windows [\#3345](https://github.com/kokkos/kokkos/pull/3345) +- Fix HBW support [\#3343](https://github.com/kokkos/kokkos/pull/3343) +- Added missing fences to unique token [\#3260](https://github.com/kokkos/kokkos/pull/3260) + +**Incompatibilities:** +- Remove unused utilities (forward, move, and expand\_variadic) from Kokkos::Impl [\#3535](https://github.com/kokkos/kokkos/pull/3535) +- Remove unused traits [\#3534](https://github.com/kokkos/kokkos/pull/3534) +- HIP: Remove old HCC code [\#3301](https://github.com/kokkos/kokkos/pull/3301) +- Prepare for deprecation of ViewAllocateWithoutInitializing [\#3264](https://github.com/kokkos/kokkos/pull/3264) +- Remove ROCm backend [\#3148](https://github.com/kokkos/kokkos/pull/3148) + +## [3.2.01](https://github.com/kokkos/kokkos/tree/3.2.01) (2020-11-17) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.2.00...3.2.01) + +**Fixed bugs:** +- Disallow KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE in shared library builds [\#3332](https://github.com/kokkos/kokkos/pull/3332) +- Do not install libprinter-tool when testing is enabled [\#3313](https://github.com/kokkos/kokkos/pull/3313) +- Fix restrict/alignment following refactor [\#3373](https://github.com/kokkos/kokkos/pull/3373) + - Intel fix: workaround compiler issue with using statement [\#3383](https://github.com/kokkos/kokkos/pull/3383) +- Fix zero-length reductions [#\3364](https://github.com/kokkos/kokkos/pull/3364) + - Pthread zero-length reduction fix [\#3452](https://github.com/kokkos/kokkos/pull/3452) + - HPX zero-length reduction fix [\#3470](https://github.com/kokkos/kokkos/pull/3470) + - cuda/9.2 zero-length reduction fix [\#3580](https://github.com/kokkos/kokkos/pull/3580) +- Fix multi-stream scratch [#\3269](https://github.com/kokkos/kokkos/pull/3269) +- Guard KOKKOS_ALL_COMPILE_OPTIONS if Cuda is not enabled [\#3387](https://github.com/kokkos/kokkos/pull/3387) +- Do not include link flags for Fortran linkage [\#3384](https://github.com/kokkos/kokkos/pull/3384) +- Fix NVIDIA GPU arch macro with autodetection [\#3473](https://github.com/kokkos/kokkos/pull/3473) +- Fix libdl/test issues with Trilinos [\#3543](https://github.com/kokkos/kokkos/pull/3543) + - Register Pthread as Tribits option to be enabled with Trilinos [\#3558](https://github.com/kokkos/kokkos/pull/3558) + +**Implemented enhancements:** +- Separate Cuda timing-based tests into their own executable [\#3407](https://github.com/kokkos/kokkos/pull/3407) + +## [3.2.00](https://github.com/kokkos/kokkos/tree/3.2.00) (2020-08-19) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.1.01...3.2.00) + +**Implemented enhancements:** + +- HIP:Enable stream in HIP [\#3163](https://github.com/kokkos/kokkos/issues/3163) +- HIP:Add support for shuffle reduction for the HIP backend [\#3154](https://github.com/kokkos/kokkos/issues/3154) +- HIP:Add implementations of missing HIPHostPinnedSpace methods for LAMMPS [\#3137](https://github.com/kokkos/kokkos/issues/3137) +- HIP:Require HIP 3.5.0 or higher [\#3099](https://github.com/kokkos/kokkos/issues/3099) +- HIP:WorkGraphPolicy for HIP [\#3096](https://github.com/kokkos/kokkos/issues/3096) +- OpenMPTarget: Significant update to the new experimental backend. Requires C++17, works on Intel GPUs, reference counting fixes. [\#3169](https://github.com/kokkos/kokkos/issues/3169) +- Windows Cuda support [\#3018](https://github.com/kokkos/kokkos/issues/3018) +- Pass `-Wext-lambda-captures-this` to NVCC when support for `__host__ __device__` lambda is enabled from CUDA 11 [\#3241](https://github.com/kokkos/kokkos/issues/3241) +- Use explicit staging buffer for constant memory kernel launches and cleanup host/device synchronization [\#3234](https://github.com/kokkos/kokkos/issues/3234) +- Various fixup to policies including making TeamPolicy default constructible and making RangePolicy and TeamPolicy assignable: [\#3202](https://github.com/kokkos/kokkos/issues/3202) , [\#3203](https://github.com/kokkos/kokkos/issues/3203) , [\#3196](https://github.com/kokkos/kokkos/issues/3196) +- Annotations for `DefaultExectutionSpace` and `DefaultHostExectutionSpace` to use in static analysis [\#3189](https://github.com/kokkos/kokkos/issues/3189) +- Add documentation on using Spack to install Kokkos and developing packages that depend on Kokkos [\#3187](https://github.com/kokkos/kokkos/issues/3187) +- Add OpenMPTarget backend flags for NVC++ compiler [\#3185](https://github.com/kokkos/kokkos/issues/3185) +- Move deep\_copy/create\_mirror\_view on Experimental::OffsetView into Kokkos:: namespace [\#3166](https://github.com/kokkos/kokkos/issues/3166) +- Allow for larger block size in HIP [\#3165](https://github.com/kokkos/kokkos/issues/3165) +- View: Added names of Views to the different View initialize/free kernels [\#3159](https://github.com/kokkos/kokkos/issues/3159) +- Cuda: Caching cudaFunctorAttributes and whether L1/Shmem prefer was set [\#3151](https://github.com/kokkos/kokkos/issues/3151) +- BuildSystem: Improved performance in default configuration by defaulting to Release build [\#3131](https://github.com/kokkos/kokkos/issues/3131) +- Cuda: Update CUDA occupancy calculation [\#3124](https://github.com/kokkos/kokkos/issues/3124) +- Vector: Adding data() to Vector [\#3123](https://github.com/kokkos/kokkos/issues/3123) +- BuildSystem: Add CUDA Ampere configuration support [\#3122](https://github.com/kokkos/kokkos/issues/3122) +- General: Apply [[noreturn]] to Kokkos::abort when applicable [\#3106](https://github.com/kokkos/kokkos/issues/3106) +- TeamPolicy: Validate storage level argument passed to TeamPolicy::set\_scratch\_size() [\#3098](https://github.com/kokkos/kokkos/issues/3098) +- BuildSystem: Make kokkos\_has\_string() function in Makefile.kokkos case insensitive [\#3091](https://github.com/kokkos/kokkos/issues/3091) +- Modify KOKKOS\_FUNCTION macro for clang-tidy analysis [\#3087](https://github.com/kokkos/kokkos/issues/3087) +- Move allocation profiling to allocate/deallocate calls [\#3084](https://github.com/kokkos/kokkos/issues/3084) +- BuildSystem: FATAL\_ERROR when attempting in-source build [\#3082](https://github.com/kokkos/kokkos/issues/3082) +- Change enums in ScatterView to types [\#3076](https://github.com/kokkos/kokkos/issues/3076) +- HIP: Changes for new compiler/runtime [\#3067](https://github.com/kokkos/kokkos/issues/3067) +- Extract and use get\_gpu [\#3061](https://github.com/kokkos/kokkos/issues/3061) , [\#3048](https://github.com/kokkos/kokkos/issues/3048) +- Add is\_allocated to View-like containers [\#3059](https://github.com/kokkos/kokkos/issues/3059) +- Combined reducers for scalar references [\#3052](https://github.com/kokkos/kokkos/issues/3052) +- Add configurable capacity for UniqueToken [\#3051](https://github.com/kokkos/kokkos/issues/3051) +- Add installation testing [\#3034](https://github.com/kokkos/kokkos/issues/3034) +- HIP: Add UniqueToken [\#3020](https://github.com/kokkos/kokkos/issues/3020) +- Autodetect number of devices [\#3013](https://github.com/kokkos/kokkos/issues/3013) + + +**Fixed bugs:** + +- Check error code from `cudaStreamSynchronize` in CUDA fences [\#3255](https://github.com/kokkos/kokkos/issues/3255) +- Fix issue with C++ standard flags when using `nvcc\_wrapper` with PGI [\#3254](https://github.com/kokkos/kokkos/issues/3254) +- Add missing threadfence in lock-based atomics [\#3208](https://github.com/kokkos/kokkos/issues/3208) +- Fix dedup of linker flags for shared lib on CMake <=3.12 [\#3176](https://github.com/kokkos/kokkos/issues/3176) +- Fix memory leak with CUDA streams [\#3170](https://github.com/kokkos/kokkos/issues/3170) +- BuildSystem: Fix OpenMP Target flags for Cray [\#3161](https://github.com/kokkos/kokkos/issues/3161) +- ScatterView: fix for OpenmpTarget remove inheritance from reducers [\#3162](https://github.com/kokkos/kokkos/issues/3162) +- BuildSystem: Set OpenMP flags according to host compiler [\#3127](https://github.com/kokkos/kokkos/issues/3127) +- OpenMP: Fix logic for nested omp in partition\_master bug [\#3101](https://github.com/kokkos/kokkos/issues/3101) +- nvcc\_wrapper: send --cudart to nvcc instead of host compiler [\#3092](https://github.com/kokkos/kokkos/issues/3092) +- BuildSystem: Fixes for Cuda/11 and c++17 [\#3085](https://github.com/kokkos/kokkos/issues/3085) +- HIP: Fix print\_configuration [\#3080](https://github.com/kokkos/kokkos/issues/3080) +- Conditionally define get\_gpu [\#3072](https://github.com/kokkos/kokkos/issues/3072) +- Fix bounds for ranges in random number generator [\#3069](https://github.com/kokkos/kokkos/issues/3069) +- Fix Cuda minor arch check [\#3035](https://github.com/kokkos/kokkos/issues/3035) +- BuildSystem: Add -expt-relaxed-constexpr flag to nvcc\_wrapper [\#3021](https://github.com/kokkos/kokkos/issues/3021) + +**Incompatibilities:** + +- Remove ETI support [\#3157](https://github.com/kokkos/kokkos/issues/3157) +- Remove KOKKOS\_INTERNAL\_ENABLE\_NON\_CUDA\_BACKEND [\#3147](https://github.com/kokkos/kokkos/issues/3147) +- Remove core/unit\_test/config [\#3146](https://github.com/kokkos/kokkos/issues/3146) +- Removed the preprocessor branch for KOKKOS\_ENABLE\_PROFILING [\#3115](https://github.com/kokkos/kokkos/issues/3115) +- Disable profiling with MSVC [\#3066](https://github.com/kokkos/kokkos/issues/3066) + +**Closed issues:** + +- Silent error (Validate storage level arg to set_scratch_size) [\#3097](https://github.com/kokkos/kokkos/issues/3097) +- Remove KOKKOS\_ENABLE\_PROFILING Option [\#3095](https://github.com/kokkos/kokkos/issues/3095) +- Cuda 11 -\> allow C++17 [\#3083](https://github.com/kokkos/kokkos/issues/3083) +- In source build failure not explained [\#3081](https://github.com/kokkos/kokkos/issues/3081) +- Allow naming of Views for initialization kernel [\#3070](https://github.com/kokkos/kokkos/issues/3070) +- DefaultInit tests failing when using CTest resource allocation feature [\#3040](https://github.com/kokkos/kokkos/issues/3040) +- Add installation testing. [\#3037](https://github.com/kokkos/kokkos/issues/3037) +- nvcc\_wrapper needs to handle `-expt-relaxed-constexpr` flag [\#3017](https://github.com/kokkos/kokkos/issues/3017) +- CPU core oversubscription warning on macOS with OpenMP backend [\#2996](https://github.com/kokkos/kokkos/issues/2996) +- Default behavior of KOKKOS\_NUM\_DEVICES to use all devices available [\#2975](https://github.com/kokkos/kokkos/issues/2975) +- Assert blocksize \> 0 [\#2974](https://github.com/kokkos/kokkos/issues/2974) +- Add ability to assign kokkos profile function from executable [\#2973](https://github.com/kokkos/kokkos/issues/2973) +- ScatterView Support for the pre/post increment operator [\#2967](https://github.com/kokkos/kokkos/issues/2967) + +- Compiler issue: Cuda build with clang 10 has errors with the atomic unit tests [\#3237](https://github.com/kokkos/kokkos/issues/3237) +- Incompatibility of flags for C++ standard with PGI v20.4 on Power9/NVIDIA V100 system [\#3252](https://github.com/kokkos/kokkos/issues/3252) +- Error configuring as subproject [\#3140](https://github.com/kokkos/kokkos/issues/3140) +- CMake fails with Nvidia compilers when the GPU architecture option is not supplied (Fix configure with OMPT and Cuda) [\#3207](https://github.com/kokkos/kokkos/issues/3207) +- PGI compiler being passed the gcc -fopenmp flag [\#3125](https://github.com/kokkos/kokkos/issues/3125) +- Cuda: Memory leak when using CUDA stream [\#3167](https://github.com/kokkos/kokkos/issues/3167) +- RangePolicy has an implicitly deleted assignment operator [\#3192](https://github.com/kokkos/kokkos/issues/3192) +- MemorySpace::allocate needs to have memory pool counting. [\#3064](https://github.com/kokkos/kokkos/issues/3064) +- Missing write fence for lock based atomics on CUDA [\#3038](https://github.com/kokkos/kokkos/issues/3038) +- CUDA compute capability version check problem [\#3026](https://github.com/kokkos/kokkos/issues/3026) +- Make DynRankView fencing consistent [\#3014](https://github.com/kokkos/kokkos/issues/3014) +- nvcc\_wrapper cant handle -Xcompiler -o out.o [\#2993](https://github.com/kokkos/kokkos/issues/2993) +- Reductions of non-trivial types of size 4 fail in CUDA shfl operations [\#2990](https://github.com/kokkos/kokkos/issues/2990) +- complex\_double misalignment in reduce, clang+CUDA [\#2989](https://github.com/kokkos/kokkos/issues/2989) +- Span of degenerated \(zero-length\) subviews is not zero in some special cases [\#2979](https://github.com/kokkos/kokkos/issues/2979) +- Rank 1 custom layouts dont work as expected. [\#2840](https://github.com/kokkos/kokkos/issues/2840) + +## [3.1.01](https://github.com/kokkos/kokkos/tree/3.1.1) (2020-04-14) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.1.00...3.1.1) + +**Fixed bugs:** + +- Fix complex_double misalignment in reduce, clang+CUDA [\#2989](https://github.com/kokkos/kokkos/issues/2989) +- Fix compilation fails when profiling disabled and CUDA enabled [\#3001](https://github.com/kokkos/kokkos/issues/3001) +- Fix cuda reduction of non-trivial scalars of size 4 [\#2990](https://github.com/kokkos/kokkos/issues/2990) +- Configure and install version file when building in Trilinos [\#2957](https://github.com/kokkos/kokkos/pull/2957) +- Fix OpenMPTarget build missing include and namespace [\#3000](https://github.com/kokkos/kokkos/issues/3000) +- fix typo in KOKKOS_SET_EXE_PROPERTY() [\#2959](https://github.com/kokkos/kokkos/issues/2959) +- Fix non-zero span subviews of zero sized subviews [\#2979](https://github.com/kokkos/kokkos/issues/2979) + +## [3.1.00](https://github.com/kokkos/kokkos/tree/3.1.00) (2020-04-14) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.0.00...3.1.00) + +**Features:** + +- HIP Support for AMD +- OpenMPTarget Support with clang +- Windows VS19 (Serial) Support [\#1533](https://github.com/kokkos/kokkos/issues/1533) + +**Implemented enhancements:** + +- generate\_makefile.bash should allow tests to be disabled [\#2886](https://github.com/kokkos/kokkos/issues/2886) +- clang/7+cuda/9 build -Werror-unused parameter error in nightly test [\#2884](https://github.com/kokkos/kokkos/issues/2884) +- ScatterView memory space is not user settable [\#2826](https://github.com/kokkos/kokkos/issues/2826) +- clang/8+cuda/10.0 build error with c++17 [\#2809](https://github.com/kokkos/kokkos/issues/2809) +- warnings.... [\#2805](https://github.com/kokkos/kokkos/issues/2805) +- Kokkos version in cpp define [\#2787](https://github.com/kokkos/kokkos/issues/2787) +- Remove Defunct QThreads Backend [\#2751](https://github.com/kokkos/kokkos/issues/2751) +- Improve Kokkos::fence behavior with multiple execution spaces [\#2659](https://github.com/kokkos/kokkos/issues/2659) +- polylithic\(?\) initialization of Kokkos [\#2658](https://github.com/kokkos/kokkos/issues/2658) +- Unnecessary\(?\) check for host execution space initialization from Cuda initialization [\#2652](https://github.com/kokkos/kokkos/issues/2652) +- Kokkos error reporting failures with CUDA GPUs in exclusive mode [\#2471](https://github.com/kokkos/kokkos/issues/2471) +- atomicMax equivalent \(and other atomics\) [\#2401](https://github.com/kokkos/kokkos/issues/2401) +- Fix alignment for Kokkos::complex [\#2255](https://github.com/kokkos/kokkos/issues/2255) +- Warnings with Cuda 10.1 [\#2206](https://github.com/kokkos/kokkos/issues/2206) +- dual view with Kokkos::ViewAllocateWithoutInitializing [\#2188](https://github.com/kokkos/kokkos/issues/2188) +- Check error code from cudaOccupancyMaxActiveBlocksPerMultiprocessor [\#2172](https://github.com/kokkos/kokkos/issues/2172) +- Add non-member Kokkos::resize/realloc for DualView [\#2170](https://github.com/kokkos/kokkos/issues/2170) +- Construct DualView without initialization [\#2046](https://github.com/kokkos/kokkos/issues/2046) +- Expose is\_assignable to determine if one view can be assigned to another [\#1936](https://github.com/kokkos/kokkos/issues/1936) +- profiling label [\#1935](https://github.com/kokkos/kokkos/issues/1935) +- team\_broadcast of bool failed on CUDA backend [\#1908](https://github.com/kokkos/kokkos/issues/1908) +- View static\_extent [\#660](https://github.com/kokkos/kokkos/issues/660) +- Misleading Kokkos::Cuda::initialize ERROR message when compiled for wrong GPU architecture [\#1944](https://github.com/kokkos/kokkos/issues/1944) +- Cryptic Error When Malloc Fails [\#2164](https://github.com/kokkos/kokkos/issues/2164) +- Drop support for intermediate standards in CMake [\#2336](https://github.com/kokkos/kokkos/issues/2336) + +**Fixed bugs:** + +- DualView sync\_device with length zero creates cuda errors [\#2946](https://github.com/kokkos/kokkos/issues/2946) +- building with nvcc and clang \(or clang based XL\) as host compiler: "Kokkos::atomic\_fetch\_min\(volatile int \*, int\)" has already been defined [\#2903](https://github.com/kokkos/kokkos/issues/2903) +- Cuda 9.1,10.1 debug builds failing due to -Werror=unused-parameter [\#2880](https://github.com/kokkos/kokkos/issues/2880) +- clang -Werror: Kokkos\_FixedBufferMemoryPool.hpp:140:28: error: unused parameter 'alloc\_size' [\#2869](https://github.com/kokkos/kokkos/issues/2869) +- intel/16.0.1, intel/17.0.1 nightly build failures with debugging enabled [\#2867](https://github.com/kokkos/kokkos/issues/2867) +- intel/16.0.1 debug build errors [\#2863](https://github.com/kokkos/kokkos/issues/2863) +- xl/16.1.1 with cpp14, openmp build, nightly test failures [\#2856](https://github.com/kokkos/kokkos/issues/2856) +- Intel nightly test failures: team\_vector [\#2852](https://github.com/kokkos/kokkos/issues/2852) +- Kokkos Views with intmax/2\\> for complex\ uses std::ostream, not std::istream [\#2313](https://github.com/kokkos/kokkos/issues/2313) +- Macros: Restrict not honored for non-intel compilers [\#1922](https://github.com/kokkos/kokkos/issues/1922) + + +## [2.9.00](https://github.com/kokkos/kokkos/tree/2.9.00) (2019-06-24) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.8.00...2.9.00) + +**Implemented enhancements:** + +- Capability: CUDA Streams [\#1723](https://github.com/kokkos/kokkos/issues/1723) +- Capability: CUDA Stream support for parallel\_reduce [\#2061](https://github.com/kokkos/kokkos/issues/2061) +- Capability: Feature Request: TeamVectorRange [\#713](https://github.com/kokkos/kokkos/issues/713) +- Capability: Adding HPX backend [\#2080](https://github.com/kokkos/kokkos/issues/2080) +- Capability: TaskScheduler to have multiple queues [\#565](https://github.com/kokkos/kokkos/issues/565) +- Capability: Support for additional reductions in ScatterView [\#1674](https://github.com/kokkos/kokkos/issues/1674) +- Capability: Request: deep\_copy within parallel regions [\#689](https://github.com/kokkos/kokkos/issues/689) +- Capability: Feature Request: `create\_mirror\_view\_without\_initializing` [\#1765](https://github.com/kokkos/kokkos/issues/1765) +- View: Use SFINAE to restrict possible View type conversions [\#2127](https://github.com/kokkos/kokkos/issues/2127) +- Deprecation: Deprecate ExecutionSpace::fence\(\) as static function and make it non-static [\#2140](https://github.com/kokkos/kokkos/issues/2140) +- Deprecation: Deprecate LayoutTileLeft [\#2122](https://github.com/kokkos/kokkos/issues/2122) +- Macros: KOKKOS\_RESTRICT defined for non-Intel compilers [\#2038](https://github.com/kokkos/kokkos/issues/2038) + +**Fixed bugs:** + +- Cuda: TeamThreadRange loop count on device is passed by reference to host static constexpr [\#1733](https://github.com/kokkos/kokkos/issues/1733) +- Cuda: Build error with relocatable device code with CUDA 10.1 GCC 7.3 [\#2134](https://github.com/kokkos/kokkos/issues/2134) +- Cuda: cudaFuncSetCacheConfig is setting CachePreferShared too often [\#2066](https://github.com/kokkos/kokkos/issues/2066) +- Cuda: TeamPolicy doesn't throw then created with non-viable vector length and also doesn't backscale to viable one [\#2020](https://github.com/kokkos/kokkos/issues/2020) +- Cuda: cudaMemcpy error for large league sizes on V100 [\#1991](https://github.com/kokkos/kokkos/issues/1991) +- Cuda: illegal warp sync in parallel\_reduce by functor on Turing 75 [\#1958](https://github.com/kokkos/kokkos/issues/1958) +- TeamThreadRange: Inconsistent results from TeamThreadRange reduction [\#1905](https://github.com/kokkos/kokkos/issues/1905) +- Atomics: atomic\_fetch\_oper & atomic\_oper\_fetch don't build for complex\ [\#1964](https://github.com/kokkos/kokkos/issues/1964) +- Views: Kokkos randomread Views leak memory [\#2155](https://github.com/kokkos/kokkos/issues/2155) +- ScatterView: LayoutLeft overload currently non-functional [\#2165](https://github.com/kokkos/kokkos/issues/2165) +- KNL: With intel 17.2.174 illegal instruction in random number test [\#2078](https://github.com/kokkos/kokkos/issues/2078) +- Bitset: Enable copy constructor on device [\#2094](https://github.com/kokkos/kokkos/issues/2094) +- Examples: do not compile due to template deduction error \(multi\_fem\) [\#1928](https://github.com/kokkos/kokkos/issues/1928) + +## [2.8.00](https://github.com/kokkos/kokkos/tree/2.8.00) (2019-02-05) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.7.24...2.8.00) + +**Implemented enhancements:** + +- Capability, Tests: C++14 support and testing [\#1914](https://github.com/kokkos/kokkos/issues/1914) +- Capability: Add environment variables for all command line arguments [\#1798](https://github.com/kokkos/kokkos/issues/1798) +- Capability: --kokkos-ndevices not working for Slurm [\#1920](https://github.com/kokkos/kokkos/issues/1920) +- View: Undefined behavior when deep copying from and to an empty unmanaged view [\#1967](https://github.com/kokkos/kokkos/issues/1967) +- BuildSystem: nvcc\_wrapper should stop immediately if nvcc is not in PATH [\#1861](https://github.com/kokkos/kokkos/issues/1861) + +**Fixed bugs:** + +- Cuda: Fix Volta Issues 1 Non-deterministic behavior on Volta, runs fine on Pascal [\#1949](https://github.com/kokkos/kokkos/issues/1949) +- Cuda: Fix Volta Issues 2 CUDA Team Scan gives wrong values on Volta with -G compile flag [\#1942](https://github.com/kokkos/kokkos/issues/1942) +- Cuda: illegal warp sync in parallel\_reduce by functor on Turing 75 [\#1958](https://github.com/kokkos/kokkos/issues/1958) +- Threads: Pthreads backend does not handle RangePolicy with offset correctly [\#1976](https://github.com/kokkos/kokkos/issues/1976) +- Atomics: atomic\_fetch\_oper has no case for Kokkos::complex\ or other 16-byte types [\#1951](https://github.com/kokkos/kokkos/issues/1951) +- MDRangePolicy: Fix zero-length range [\#1948](https://github.com/kokkos/kokkos/issues/1948) +- TeamThreadRange: TeamThreadRange MaxLoc reduce doesnt compile [\#1909](https://github.com/kokkos/kokkos/issues/1909) + +## [2.7.24](https://github.com/kokkos/kokkos/tree/2.7.24) (2018-11-04) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.7.00...2.7.24) + +**Implemented enhancements:** + +- DualView: Add non-templated functions for sync, need\_sync, view, modify [\#1858](https://github.com/kokkos/kokkos/issues/1858) +- DualView: Avoid needlessly allocates and initializes modify\_host and modify\_device flag views [\#1831](https://github.com/kokkos/kokkos/issues/1831) +- DualView: Incorrect deduction of "not device type" [\#1659](https://github.com/kokkos/kokkos/issues/1659) +- BuildSystem: Add KOKKOS\_ENABLE\_CXX14 and KOKKOS\_ENABLE\_CXX17 [\#1602](https://github.com/kokkos/kokkos/issues/1602) +- BuildSystem: Installed kokkos\_generated\_settings.cmake contains build directories instead of install directories [\#1838](https://github.com/kokkos/kokkos/issues/1838) +- BuildSystem: KOKKOS\_ARCH: add ticks to printout of improper arch setting [\#1649](https://github.com/kokkos/kokkos/issues/1649) +- BuildSystem: Make core/src/Makefile for Cuda use needed nvcc\_wrapper [\#1296](https://github.com/kokkos/kokkos/issues/1296) +- Build: Support PGI as host compiler for NVCC [\#1828](https://github.com/kokkos/kokkos/issues/1828) +- Build: Many Warnings Fixed e.g.[\#1786](https://github.com/kokkos/kokkos/issues/1786) +- Capability: OffsetView with non-zero begin index [\#567](https://github.com/kokkos/kokkos/issues/567) +- Capability: Reductions into device side view [\#1788](https://github.com/kokkos/kokkos/issues/1788) +- Capability: Add max\_size to Kokkos::Array [\#1760](https://github.com/kokkos/kokkos/issues/1760) +- Capability: View Assignment: LayoutStride -\> LayoutLeft and LayoutStride -\> LayoutRight [\#1594](https://github.com/kokkos/kokkos/issues/1594) +- Capability: Atomic function allow implicit conversion of update argument [\#1571](https://github.com/kokkos/kokkos/issues/1571) +- Capability: Add team\_size\_max with tagged functors [\#663](https://github.com/kokkos/kokkos/issues/663) +- Capability: Fix allignment of views from Kokkos\_ScratchSpace should use different alignment [\#1700](https://github.com/kokkos/kokkos/issues/1700) +- Capabilitiy: create\_mirror\_view\_and\_copy for DynRankView [\#1651](https://github.com/kokkos/kokkos/issues/1651) +- Capability: DeepCopy HBWSpace / HostSpace [\#548](https://github.com/kokkos/kokkos/issues/548) +- ROCm: support team vector scan [\#1645](https://github.com/kokkos/kokkos/issues/1645) +- ROCm: Merge from rocm-hackathon2 [\#1636](https://github.com/kokkos/kokkos/issues/1636) +- ROCm: Add ParallelScanWithTotal [\#1611](https://github.com/kokkos/kokkos/issues/1611) +- ROCm: Implement MDRange in ROCm [\#1314](https://github.com/kokkos/kokkos/issues/1314) +- ROCm: Implement Reducers for Nested Parallelism Levels [\#963](https://github.com/kokkos/kokkos/issues/963) +- ROCm: Add asynchronous deep copy [\#959](https://github.com/kokkos/kokkos/issues/959) +- Tests: Memory pool test seems to allocate 8GB [\#1830](https://github.com/kokkos/kokkos/issues/1830) +- Tests: Add unit\_test for team\_broadcast [\#734](https://github.com/kokkos/kokkos/issues/734) + +**Fixed bugs:** + +- BuildSystem: Makefile.kokkos gets gcc-toolchain wrong if gcc is cached [\#1841](https://github.com/kokkos/kokkos/issues/1841) +- BuildSystem: kokkos\_generated\_settings.cmake placement is inconsistent [\#1771](https://github.com/kokkos/kokkos/issues/1771) +- BuildSystem: Invalid escape sequence \. in kokkos\_functions.cmake [\#1661](https://github.com/kokkos/kokkos/issues/1661) +- BuildSystem: Problem in Kokkos generated cmake file [\#1770](https://github.com/kokkos/kokkos/issues/1770) +- BuildSystem: invalid file names on windows [\#1671](https://github.com/kokkos/kokkos/issues/1671) +- Tests: reducers min/max\_loc test fails randomly due to multiple min values and thus multiple valid locations [\#1681](https://github.com/kokkos/kokkos/issues/1681) +- Tests: cuda.scatterview unit test causes "Bus error" when force\_uvm and enable\_lambda are enabled [\#1852](https://github.com/kokkos/kokkos/issues/1852) +- Tests: cuda.cxx11 unit test fails when force\_uvm and enable\_lambda are enabled [\#1850](https://github.com/kokkos/kokkos/issues/1850) +- Tests: threads.reduce\_device\_view\_range\_policy failing with Cuda/8.0.44 and RDC [\#1836](https://github.com/kokkos/kokkos/issues/1836) +- Build: compile error when compiling Kokkos with hwloc 2.0.1 \(on OSX 10.12.6, with g++ 7.2.0\) [\#1506](https://github.com/kokkos/kokkos/issues/1506) +- Build: dual\_view.view broken with UVM [\#1834](https://github.com/kokkos/kokkos/issues/1834) +- Build: White cuda/9.2 + gcc/7.2 warnings triggering errors [\#1833](https://github.com/kokkos/kokkos/issues/1833) +- Build: warning: enum constant in boolean context [\#1813](https://github.com/kokkos/kokkos/issues/1813) +- Capability: Fix overly conservative max\_team\_size thingy [\#1808](https://github.com/kokkos/kokkos/issues/1808) +- DynRankView: Ctors taking ViewAllocateWithoutInitializing broken [\#1783](https://github.com/kokkos/kokkos/issues/1783) +- Cuda: Apollo cuda.team\_broadcast test fail with clang-6.0 [\#1762](https://github.com/kokkos/kokkos/issues/1762) +- Cuda: Clang spurious test failure in impl\_view\_accessible [\#1753](https://github.com/kokkos/kokkos/issues/1753) +- Cuda: Kokkos::complex\ atomic deadlocks with Clang 6 Cuda build with -O0 [\#1752](https://github.com/kokkos/kokkos/issues/1752) +- Cuda: LayoutStride Test fails for UVM as default memory space [\#1688](https://github.com/kokkos/kokkos/issues/1688) +- Cuda: Scan wrong values on Volta [\#1676](https://github.com/kokkos/kokkos/issues/1676) +- Cuda: Kokkos::deep\_copy error with CudaUVM and Kokkos::Serial spaces [\#1652](https://github.com/kokkos/kokkos/issues/1652) +- Cuda: cudaErrorInvalidConfiguration with debug build [\#1647](https://github.com/kokkos/kokkos/issues/1647) +- Cuda: parallel\_for with TeamPolicy::team\_size\_recommended with launch bounds not working -- reported by Daniel Holladay [\#1283](https://github.com/kokkos/kokkos/issues/1283) +- Cuda: Using KOKKOS\_CLASS\_LAMBDA in a class with Kokkos::Random\_XorShift64\_Pool member data [\#1696](https://github.com/kokkos/kokkos/issues/1696) +- Long Build Times on Darwin [\#1721](https://github.com/kokkos/kokkos/issues/1721) +- Capability: Typo in Kokkos\_Sort.hpp - BinOp3D - wrong comparison [\#1720](https://github.com/kokkos/kokkos/issues/1720) +- Buffer overflow in SharedAllocationRecord in Kokkos\_HostSpace.cpp [\#1673](https://github.com/kokkos/kokkos/issues/1673) +- Serial unit test failure [\#1632](https://github.com/kokkos/kokkos/issues/1632) + +## [2.7.00](https://github.com/kokkos/kokkos/tree/2.7.00) (2018-05-24) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.6.00...2.7.00) + +**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.7** + +**Implemented enhancements:** + +- Deprecate team\_size auto adjusting to maximal value possible [\#1618](https://github.com/kokkos/kokkos/issues/1618) +- DynamicView - remove restrictions to std::is\_trivial types and value\_type is power of two [\#1586](https://github.com/kokkos/kokkos/issues/1586) +- Kokkos::StaticCrsGraph does not propagate memory traits \(e.g., Unmanaged\) [\#1581](https://github.com/kokkos/kokkos/issues/1581) +- Adding ETI for DeepCopy / ViewFill etc. [\#1578](https://github.com/kokkos/kokkos/issues/1578) +- Deprecate all the left over KOKKOS\_HAVE\_ Macros and Kokkos\_OldMacros.hpp [\#1572](https://github.com/kokkos/kokkos/issues/1572) +- Error if Kokkos\_ARCH set in CMake [\#1555](https://github.com/kokkos/kokkos/issues/1555) +- Deprecate ExecSpace::initialize / ExecSpace::finalize [\#1532](https://github.com/kokkos/kokkos/issues/1532) +- New API for TeamPolicy property setting [\#1531](https://github.com/kokkos/kokkos/issues/1531) +- clang 6.0 + cuda debug out-of-memory test failure [\#1521](https://github.com/kokkos/kokkos/issues/1521) +- Cuda UniqueToken interface not consistent with other backends [\#1505](https://github.com/kokkos/kokkos/issues/1505) +- Move Reducers out of Experimental namespace [\#1494](https://github.com/kokkos/kokkos/issues/1494) +- Provide scope guard for initialize/finalize [\#1479](https://github.com/kokkos/kokkos/issues/1479) +- Check Kokkos::is\_initialized in SharedAllocationRecord dtor [\#1465](https://github.com/kokkos/kokkos/issues/1465) +- Remove static list of allocations [\#1464](https://github.com/kokkos/kokkos/issues/1464) +- Makefiles: Support single compile/link line use case [\#1402](https://github.com/kokkos/kokkos/issues/1402) +- ThreadVectorRange with a range [\#1400](https://github.com/kokkos/kokkos/issues/1400) +- Exclusive scan + last value API [\#1358](https://github.com/kokkos/kokkos/issues/1358) +- Install kokkos\_generated\_settings.cmake [\#1348](https://github.com/kokkos/kokkos/issues/1348) +- Kokkos arrays \(not views!\) don't do bounds checking in debug mode [\#1342](https://github.com/kokkos/kokkos/issues/1342) +- Expose round-robin GPU assignment outside of initialize\(int, char\*\*\) [\#1318](https://github.com/kokkos/kokkos/issues/1318) +- DynamicView misses use\_count and label function [\#1298](https://github.com/kokkos/kokkos/issues/1298) +- View constructor should check arguments [\#1286](https://github.com/kokkos/kokkos/issues/1286) +- False Positive on Oversubscription Warning [\#1207](https://github.com/kokkos/kokkos/issues/1207) +- Allow \(require\) execution space for 1st arg of VerifyExecutionCanAccessMemorySpace [\#1192](https://github.com/kokkos/kokkos/issues/1192) +- ROCm: Add ROCmHostPinnedSpace [\#958](https://github.com/kokkos/kokkos/issues/958) +- power of two functions [\#656](https://github.com/kokkos/kokkos/issues/656) +- CUDA 8 has 64bit \_\_shfl [\#361](https://github.com/kokkos/kokkos/issues/361) +- Add TriBITS/CMake configure information about node types [\#243](https://github.com/kokkos/kokkos/issues/243) + +**Fixed bugs:** + +- CUDA atomic\_fetch\_sub for doubles is hitting CAS instead of intrinsic [\#1624](https://github.com/kokkos/kokkos/issues/1624) +- Bug: use of ballot on Volta [\#1612](https://github.com/kokkos/kokkos/issues/1612) +- Kokkos::deep\_copy memory access failures [\#1583](https://github.com/kokkos/kokkos/issues/1583) +- g++ -std option doubly set for cmake project [\#1548](https://github.com/kokkos/kokkos/issues/1548) +- ViewFill for 1D Views of larger 32bit entries fails [\#1541](https://github.com/kokkos/kokkos/issues/1541) +- CUDA Volta another warpsync bug [\#1520](https://github.com/kokkos/kokkos/issues/1520) +- triple\_nested\_parallelism fails with KOKKOS\_DEBUG and CUDA [\#1513](https://github.com/kokkos/kokkos/issues/1513) +- Jenkins errors in Kokkos\_SharedAlloc.cpp with debug build [\#1511](https://github.com/kokkos/kokkos/issues/1511) +- Kokkos::Sort out-of-bounds with empty bins [\#1504](https://github.com/kokkos/kokkos/issues/1504) +- Get rid of deprecated functions inside Kokkos [\#1484](https://github.com/kokkos/kokkos/issues/1484) +- get\_work\_partition casts int64\_t to int, causing a seg fault [\#1481](https://github.com/kokkos/kokkos/issues/1481) +- NVCC bug with \_\_device\_\_ on defaulted function [\#1470](https://github.com/kokkos/kokkos/issues/1470) +- CMake example broken with CUDA backend [\#1468](https://github.com/kokkos/kokkos/issues/1468) + + +## [2.6.00](https://github.com/kokkos/kokkos/tree/2.6.00) (2018-03-07) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.5.00...2.6.00) + +**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.6** + +**Implemented enhancements:** + +- Support NVIDIA Volta microarchitecture [\#1466](https://github.com/kokkos/kokkos/issues/1466) +- Kokkos - Define empty functions when profiling disabled [\#1424](https://github.com/kokkos/kokkos/issues/1424) +- Don't use \_\_constant\_\_ cache for lock arrays, enable once per run update instead of once per call [\#1385](https://github.com/kokkos/kokkos/issues/1385) +- task dag enhancement. [\#1354](https://github.com/kokkos/kokkos/issues/1354) +- Cuda task team collectives and stack size [\#1353](https://github.com/kokkos/kokkos/issues/1353) +- Replace View operator acceptance of more than rank integers with 'access' function [\#1333](https://github.com/kokkos/kokkos/issues/1333) +- Interoperability: Do not shut down backend execution space runtimes upon calling finalize. [\#1305](https://github.com/kokkos/kokkos/issues/1305) +- shmem\_size for LayoutStride [\#1291](https://github.com/kokkos/kokkos/issues/1291) +- Kokkos::resize performs poorly on 1D Views [\#1270](https://github.com/kokkos/kokkos/issues/1270) +- stride\(\) is inconsistent with dimension\(\), extent\(\), etc. [\#1214](https://github.com/kokkos/kokkos/issues/1214) +- Kokkos::sort defaults to std::sort on host [\#1208](https://github.com/kokkos/kokkos/issues/1208) +- DynamicView with host size grow [\#1206](https://github.com/kokkos/kokkos/issues/1206) +- Unmanaged View with Anonymous Memory Space [\#1175](https://github.com/kokkos/kokkos/issues/1175) +- Sort subset of Kokkos::DynamicView [\#1160](https://github.com/kokkos/kokkos/issues/1160) +- MDRange policy doesn't support lambda reductions [\#1054](https://github.com/kokkos/kokkos/issues/1054) +- Add ability to set hook on Kokkos::finalize [\#714](https://github.com/kokkos/kokkos/issues/714) +- Atomics with Serial Backend - Default should be Disable? [\#549](https://github.com/kokkos/kokkos/issues/549) +- KOKKOS\_ENABLE\_DEPRECATED\_CODE [\#1359](https://github.com/kokkos/kokkos/issues/1359) + +**Fixed bugs:** + +- cuda\_internal\_maximum\_warp\_count returns 8, but I believe it should return 16 for P100 [\#1269](https://github.com/kokkos/kokkos/issues/1269) +- Cuda: level 1 scratch memory bug \(reported by Stan Moore\) [\#1434](https://github.com/kokkos/kokkos/issues/1434) +- MDRangePolicy Reduction requires value\_type typedef in Functor [\#1379](https://github.com/kokkos/kokkos/issues/1379) +- Kokkos DeepCopy between empty views fails [\#1369](https://github.com/kokkos/kokkos/issues/1369) +- Several issues with new CMake build infrastructure \(reported by Eric Phipps\) [\#1365](https://github.com/kokkos/kokkos/issues/1365) +- deep\_copy between rank-1 host/device views of differing layouts without UVM no longer works \(reported by Eric Phipps\) [\#1363](https://github.com/kokkos/kokkos/issues/1363) +- Profiling can't be disabled in CMake, and a parallel\_for is missing for tasks \(reported by Kyungjoo Kim\) [\#1349](https://github.com/kokkos/kokkos/issues/1349) +- get\_work\_partition int overflow \(reported by berryj5\) [\#1327](https://github.com/kokkos/kokkos/issues/1327) +- Kokkos::deep\_copy must fence even if the two views are the same [\#1303](https://github.com/kokkos/kokkos/issues/1303) +- CudaUVMSpace::allocate/deallocate must fence [\#1302](https://github.com/kokkos/kokkos/issues/1302) +- ViewResize on CUDA fails in Debug because of too many resources requested [\#1299](https://github.com/kokkos/kokkos/issues/1299) +- Cuda 9 and intrepid2 calls from Panzer. [\#1183](https://github.com/kokkos/kokkos/issues/1183) +- Slowdown due to tracking\_enabled\(\) in 2.04.00 \(found by Albany app\) [\#1016](https://github.com/kokkos/kokkos/issues/1016) +- Bounds checking fails with zero-span Views \(reported by Stan Moore\) [\#1411](https://github.com/kokkos/kokkos/issues/1411) + + +## [2.5.00](https://github.com/kokkos/kokkos/tree/2.5.00) (2017-12-15) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.04.11...2.5.00) + +**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.5** + +**Implemented enhancements:** + +- Provide Makefile.kokkos logic for CMake and TriBITS [\#878](https://github.com/kokkos/kokkos/issues/878) +- Add Scatter View [\#825](https://github.com/kokkos/kokkos/issues/825) +- Drop gcc 4.7 and intel 14 from supported compiler list [\#603](https://github.com/kokkos/kokkos/issues/603) +- Enable construction of unmanaged view using common\_view\_alloc\_prop [\#1170](https://github.com/kokkos/kokkos/issues/1170) +- Unused Function Warning with XL [\#1267](https://github.com/kokkos/kokkos/issues/1267) +- Add memory pool parameter check [\#1218](https://github.com/kokkos/kokkos/issues/1218) +- CUDA9: Fix warning for unsupported long double [\#1189](https://github.com/kokkos/kokkos/issues/1189) +- CUDA9: fix warning on defaulted function marking [\#1188](https://github.com/kokkos/kokkos/issues/1188) +- CUDA9: fix warnings for deprecated warp level functions [\#1187](https://github.com/kokkos/kokkos/issues/1187) +- Add CUDA 9.0 nightly testing [\#1174](https://github.com/kokkos/kokkos/issues/1174) +- {OMPI,MPICH}\_CXX hack breaks nvcc\_wrapper use case [\#1166](https://github.com/kokkos/kokkos/issues/1166) +- KOKKOS\_HAVE\_CUDA\_LAMBDA became KOKKOS\_CUDA\_USE\_LAMBDA [\#1274](https://github.com/kokkos/kokkos/issues/1274) + +**Fixed bugs:** + +- MinMax Reducer with tagged operator doesn't compile [\#1251](https://github.com/kokkos/kokkos/issues/1251) +- Reducers for Tagged operators give wrong answer [\#1250](https://github.com/kokkos/kokkos/issues/1250) +- Kokkos not Compatible with Big Endian Machines? [\#1235](https://github.com/kokkos/kokkos/issues/1235) +- Parallel Scan hangs forever on BG/Q [\#1234](https://github.com/kokkos/kokkos/issues/1234) +- Threads backend doesn't compile with Clang on OS X [\#1232](https://github.com/kokkos/kokkos/issues/1232) +- $\(shell date\) needs quote [\#1264](https://github.com/kokkos/kokkos/issues/1264) +- Unqualified parallel\_for call conflicts with user-defined parallel\_for [\#1219](https://github.com/kokkos/kokkos/issues/1219) +- KokkosAlgorithms: CMake issue in unit tests [\#1212](https://github.com/kokkos/kokkos/issues/1212) +- Intel 18 Error: "simd pragma has been deprecated" [\#1210](https://github.com/kokkos/kokkos/issues/1210) +- Memory leak in Kokkos::initialize [\#1194](https://github.com/kokkos/kokkos/issues/1194) +- CUDA9: compiler error with static assert template arguments [\#1190](https://github.com/kokkos/kokkos/issues/1190) +- Kokkos::Serial::is\_initialized returns always true [\#1184](https://github.com/kokkos/kokkos/issues/1184) +- Triple nested parallelism still fails on bowman [\#1093](https://github.com/kokkos/kokkos/issues/1093) +- OpenMP openmp.range on Develop Runs Forever on POWER7+ with RHEL7 and GCC4.8.5 [\#995](https://github.com/kokkos/kokkos/issues/995) +- Rendezvous performance at global scope [\#985](https://github.com/kokkos/kokkos/issues/985) + + +## [2.04.11](https://github.com/kokkos/kokkos/tree/2.04.11) (2017-10-28) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.04.04...2.04.11) + +**Implemented enhancements:** + +- Add Subview pattern. [\#648](https://github.com/kokkos/kokkos/issues/648) +- Add Kokkos "global" is\_initialized [\#1060](https://github.com/kokkos/kokkos/issues/1060) +- Add create\_mirror\_view\_and\_copy [\#1161](https://github.com/kokkos/kokkos/issues/1161) +- Add KokkosConcepts SpaceAccessibility function [\#1092](https://github.com/kokkos/kokkos/issues/1092) +- Option to Disable Initialize Warnings [\#1142](https://github.com/kokkos/kokkos/issues/1142) +- Mature task-DAG capability [\#320](https://github.com/kokkos/kokkos/issues/320) +- Promote Work DAG from experimental [\#1126](https://github.com/kokkos/kokkos/issues/1126) +- Implement new WorkGraph push/pop [\#1108](https://github.com/kokkos/kokkos/issues/1108) +- Kokkos\_ENABLE\_Cuda\_Lambda should default ON [\#1101](https://github.com/kokkos/kokkos/issues/1101) +- Add multidimensional parallel for example and improve unit test [\#1064](https://github.com/kokkos/kokkos/issues/1064) +- Fix ROCm: Performance tests not building [\#1038](https://github.com/kokkos/kokkos/issues/1038) +- Make KOKKOS\_ALIGN\_SIZE a configure-time option [\#1004](https://github.com/kokkos/kokkos/issues/1004) +- Make alignment consistent [\#809](https://github.com/kokkos/kokkos/issues/809) +- Improve subview construction on Cuda backend [\#615](https://github.com/kokkos/kokkos/issues/615) + +**Fixed bugs:** + +- Kokkos::vector fixes for application [\#1134](https://github.com/kokkos/kokkos/issues/1134) +- DynamicView non-power of two value\_type [\#1177](https://github.com/kokkos/kokkos/issues/1177) +- Memory pool bug [\#1154](https://github.com/kokkos/kokkos/issues/1154) +- Cuda launch bounds performance regression bug [\#1140](https://github.com/kokkos/kokkos/issues/1140) +- Significant performance regression in LAMMPS after updating Kokkos [\#1139](https://github.com/kokkos/kokkos/issues/1139) +- CUDA compile error [\#1128](https://github.com/kokkos/kokkos/issues/1128) +- MDRangePolicy neg idx test failure in debug mode [\#1113](https://github.com/kokkos/kokkos/issues/1113) +- subview construction on Cuda backend [\#615](https://github.com/kokkos/kokkos/issues/615) + +## [2.04.04](https://github.com/kokkos/kokkos/tree/2.04.04) (2017-09-11) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.04.00...2.04.04) + +**Implemented enhancements:** + +- OpenMP partition: set number of threads on nested level [\#1082](https://github.com/kokkos/kokkos/issues/1082) +- Add StaticCrsGraph row\(\) method [\#1071](https://github.com/kokkos/kokkos/issues/1071) +- Enhance Kokkos complex operator overloading [\#1052](https://github.com/kokkos/kokkos/issues/1052) +- Tell Trilinos packages about host+device lambda [\#1019](https://github.com/kokkos/kokkos/issues/1019) +- Function markup for defaulted class members [\#952](https://github.com/kokkos/kokkos/issues/952) +- Add deterministic random number generator [\#857](https://github.com/kokkos/kokkos/issues/857) + +**Fixed bugs:** + +- Fix reduction\_identity\::max for floating point numbers [\#1048](https://github.com/kokkos/kokkos/issues/1048) +- Fix MD iteration policy ignores lower bound on GPUs [\#1041](https://github.com/kokkos/kokkos/issues/1041) +- (Experimental) HBWSpace Linking issues in KokkosKernels [\#1094](https://github.com/kokkos/kokkos/issues/1094) +- (Experimental) ROCm: algorithms/unit\_tests test\_sort failing with segfault [\#1070](https://github.com/kokkos/kokkos/issues/1070) + +## [2.04.00](https://github.com/kokkos/kokkos/tree/2.04.00) (2017-08-16) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.03.13...2.04.00) + +**Implemented enhancements:** + +- Added ROCm backend to support AMD GPUs +- Kokkos::complex\ behaves slightly differently from std::complex\ [\#1011](https://github.com/kokkos/kokkos/issues/1011) +- Kokkos::Experimental::Crs constructor arguments were in the wrong order [\#992](https://github.com/kokkos/kokkos/issues/992) +- Work graph construction ease-of-use (one lambda for count and fill) [\#991](https://github.com/kokkos/kokkos/issues/991) +- when\_all returns pointer of futures (improved interface) [\#990](https://github.com/kokkos/kokkos/issues/990) +- Allow assignment of LayoutLeft to LayoutRight or vice versa for rank-0 Views [\#594](https://github.com/kokkos/kokkos/issues/594) +- Changed the meaning of Kokkos\_ENABLE\_CXX11\_DISPATCH\_LAMBDA [\#1035](https://github.com/kokkos/kokkos/issues/1035) + +**Fixed bugs:** + +- memory pool default constructor does not properly set member variables. [\#1007](https://github.com/kokkos/kokkos/issues/1007) + +## [2.03.13](https://github.com/kokkos/kokkos/tree/2.03.13) (2017-07-27) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.03.05...2.03.13) + +**Implemented enhancements:** + +- Disallow enabling both OpenMP and Threads in the same executable [\#406](https://github.com/kokkos/kokkos/issues/406) +- Make Kokkos::OpenMP respect OMP environment even if hwloc is available [\#630](https://github.com/kokkos/kokkos/issues/630) +- Improve Atomics Performance on KNL/Broadwell where PREFETCHW/RFO is Available [\#898](https://github.com/kokkos/kokkos/issues/898) +- Kokkos::resize should test whether dimensions have changed before resizing [\#904](https://github.com/kokkos/kokkos/issues/904) +- Develop performance-regression/acceptance tests [\#737](https://github.com/kokkos/kokkos/issues/737) +- Make the deep\_copy Profiling hook a start/end system [\#890](https://github.com/kokkos/kokkos/issues/890) +- Add deep\_copy Profiling hook [\#843](https://github.com/kokkos/kokkos/issues/843) +- Append tag name to parallel construct name for Profiling [\#842](https://github.com/kokkos/kokkos/issues/842) +- Add view label to `View bounds error` message for CUDA backend [\#870](https://github.com/kokkos/kokkos/issues/870) +- Disable printing the loaded profiling library [\#824](https://github.com/kokkos/kokkos/issues/824) +- "Declared but never referenced" warnings [\#853](https://github.com/kokkos/kokkos/issues/853) +- Warnings about lock\_address\_cuda\_space [\#852](https://github.com/kokkos/kokkos/issues/852) +- WorkGraph execution policy [\#771](https://github.com/kokkos/kokkos/issues/771) +- Simplify makefiles by guarding compilation with appropriate KOKKOS\_ENABLE\_\#\#\# macros [\#716](https://github.com/kokkos/kokkos/issues/716) +- Cmake build: wrong include install directory [\#668](https://github.com/kokkos/kokkos/issues/668) +- Derived View type and allocation [\#566](https://github.com/kokkos/kokkos/issues/566) +- Fix Compiler warnings when compiling core unit tests for Cuda [\#214](https://github.com/kokkos/kokkos/issues/214) + +**Fixed bugs:** + +- Out-of-bounds read in Kokkos\_Layout.hpp [\#975](https://github.com/kokkos/kokkos/issues/975) +- CudaClang: Fix failing test with Clang 4.0 [\#941](https://github.com/kokkos/kokkos/issues/941) +- Respawn when memory pool allocation fails \(not available memory\) [\#940](https://github.com/kokkos/kokkos/issues/940) +- Memory pool aborts on zero allocation request, returns NULL for \< minimum [\#939](https://github.com/kokkos/kokkos/issues/939) +- Error with TaskScheduler query of underlying memory pool [\#917](https://github.com/kokkos/kokkos/issues/917) +- Profiling::\*Callee static variables declared in header [\#863](https://github.com/kokkos/kokkos/issues/863) +- calling \*Space::name\(\) causes compile error [\#862](https://github.com/kokkos/kokkos/issues/862) +- bug in Profiling::deallocateData [\#860](https://github.com/kokkos/kokkos/issues/860) +- task\_depend test failing, CUDA 8.0 + Pascal + RDC [\#829](https://github.com/kokkos/kokkos/issues/829) +- \[develop branch\] Standalone cmake issues [\#826](https://github.com/kokkos/kokkos/issues/826) +- Kokkos CUDA failes to compile with OMPI\_CXX and MPICH\_CXX wrappers [\#776](https://github.com/kokkos/kokkos/issues/776) +- Task Team reduction on Pascal [\#767](https://github.com/kokkos/kokkos/issues/767) +- CUDA stack overflow with TaskDAG test [\#758](https://github.com/kokkos/kokkos/issues/758) +- TeamVector test on Cuda [\#670](https://github.com/kokkos/kokkos/issues/670) +- Clang 4.0 Cuda Build broken again [\#560](https://github.com/kokkos/kokkos/issues/560) + + +## [2.03.05](https://github.com/kokkos/kokkos/tree/2.03.05) (2017-05-27) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.03.00...2.03.05) + +**Implemented enhancements:** + +- Harmonize Custom Reductions over nesting levels [\#802](https://github.com/kokkos/kokkos/issues/802) +- Prevent users directly including KokkosCore\_config.h [\#815](https://github.com/kokkos/kokkos/issues/815) +- DualView aborts on concurrent host/device modify \(in debug mode\) [\#814](https://github.com/kokkos/kokkos/issues/814) +- Abort when running on a NVIDIA CC5.0 or higher architecture with code compiled for CC \< 5.0 [\#813](https://github.com/kokkos/kokkos/issues/813) +- Add "name" function to ExecSpaces [\#806](https://github.com/kokkos/kokkos/issues/806) +- Allow null Future in task spawn dependences [\#795](https://github.com/kokkos/kokkos/issues/795) +- Add Unit Tests for Kokkos::complex [\#785](https://github.com/kokkos/kokkos/issues/785) +- Add pow function for Kokkos::complex [\#784](https://github.com/kokkos/kokkos/issues/784) +- Square root of a complex [\#729](https://github.com/kokkos/kokkos/issues/729) +- Command line processing of --threads argument prevents users from having any commandline arguments starting with --threads [\#760](https://github.com/kokkos/kokkos/issues/760) +- Protected deprecated API with appropriate macro [\#756](https://github.com/kokkos/kokkos/issues/756) +- Allow task scheduler memory pool to be used by tasks [\#747](https://github.com/kokkos/kokkos/issues/747) +- View bounds checking on host-side performance: constructing a std::string [\#723](https://github.com/kokkos/kokkos/issues/723) +- Add check for AppleClang as compiler distinct from check for Clang. [\#705](https://github.com/kokkos/kokkos/issues/705) +- Uninclude source files for specific configurations to prevent link warning. [\#701](https://github.com/kokkos/kokkos/issues/701) +- Add --small option to snapshot script [\#697](https://github.com/kokkos/kokkos/issues/697) +- CMake Standalone Support [\#674](https://github.com/kokkos/kokkos/issues/674) +- CMake build unit test and install [\#808](https://github.com/kokkos/kokkos/issues/808) +- CMake: Fix having kokkos as a subdirectory in a pure cmake project [\#629](https://github.com/kokkos/kokkos/issues/629) +- Tribits macro assumes build directory is in top level source directory [\#654](https://github.com/kokkos/kokkos/issues/654) +- Use bin/nvcc\_wrapper, not config/nvcc\_wrapper [\#562](https://github.com/kokkos/kokkos/issues/562) +- Allow MemoryPool::allocate\(\) to be called from multiple threads per warp. [\#487](https://github.com/kokkos/kokkos/issues/487) +- Allow MemoryPool::allocate\\(\\) to be called from multiple threads per warp. [\#487](https://github.com/kokkos/kokkos/issues/487) +- Move OpenMP 4.5 OpenMPTarget backend into Develop [\#456](https://github.com/kokkos/kokkos/issues/456) +- Testing on ARM testbed [\#288](https://github.com/kokkos/kokkos/issues/288) + +**Fixed bugs:** + +- Fix label in OpenMP parallel\_reduce verify\_initialized [\#834](https://github.com/kokkos/kokkos/issues/834) +- TeamScratch Level 1 on Cuda hangs [\#820](https://github.com/kokkos/kokkos/issues/820) +- \[bug\] memory pool. [\#786](https://github.com/kokkos/kokkos/issues/786) +- Some Reduction Tests fail on Intel 18 with aggressive vectorization on [\#774](https://github.com/kokkos/kokkos/issues/774) +- Error copying dynamic view on copy of memory pool [\#773](https://github.com/kokkos/kokkos/issues/773) +- CUDA stack overflow with TaskDAG test [\#758](https://github.com/kokkos/kokkos/issues/758) +- ThreadVectorRange Customized Reduction Bug [\#739](https://github.com/kokkos/kokkos/issues/739) +- set\_scratch\_size overflows [\#726](https://github.com/kokkos/kokkos/issues/726) +- Get wrong results for compiler checks in Makefile on OS X. [\#706](https://github.com/kokkos/kokkos/issues/706) +- Fix check if multiple host architectures enabled. [\#702](https://github.com/kokkos/kokkos/issues/702) +- Threads Backend Does not Pass on Cray Compilers [\#609](https://github.com/kokkos/kokkos/issues/609) +- Rare bug in memory pool where allocation can finish on superblock in empty state [\#452](https://github.com/kokkos/kokkos/issues/452) +- LDFLAGS in core/unit\_test/Makefile: potential "undefined reference" to pthread lib [\#148](https://github.com/kokkos/kokkos/issues/148) + +## [2.03.00](https://github.com/kokkos/kokkos/tree/2.03.00) (2017-04-25) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.02.15...2.03.00) + +**Implemented enhancements:** + +- UnorderedMap: make it accept Devices or MemorySpaces [\#711](https://github.com/kokkos/kokkos/issues/711) +- sort to accept DynamicView and \[begin,end\) indices [\#691](https://github.com/kokkos/kokkos/issues/691) +- ENABLE Macros should only be used via \#ifdef or \#if defined [\#675](https://github.com/kokkos/kokkos/issues/675) +- Remove impl/Kokkos\_Synchronic\_\* [\#666](https://github.com/kokkos/kokkos/issues/666) +- Turning off IVDEP for Intel 14. [\#638](https://github.com/kokkos/kokkos/issues/638) +- Using an installed Kokkos in a target application using CMake [\#633](https://github.com/kokkos/kokkos/issues/633) +- Create Kokkos Bill of Materials [\#632](https://github.com/kokkos/kokkos/issues/632) +- MDRangePolicy and tagged evaluators [\#547](https://github.com/kokkos/kokkos/issues/547) +- Add PGI support [\#289](https://github.com/kokkos/kokkos/issues/289) + +**Fixed bugs:** + +- Output from PerTeam fails [\#733](https://github.com/kokkos/kokkos/issues/733) +- Cuda: architecture flag not added to link line [\#688](https://github.com/kokkos/kokkos/issues/688) +- Getting large chunks of memory for a thread team in a universal way [\#664](https://github.com/kokkos/kokkos/issues/664) +- Kokkos RNG normal\(\) function hangs for small seed value [\#655](https://github.com/kokkos/kokkos/issues/655) +- Kokkos Tests Errors on Shepard/HSW Builds [\#644](https://github.com/kokkos/kokkos/issues/644) + +## [2.02.15](https://github.com/kokkos/kokkos/tree/2.02.15) (2017-02-10) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.02.07...2.02.15) + +**Implemented enhancements:** + +- Containers: Adding block partitioning to StaticCrsGraph [\#625](https://github.com/kokkos/kokkos/issues/625) +- Kokkos Make System can induce Errors on Cray Volta System [\#610](https://github.com/kokkos/kokkos/issues/610) +- OpenMP: error out if KOKKOS\_HAVE\_OPENMP is defined but not \_OPENMP [\#605](https://github.com/kokkos/kokkos/issues/605) +- CMake: fix standalone build with tests [\#604](https://github.com/kokkos/kokkos/issues/604) +- Change README \(that GitHub shows when opening Kokkos project page\) to tell users how to submit PRs [\#597](https://github.com/kokkos/kokkos/issues/597) +- Add correctness testing for all operators of Atomic View [\#420](https://github.com/kokkos/kokkos/issues/420) +- Allow assignment of Views with compatible memory spaces [\#290](https://github.com/kokkos/kokkos/issues/290) +- Build only one version of Kokkos library for tests [\#213](https://github.com/kokkos/kokkos/issues/213) +- Clean out old KOKKOS\_HAVE\_CXX11 macros clauses [\#156](https://github.com/kokkos/kokkos/issues/156) +- Harmonize Macro names [\#150](https://github.com/kokkos/kokkos/issues/150) + +**Fixed bugs:** + +- Cray and PGI: Kokkos\_Parallel\_Reduce [\#634](https://github.com/kokkos/kokkos/issues/634) +- Kokkos Make System can induce Errors on Cray Volta System [\#610](https://github.com/kokkos/kokkos/issues/610) +- Normal\(\) function random number generator doesn't give the expected distribution [\#592](https://github.com/kokkos/kokkos/issues/592) + +## [2.02.07](https://github.com/kokkos/kokkos/tree/2.02.07) (2016-12-16) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.02.01...2.02.07) + +**Implemented enhancements:** + +- Add CMake option to enable Cuda Lambda support [\#589](https://github.com/kokkos/kokkos/issues/589) +- Add CMake option to enable Cuda RDC support [\#588](https://github.com/kokkos/kokkos/issues/588) +- Add Initial Intel Sky Lake Xeon-HPC Compiler Support to Kokkos Make System [\#584](https://github.com/kokkos/kokkos/issues/584) +- Building Tutorial Examples [\#582](https://github.com/kokkos/kokkos/issues/582) +- Internal way for using ThreadVectorRange without TeamHandle [\#574](https://github.com/kokkos/kokkos/issues/574) +- Testing: Add testing for uvm and rdc [\#571](https://github.com/kokkos/kokkos/issues/571) +- Profiling: Add Memory Tracing and Region Markers [\#557](https://github.com/kokkos/kokkos/issues/557) +- nvcc\_wrapper not installed with Kokkos built with CUDA through CMake [\#543](https://github.com/kokkos/kokkos/issues/543) +- Improve DynRankView debug check [\#541](https://github.com/kokkos/kokkos/issues/541) +- Benchmarks: Add Gather benchmark [\#536](https://github.com/kokkos/kokkos/issues/536) +- Testing: add spot\_check option to test\_all\_sandia [\#535](https://github.com/kokkos/kokkos/issues/535) +- Deprecate Kokkos::Impl::VerifyExecutionCanAccessMemorySpace [\#527](https://github.com/kokkos/kokkos/issues/527) +- Add AtomicAdd support for 64bit float for Pascal [\#522](https://github.com/kokkos/kokkos/issues/522) +- Add Restrict and Aligned memory trait [\#517](https://github.com/kokkos/kokkos/issues/517) +- Kokkos Tests are Not Run using Compiler Optimization [\#501](https://github.com/kokkos/kokkos/issues/501) +- Add support for clang 3.7 w/ openmp backend [\#393](https://github.com/kokkos/kokkos/issues/393) +- Provide an error throw class [\#79](https://github.com/kokkos/kokkos/issues/79) + +**Fixed bugs:** + +- Cuda UVM Allocation test broken with UVM as default space [\#586](https://github.com/kokkos/kokkos/issues/586) +- Bug \(develop branch only\): multiple tests are now failing when forcing uvm usage. [\#570](https://github.com/kokkos/kokkos/issues/570) +- Error in generate\_makefile.sh for Kokkos when Compiler is Empty String/Fails [\#568](https://github.com/kokkos/kokkos/issues/568) +- XL 13.1.4 incorrect C++11 flag [\#553](https://github.com/kokkos/kokkos/issues/553) +- Improve DynRankView debug check [\#541](https://github.com/kokkos/kokkos/issues/541) +- Installing Library on MAC broken due to cp -u [\#539](https://github.com/kokkos/kokkos/issues/539) +- Intel Nightly Testing with Debug enabled fails [\#534](https://github.com/kokkos/kokkos/issues/534) + +## [2.02.01](https://github.com/kokkos/kokkos/tree/2.02.01) (2016-11-01) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.02.00...2.02.01) + +**Implemented enhancements:** + +- Add Changelog generation to our process. [\#506](https://github.com/kokkos/kokkos/issues/506) + +**Fixed bugs:** + +- Test scratch\_request fails in Serial with Debug enabled [\#520](https://github.com/kokkos/kokkos/issues/520) +- Bug In BoundsCheck for DynRankView [\#516](https://github.com/kokkos/kokkos/issues/516) + +## [2.02.00](https://github.com/kokkos/kokkos/tree/2.02.00) (2016-10-30) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.01.10...2.02.00) + +**Implemented enhancements:** + +- Add PowerPC assembly for grabbing clock register in memory pool [\#511](https://github.com/kokkos/kokkos/issues/511) +- Add GCC 6.x support [\#508](https://github.com/kokkos/kokkos/issues/508) +- Test install and build against installed library [\#498](https://github.com/kokkos/kokkos/issues/498) +- Makefile.kokkos adds expt-extended-lambda to cuda build with clang [\#490](https://github.com/kokkos/kokkos/issues/490) +- Add top-level makefile option to just test kokkos-core unit-test [\#485](https://github.com/kokkos/kokkos/issues/485) +- Split and harmonize Object Files of Core UnitTests to increase build parallelism [\#484](https://github.com/kokkos/kokkos/issues/484) +- LayoutLeft to LayoutLeft subview for 3D and 4D views [\#473](https://github.com/kokkos/kokkos/issues/473) +- Add official Cuda 8.0 support [\#468](https://github.com/kokkos/kokkos/issues/468) +- Allow C++1Z Flag for Class Lambda capture [\#465](https://github.com/kokkos/kokkos/issues/465) +- Add Clang 4.0+ compilation of Cuda code [\#455](https://github.com/kokkos/kokkos/issues/455) +- Possible Issue with Intel 17.0.098 and GCC 6.1.0 in Develop Branch [\#445](https://github.com/kokkos/kokkos/issues/445) +- Add name of view to "View bounds error" [\#432](https://github.com/kokkos/kokkos/issues/432) +- Move Sort Binning Operators into Kokkos namespace [\#421](https://github.com/kokkos/kokkos/issues/421) +- TaskPolicy - generate error when attempt to use uninitialized [\#396](https://github.com/kokkos/kokkos/issues/396) +- Import WithoutInitializing and AllowPadding into Kokkos namespace [\#325](https://github.com/kokkos/kokkos/issues/325) +- TeamThreadRange requires begin, end to be the same type [\#305](https://github.com/kokkos/kokkos/issues/305) +- CudaUVMSpace should track \# allocations, due to CUDA limit on \# UVM allocations [\#300](https://github.com/kokkos/kokkos/issues/300) +- Remove old View and its infrastructure [\#259](https://github.com/kokkos/kokkos/issues/259) + +**Fixed bugs:** + +- Bug in TestCuda\_Other.cpp: most likely assembly inserted into Device code [\#515](https://github.com/kokkos/kokkos/issues/515) +- Cuda Compute Capability check of GPU is outdated [\#509](https://github.com/kokkos/kokkos/issues/509) +- multi\_scratch test with hwloc and pthreads seg-faults. [\#504](https://github.com/kokkos/kokkos/issues/504) +- generate\_makefile.bash: "make install" is broken [\#503](https://github.com/kokkos/kokkos/issues/503) +- make clean in Out of Source Build/Tests Does Not Work Correctly [\#502](https://github.com/kokkos/kokkos/issues/502) +- Makefiles for test and examples have issues in Cuda when CXX is not explicitly specified [\#497](https://github.com/kokkos/kokkos/issues/497) +- Dispatch lambda test directly inside GTEST macro doesn't work with nvcc [\#491](https://github.com/kokkos/kokkos/issues/491) +- UnitTests with HWLOC enabled fail if run with mpirun bound to a single core [\#489](https://github.com/kokkos/kokkos/issues/489) +- Failing Reducer Test on Mac with Pthreads [\#479](https://github.com/kokkos/kokkos/issues/479) +- make test Dumps Error with Clang Not Found [\#471](https://github.com/kokkos/kokkos/issues/471) +- OpenMP TeamPolicy member broadcast not using correct volatile shared variable [\#424](https://github.com/kokkos/kokkos/issues/424) +- TaskPolicy - generate error when attempt to use uninitialized [\#396](https://github.com/kokkos/kokkos/issues/396) +- New task policy implementation is pulling in old experimental code. [\#372](https://github.com/kokkos/kokkos/issues/372) +- MemoryPool unit test hangs on Power8 with GCC 6.1.0 [\#298](https://github.com/kokkos/kokkos/issues/298) + +## [2.01.10](https://github.com/kokkos/kokkos/tree/2.01.10) (2016-09-27) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.01.06...2.01.10) + +**Implemented enhancements:** + +- Enable Profiling by default in Tribits build [\#438](https://github.com/kokkos/kokkos/issues/438) +- parallel\_reduce\(0\), parallel\_scan\(0\) unit tests [\#436](https://github.com/kokkos/kokkos/issues/436) +- data\(\)==NULL after realloc with LayoutStride [\#351](https://github.com/kokkos/kokkos/issues/351) +- Fix tutorials to track new Kokkos::View [\#323](https://github.com/kokkos/kokkos/issues/323) +- Rename team policy set\_scratch\_size. [\#195](https://github.com/kokkos/kokkos/issues/195) + +**Fixed bugs:** + +- Possible Issue with Intel 17.0.098 and GCC 6.1.0 in Develop Branch [\#445](https://github.com/kokkos/kokkos/issues/445) +- Makefile spits syntax error [\#435](https://github.com/kokkos/kokkos/issues/435) +- Kokkos::sort fails for view with all the same values [\#422](https://github.com/kokkos/kokkos/issues/422) +- Generic Reducers: can't accept inline constructed reducer [\#404](https://github.com/kokkos/kokkos/issues/404) +- data\\(\\)==NULL after realloc with LayoutStride [\#351](https://github.com/kokkos/kokkos/issues/351) +- const subview of const view with compile time dimensions on Cuda backend [\#310](https://github.com/kokkos/kokkos/issues/310) +- Kokkos \(in Trilinos\) Causes Internal Compiler Error on CUDA 8.0.21-EA on POWER8 [\#307](https://github.com/kokkos/kokkos/issues/307) +- Core Oversubscription Detection Broken? [\#159](https://github.com/kokkos/kokkos/issues/159) + + +## [2.01.06](https://github.com/kokkos/kokkos/tree/2.01.06) (2016-09-02) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.01.00...2.01.06) + +**Implemented enhancements:** + +- Add "standard" reducers for lambda-supportable customized reduce [\#411](https://github.com/kokkos/kokkos/issues/411) +- TaskPolicy - single thread back-end execution [\#390](https://github.com/kokkos/kokkos/issues/390) +- Kokkos master clone tag [\#387](https://github.com/kokkos/kokkos/issues/387) +- Query memory requirements from task policy [\#378](https://github.com/kokkos/kokkos/issues/378) +- Output order of test\_atomic.cpp is confusing [\#373](https://github.com/kokkos/kokkos/issues/373) +- Missing testing for atomics [\#341](https://github.com/kokkos/kokkos/issues/341) +- Feature request for Kokkos to provide Kokkos::atomic\_fetch\_max and atomic\_fetch\_min [\#336](https://github.com/kokkos/kokkos/issues/336) +- TaskPolicy\ performance requires teams mapped to warps [\#218](https://github.com/kokkos/kokkos/issues/218) + +**Fixed bugs:** + +- Reduce with Teams broken for custom initialize [\#407](https://github.com/kokkos/kokkos/issues/407) +- Failing Kokkos build on Debian [\#402](https://github.com/kokkos/kokkos/issues/402) +- Failing Tests on NVIDIA Pascal GPUs [\#398](https://github.com/kokkos/kokkos/issues/398) +- Algorithms: fill\_random assumes dimensions fit in unsigned int [\#389](https://github.com/kokkos/kokkos/issues/389) +- Kokkos::subview with RandomAccess Memory Trait [\#385](https://github.com/kokkos/kokkos/issues/385) +- Build warning \(signed / unsigned comparison\) in Cuda implementation [\#365](https://github.com/kokkos/kokkos/issues/365) +- wrong results for a parallel\_reduce with CUDA8 / Maxwell50 [\#352](https://github.com/kokkos/kokkos/issues/352) +- Hierarchical parallelism - 3 level unit test [\#344](https://github.com/kokkos/kokkos/issues/344) +- Can I allocate a View w/ both WithoutInitializing & AllowPadding? [\#324](https://github.com/kokkos/kokkos/issues/324) +- subview View layout determination [\#309](https://github.com/kokkos/kokkos/issues/309) +- Unit tests with Cuda - Maxwell [\#196](https://github.com/kokkos/kokkos/issues/196) + +## [2.01.00](https://github.com/kokkos/kokkos/tree/2.01.00) (2016-07-21) +[Full Changelog](https://github.com/kokkos/kokkos/compare/End_C++98...2.01.00) + +**Implemented enhancements:** + +- Edit ViewMapping so assigning Views with the same custom layout compiles when const casting [\#327](https://github.com/kokkos/kokkos/issues/327) +- DynRankView: Performance improvement for operator\(\) [\#321](https://github.com/kokkos/kokkos/issues/321) +- Interoperability between static and dynamic rank views [\#295](https://github.com/kokkos/kokkos/issues/295) +- subview member function ? [\#280](https://github.com/kokkos/kokkos/issues/280) +- Inter-operatibility between View and DynRankView. [\#245](https://github.com/kokkos/kokkos/issues/245) +- \(Trilinos\) build warning in atomic\_assign, with Kokkos::complex [\#177](https://github.com/kokkos/kokkos/issues/177) +- View\<\>::shmem\_size should runtime check for number of arguments equal to rank [\#176](https://github.com/kokkos/kokkos/issues/176) +- Custom reduction join via lambda argument [\#99](https://github.com/kokkos/kokkos/issues/99) +- DynRankView with 0 dimensions passed in at construction [\#293](https://github.com/kokkos/kokkos/issues/293) +- Inject view\_alloc and friends into Kokkos namespace [\#292](https://github.com/kokkos/kokkos/issues/292) +- Less restrictive TeamPolicy reduction on Cuda [\#286](https://github.com/kokkos/kokkos/issues/286) +- deep\_copy using remap with source execution space [\#267](https://github.com/kokkos/kokkos/issues/267) +- Suggestion: Enable opt-in L1 caching via nvcc-wrapper [\#261](https://github.com/kokkos/kokkos/issues/261) +- More flexible create\_mirror functions [\#260](https://github.com/kokkos/kokkos/issues/260) +- Rename View::memory\_span to View::required\_allocation\_size [\#256](https://github.com/kokkos/kokkos/issues/256) +- Use of subviews and views with compile-time dimensions [\#237](https://github.com/kokkos/kokkos/issues/237) +- Use of subviews and views with compile-time dimensions [\#237](https://github.com/kokkos/kokkos/issues/237) +- Kokkos::Timer [\#234](https://github.com/kokkos/kokkos/issues/234) +- Fence CudaUVMSpace allocations [\#230](https://github.com/kokkos/kokkos/issues/230) +- View::operator\(\) accept std::is\_integral and std::is\_enum [\#227](https://github.com/kokkos/kokkos/issues/227) +- Allocating zero size View [\#216](https://github.com/kokkos/kokkos/issues/216) +- Thread scalable memory pool [\#212](https://github.com/kokkos/kokkos/issues/212) +- Add a way to disable memory leak output [\#194](https://github.com/kokkos/kokkos/issues/194) +- Kokkos exec space init should init Kokkos profiling [\#192](https://github.com/kokkos/kokkos/issues/192) +- Runtime rank wrapper for View [\#189](https://github.com/kokkos/kokkos/issues/189) +- Profiling Interface [\#158](https://github.com/kokkos/kokkos/issues/158) +- Fix View assignment \(of managed to unmanaged\) [\#153](https://github.com/kokkos/kokkos/issues/153) +- Add unit test for assignment of managed View to unmanaged View [\#152](https://github.com/kokkos/kokkos/issues/152) +- Check for oversubscription of threads with MPI in Kokkos::initialize [\#149](https://github.com/kokkos/kokkos/issues/149) +- Dynamic resizeable 1dimensional view [\#143](https://github.com/kokkos/kokkos/issues/143) +- Develop TaskPolicy for CUDA [\#142](https://github.com/kokkos/kokkos/issues/142) +- New View : Test Compilation Downstream [\#138](https://github.com/kokkos/kokkos/issues/138) +- New View Implementation [\#135](https://github.com/kokkos/kokkos/issues/135) +- Add variant of subview that lets users add traits [\#134](https://github.com/kokkos/kokkos/issues/134) +- NVCC-WRAPPER: Add --host-only flag [\#121](https://github.com/kokkos/kokkos/issues/121) +- Address gtest issue with TriBITS Kokkos build outside of Trilinos [\#117](https://github.com/kokkos/kokkos/issues/117) +- Make tests pass with -expt-extended-lambda on CUDA [\#108](https://github.com/kokkos/kokkos/issues/108) +- Dynamic scheduling for parallel\_for and parallel\_reduce [\#106](https://github.com/kokkos/kokkos/issues/106) +- Runtime or compile time error when reduce functor's join is not properly specified as const member function or with volatile arguments [\#105](https://github.com/kokkos/kokkos/issues/105) +- Error out when the number of threads is modified after kokkos is initialized [\#104](https://github.com/kokkos/kokkos/issues/104) +- Porting to POWER and remove assumption of X86 default [\#103](https://github.com/kokkos/kokkos/issues/103) +- Dynamic scheduling option for RangePolicy [\#100](https://github.com/kokkos/kokkos/issues/100) +- SharedMemory Support for Lambdas [\#81](https://github.com/kokkos/kokkos/issues/81) +- Recommended TeamSize for Lambdas [\#80](https://github.com/kokkos/kokkos/issues/80) +- Add Aggressive Vectorization Compilation mode [\#72](https://github.com/kokkos/kokkos/issues/72) +- Dynamic scheduling team execution policy [\#53](https://github.com/kokkos/kokkos/issues/53) +- UVM allocations in multi-GPU systems [\#50](https://github.com/kokkos/kokkos/issues/50) +- Synchronic in Kokkos::Impl [\#44](https://github.com/kokkos/kokkos/issues/44) +- index and dimension types in for loops [\#28](https://github.com/kokkos/kokkos/issues/28) +- Subview assign of 1D Strided with stride 1 to LayoutLeft/Right [\#1](https://github.com/kokkos/kokkos/issues/1) + +**Fixed bugs:** + +- misspelled variable name in Kokkos\_Atomic\_Fetch + missing unit tests [\#340](https://github.com/kokkos/kokkos/issues/340) +- seg fault Kokkos::Impl::CudaInternal::print\_configuration [\#338](https://github.com/kokkos/kokkos/issues/338) +- Clang compiler error with named parallel\_reduce, tags, and TeamPolicy. [\#335](https://github.com/kokkos/kokkos/issues/335) +- Shared Memory Allocation Error at parallel\_reduce [\#311](https://github.com/kokkos/kokkos/issues/311) +- DynRankView: Fix resize and realloc [\#303](https://github.com/kokkos/kokkos/issues/303) +- Scratch memory and dynamic scheduling [\#279](https://github.com/kokkos/kokkos/issues/279) +- MemoryPool infinite loop when out of memory [\#312](https://github.com/kokkos/kokkos/issues/312) +- Kokkos DynRankView changes break Sacado and Panzer [\#299](https://github.com/kokkos/kokkos/issues/299) +- MemoryPool fails to compile on non-cuda non-x86 [\#297](https://github.com/kokkos/kokkos/issues/297) +- Random Number Generator Fix [\#296](https://github.com/kokkos/kokkos/issues/296) +- View template parameter ordering Bug [\#282](https://github.com/kokkos/kokkos/issues/282) +- Serial task policy broken. [\#281](https://github.com/kokkos/kokkos/issues/281) +- deep\_copy with LayoutStride should not memcpy [\#262](https://github.com/kokkos/kokkos/issues/262) +- DualView::need\_sync should be a const method [\#248](https://github.com/kokkos/kokkos/issues/248) +- Arbitrary-sized atomics on GPUs broken; loop forever [\#238](https://github.com/kokkos/kokkos/issues/238) +- boolean reduction value\_type changes answer [\#225](https://github.com/kokkos/kokkos/issues/225) +- Custom init\(\) function for parallel\_reduce with array value\_type [\#210](https://github.com/kokkos/kokkos/issues/210) +- unit\_test Makefile is Broken - Recursively Calls itself until Machine Apocalypse. [\#202](https://github.com/kokkos/kokkos/issues/202) +- nvcc\_wrapper Does Not Support -Xcompiler \ [\#198](https://github.com/kokkos/kokkos/issues/198) +- Kokkos exec space init should init Kokkos profiling [\#192](https://github.com/kokkos/kokkos/issues/192) +- Kokkos Threads Backend impl\_shared\_alloc Broken on Intel 16.1 \(Shepard Haswell\) [\#186](https://github.com/kokkos/kokkos/issues/186) +- pthread back end hangs if used uninitialized [\#182](https://github.com/kokkos/kokkos/issues/182) +- parallel\_reduce of size 0, not calling init/join [\#175](https://github.com/kokkos/kokkos/issues/175) +- Bug in Threads with OpenMP enabled [\#173](https://github.com/kokkos/kokkos/issues/173) +- KokkosExp\_SharedAlloc, m\_team\_work\_index inaccessible [\#166](https://github.com/kokkos/kokkos/issues/166) +- 128-bit CAS without Assembly Broken? [\#161](https://github.com/kokkos/kokkos/issues/161) +- fatal error: Cuda/Kokkos\_Cuda\_abort.hpp: No such file or directory [\#157](https://github.com/kokkos/kokkos/issues/157) +- Power8: Fix OpenMP backend [\#139](https://github.com/kokkos/kokkos/issues/139) +- Data race in Kokkos OpenMP initialization [\#131](https://github.com/kokkos/kokkos/issues/131) +- parallel\_launch\_local\_memory and cuda 7.5 [\#125](https://github.com/kokkos/kokkos/issues/125) +- Resize can fail with Cuda due to asynchronous dispatch [\#119](https://github.com/kokkos/kokkos/issues/119) +- Qthread taskpolicy initialization bug. [\#92](https://github.com/kokkos/kokkos/issues/92) +- Windows: sys/mman.h [\#89](https://github.com/kokkos/kokkos/issues/89) +- Windows: atomic\_fetch\_sub\(\) [\#88](https://github.com/kokkos/kokkos/issues/88) +- Windows: snprintf [\#87](https://github.com/kokkos/kokkos/issues/87) +- Parallel\_Reduce with TeamPolicy and league size of 0 returns garbage [\#85](https://github.com/kokkos/kokkos/issues/85) +- Throw with Cuda when using \(2D\) team\_policy parallel\_reduce with less than a warp size [\#76](https://github.com/kokkos/kokkos/issues/76) +- Scalar views don't work with Kokkos::Atomic memory trait [\#69](https://github.com/kokkos/kokkos/issues/69) +- Reduce the number of threads per team for Cuda [\#63](https://github.com/kokkos/kokkos/issues/63) +- Named Kernels fail for reductions with CUDA [\#60](https://github.com/kokkos/kokkos/issues/60) +- Kokkos View dimension\_\(\) for long returning unsigned int [\#20](https://github.com/kokkos/kokkos/issues/20) +- atomic test hangs with LLVM [\#6](https://github.com/kokkos/kokkos/issues/6) +- OpenMP Test should set omp\_set\_num\_threads to 1 [\#4](https://github.com/kokkos/kokkos/issues/4) + +**Closed issues:** + +- develop branch broken with CUDA 8 and --expt-extended-lambda [\#354](https://github.com/kokkos/kokkos/issues/354) +- --arch=KNL with Intel 2016 build failure [\#349](https://github.com/kokkos/kokkos/issues/349) +- Error building with Cuda when passing -DKOKKOS\_CUDA\_USE\_LAMBDA to generate\_makefile.bash [\#343](https://github.com/kokkos/kokkos/issues/343) +- Can I safely use int indices in a 2-D View with capacity \> 2B? [\#318](https://github.com/kokkos/kokkos/issues/318) +- Kokkos::ViewAllocateWithoutInitializing is not working [\#317](https://github.com/kokkos/kokkos/issues/317) +- Intel build on Mac OS X [\#277](https://github.com/kokkos/kokkos/issues/277) +- deleted [\#271](https://github.com/kokkos/kokkos/issues/271) +- Broken Mira build [\#268](https://github.com/kokkos/kokkos/issues/268) +- 32-bit build [\#246](https://github.com/kokkos/kokkos/issues/246) +- parallel\_reduce with RDC crashes linker [\#232](https://github.com/kokkos/kokkos/issues/232) +- build of Kokkos\_Sparse\_MV\_impl\_spmv\_Serial.cpp.o fails if you use nvcc and have cuda disabled [\#209](https://github.com/kokkos/kokkos/issues/209) +- Kokkos Serial execution space is not tested with TeamPolicy. [\#207](https://github.com/kokkos/kokkos/issues/207) +- Unit test failure on Hansen KokkosCore\_UnitTest\_Cuda\_MPI\_1 [\#200](https://github.com/kokkos/kokkos/issues/200) +- nvcc compiler warning: calling a \_\_host\_\_ function from a \_\_host\_\_ \_\_device\_\_ function is not allowed [\#180](https://github.com/kokkos/kokkos/issues/180) +- Intel 15 build error with defaulted "move" operators [\#171](https://github.com/kokkos/kokkos/issues/171) +- missing libkokkos.a during Trilinos 12.4.2 build, yet other libkokkos\*.a libs are there [\#165](https://github.com/kokkos/kokkos/issues/165) +- Tie atomic updates to execution space or even to thread team? \(speculation\) [\#144](https://github.com/kokkos/kokkos/issues/144) +- New View: Compiletime/size Test [\#137](https://github.com/kokkos/kokkos/issues/137) +- New View : Performance Test [\#136](https://github.com/kokkos/kokkos/issues/136) +- Signed/unsigned comparison warning in CUDA parallel [\#130](https://github.com/kokkos/kokkos/issues/130) +- Kokkos::complex: Need op\* w/ std::complex & real [\#126](https://github.com/kokkos/kokkos/issues/126) +- Use uintptr\_t for casting pointers [\#110](https://github.com/kokkos/kokkos/issues/110) +- Default thread mapping behavior between P and Q threads. [\#91](https://github.com/kokkos/kokkos/issues/91) +- Windows: Atomic\_Fetch\_Exchange\(\) return type [\#90](https://github.com/kokkos/kokkos/issues/90) +- Synchronic unit test is way too long [\#84](https://github.com/kokkos/kokkos/issues/84) +- nvcc\_wrapper -\> $\(NVCC\_WRAPPER\) [\#42](https://github.com/kokkos/kokkos/issues/42) +- Check compiler version and print helpful message [\#39](https://github.com/kokkos/kokkos/issues/39) +- Kokkos shared memory on Cuda uses a lot of registers [\#31](https://github.com/kokkos/kokkos/issues/31) +- Can not pass unit test `cuda.space` without a GT 720 [\#25](https://github.com/kokkos/kokkos/issues/25) +- Makefile.kokkos lacks bounds checking option that CMake has [\#24](https://github.com/kokkos/kokkos/issues/24) +- Kokkos can not complete unit tests with CUDA UVM enabled [\#23](https://github.com/kokkos/kokkos/issues/23) +- Simplify teams + shared memory histogram example to remove vectorization [\#21](https://github.com/kokkos/kokkos/issues/21) +- Kokkos needs to rever to ${PROJECT\_NAME}\_ENABLE\_CXX11 not Trilinos\_ENABLE\_CXX11 [\#17](https://github.com/kokkos/kokkos/issues/17) +- Kokkos Base Makefile adds AVX to KNC Build [\#16](https://github.com/kokkos/kokkos/issues/16) +- MS Visual Studio 2013 Build Errors [\#9](https://github.com/kokkos/kokkos/issues/9) +- subview\(X, ALL\(\), j\) for 2-D LayoutRight View X: should it view a column? [\#5](https://github.com/kokkos/kokkos/issues/5) + +## [End_C++98](https://github.com/kokkos/kokkos/tree/End_C++98) (2015-04-15) + + +\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* diff --git a/bundled/kokkos-4.5.01/CITATION.cff b/bundled/kokkos-4.5.01/CITATION.cff new file mode 100644 index 0000000000..28c674c451 --- /dev/null +++ b/bundled/kokkos-4.5.01/CITATION.cff @@ -0,0 +1,65 @@ +cff-version: 1.2.0 +title: Kokkos +message: >- + If you use this software, please cite the overview paper +type: software +authors: + - name: The Kokkos authors + website: https://kokkos.org/community/team/ +identifiers: + - type: url + website: https://kokkos.org/kokkos-core-wiki/citation.html +repository-code: 'https://github.com/kokkos/kokkos' +url: 'https://kokkos.org/' +license: Apache-2.0 +preferred-citation: + type: article + authors: + - given-names: Christian R. + family-names: Trott + - given-names: Damien + family-names: Lebrun-Grandié + - given-names: Daniel + family-names: Arndt + - family-names: Ciesko + given-names: Jan + - given-names: Vinh + family-names: Dang + - family-names: Ellingwood + given-names: Nathan + - given-names: Rahulkumar + family-names: Gayatri + - given-names: Evan + family-names: Harvey + - given-names: Daisy S. + family-names: Hollman + - given-names: Dan + family-names: Ibanez + - given-names: Nevin + family-names: Liber + - given-names: Jonathan + family-names: Madsen + - given-names: Jeff + family-names: Miles + - given-names: David + family-names: Poliakoff + - given-names: Amy + family-names: Powell + - given-names: Sivasankaran + family-names: Rajamanickam + - given-names: Mikael + family-names: Simberg + - given-names: Dan + family-names: Sunderland + - given-names: Bruno + family-names: Turcksin + - given-names: Jeremiah + family-names: Wilke + doi: 10.1109/TPDS.2021.3097283 + journal: IEEE Transactions on Parallel and Distributed Systems + start: 805 + end: 817 + title: "Kokkos 3: Programming Model Extensions for the Exascale Era" + volume: 33 + issue: 4 + year: 2022 diff --git a/bundled/kokkos-4.5.01/CMakeLists.txt b/bundled/kokkos-4.5.01/CMakeLists.txt new file mode 100644 index 0000000000..6a70bea149 --- /dev/null +++ b/bundled/kokkos-4.5.01/CMakeLists.txt @@ -0,0 +1,287 @@ +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) + +# Disable in-source builds to prevent source tree corruption. +if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") + message( + FATAL_ERROR + "FATAL: In-source builds are not allowed. You should create a separate directory for build files and delete CMakeCache.txt." + ) +endif() + +# We want to determine if options are given with the wrong case +# In order to detect which arguments are given to compare against +# the list of valid arguments, at the beginning here we need to +# form a list of all the given variables. If it begins with any +# case of KoKkOS, we add it to the list. + +get_cmake_property(_variableNames VARIABLES) +set(KOKKOS_GIVEN_VARIABLES) +foreach(var ${_variableNames}) + string(TOUPPER ${var} UC_VAR) + string(FIND ${UC_VAR} KOKKOS IDX) + if(${IDX} EQUAL 0) + list(APPEND KOKKOS_GIVEN_VARIABLES ${var}) + endif() +endforeach() + +# Basic initialization (Used in KOKKOS_SETTINGS) +set(Kokkos_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(KOKKOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) +set(KOKKOS_PATH ${Kokkos_SOURCE_DIR}) +set(KOKKOS_TOP_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +set(PACKAGE_NAME Kokkos) +set(PACKAGE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + +# Is this build a subdirectory of another project +get_directory_property(HAS_PARENT PARENT_DIRECTORY) + +include(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) +include(${KOKKOS_SRC_PATH}/cmake/kokkos_pick_cxx_std.cmake) + +set(KOKKOS_ENABLED_OPTIONS) #exported in config file +set(KOKKOS_ENABLED_DEVICES) #exported in config file +set(KOKKOS_ENABLED_TPLS) #exported in config file +set(KOKKOS_ENABLED_ARCH_LIST) #exported in config file + +#These are helper flags used for sanity checks during config +#Certain features should depend on other features being configured first +set(KOKKOS_CFG_DAG_NONE On) #sentinel to indicate no dependencies +set(KOKKOS_CFG_DAG_DEVICES_DONE Off) +set(KOKKOS_CFG_DAG_OPTIONS_DONE Off) +set(KOKKOS_CFG_DAG_ARCH_DONE Off) +set(KOKKOS_CFG_DAG_CXX_STD_DONE Off) +set(KOKKOS_CFG_DAG_COMPILER_ID_DONE Off) +function(KOKKOS_CFG_DEPENDS SUCCESSOR PRECURSOR) + set(PRE_FLAG KOKKOS_CFG_DAG_${PRECURSOR}) + set(POST_FLAG KOKKOS_CFG_DAG_${SUCCESSOR}) + if(NOT ${PRE_FLAG}) + message( + FATAL_ERROR "Bad CMake refactor: feature ${SUCCESSOR} cannot be configured until ${PRECURSOR} is configured" + ) + endif() + global_set(${POST_FLAG} On) +endfunction() + +list(APPEND CMAKE_MODULE_PATH cmake/Modules) + +set(CMAKE_DISABLE_SOURCE_CHANGES ON) +set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) + +# What language are we compiling Kokkos as +# downstream dependencies need to match this! +set(KOKKOS_COMPILE_LANGUAGE CXX) +# use lower case here since we didn't parse options yet +if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE AND Kokkos_ENABLE_CUDA) + + # Without this as a language for the package we would get a C++ compiler enabled. + # but we still need a C++ compiler even if we build all our cpp files as CUDA only + # because otherwise the C++ features don't work etc. + # This is just the rather odd way CMake does this, since CUDA doesn't imply C++ even + # though it is a C++ extension ... (but I guess it didn't use to be back in CUDA 4 or 5 + # days. + set(KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE CXX) + + set(KOKKOS_COMPILE_LANGUAGE CUDA) +endif() +# use lower case here since we haven't parsed options yet +if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE AND Kokkos_ENABLE_HIP) + + # Without this as a language for the package we would get a C++ compiler enabled. + # but we still need a C++ compiler even if we build all our cpp files as HIP only + # because otherwise the C++ features don't work etc. + set(KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE CXX) + + set(KOKKOS_COMPILE_LANGUAGE HIP) +endif() + +if(Spack_WORKAROUND) + if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) + message(FATAL_ERROR "Can't currently use Kokkos_ENABLE_COMPILER_AS_CMAKE_LANGUAGE in a spack installation!") + endif() + + #if we are explicitly using Spack for development, + #nuke the Spack compiler + set(SPACK_CXX $ENV{SPACK_CXX}) + if(SPACK_CXX) + set(CMAKE_CXX_COMPILER ${SPACK_CXX} CACHE STRING "the C++ compiler" FORCE) + set(ENV{CXX} ${SPACK_CXX}) + endif() +endif() +# Always call the project command to define Kokkos_ variables +# and to make sure that C++ is an enabled language +project(Kokkos ${KOKKOS_COMPILE_LANGUAGE} ${KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE}) +if(NOT HAS_PARENT) + if(NOT CMAKE_BUILD_TYPE) + set(DEFAULT_BUILD_TYPE "RelWithDebInfo") + message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' as none was specified.") + set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" + CACHE STRING "Choose the type of build, options are: Debug, Release, RelWithDebInfo and MinSizeRel." FORCE + ) + endif() +endif() + +if(NOT CMAKE_SIZEOF_VOID_P) + string(FIND ${CMAKE_CXX_COMPILER} nvcc_wrapper FIND_IDX) + if(NOT FIND_IDX STREQUAL -1) + message( + FATAL_ERROR + "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is CUDA linkage using nvcc_wrapper. Please ensure your CUDA environment is correctly configured." + ) + else() + message( + FATAL_ERROR + "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is linkage errors during CMake compiler validation. Please consult the CMake error log shown below for the exact error during compiler validation" + ) + endif() +elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + message(WARNING "32-bit builds are experimental and not officially supported.") + set(KOKKOS_IMPL_32BIT ON) + else() + message( + FATAL_ERROR + "Kokkos assumes a 64-bit build, i.e., 8-byte pointers, but found ${CMAKE_SIZEOF_VOID_P}-byte pointers instead;" + ) + endif() +endif() + +set(Kokkos_VERSION_MAJOR 4) +set(Kokkos_VERSION_MINOR 5) +set(Kokkos_VERSION_PATCH 1) +set(Kokkos_VERSION "${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR}.${Kokkos_VERSION_PATCH}") +message(STATUS "Kokkos version: ${Kokkos_VERSION}") +math(EXPR KOKKOS_VERSION "${Kokkos_VERSION_MAJOR} * 10000 + ${Kokkos_VERSION_MINOR} * 100 + ${Kokkos_VERSION_PATCH}") +# mathematical expressions below are not stricly necessary but they eliminate +# the rather aggravating leading 0 in the releases patch version number, and, +# in some way, are a sanity check for our arithmetic +math(EXPR KOKKOS_VERSION_MAJOR "${KOKKOS_VERSION} / 10000") +math(EXPR KOKKOS_VERSION_MINOR "${KOKKOS_VERSION} / 100 % 100") +math(EXPR KOKKOS_VERSION_PATCH "${KOKKOS_VERSION} % 100") + +# Load either the real TriBITS or a TriBITS wrapper +# for certain utility functions that are universal (like GLOBAL_SET) +include(${KOKKOS_SRC_PATH}/cmake/fake_tribits.cmake) + +if(Kokkos_ENABLE_CUDA) + # If we are building CUDA, we have tricked CMake because we declare a CXX project + # If the default C++ standard for a given compiler matches the requested + # standard, then CMake just omits the -std flag in later versions of CMake + # This breaks CUDA compilation (CUDA compiler can have a different default + # -std then the underlying host compiler by itself). Setting this variable + # forces CMake to always add the -std flag even if it thinks it doesn't need it + global_set(CMAKE_CXX_STANDARD_DEFAULT 98) +endif() + +# These are the variables we will append to as we go +# I really wish these were regular variables +# but scoping issues can make it difficult +global_set(KOKKOS_COMPILE_OPTIONS) +global_set(KOKKOS_LINK_OPTIONS) +global_set(KOKKOS_AMDGPU_OPTIONS) +global_set(KOKKOS_CUDA_OPTIONS) +global_set(KOKKOS_CUDAFE_OPTIONS) +global_set(KOKKOS_XCOMPILER_OPTIONS) +# We need to append text here for making sure TPLs +# we import are available for an installed Kokkos +global_set(KOKKOS_TPL_EXPORTS) +# KOKKOS_DEPENDENCE is used by kokkos_launch_compiler +global_set(KOKKOS_COMPILE_DEFINITIONS KOKKOS_DEPENDENCE) +# MSVC never goes through kokkos_launch_compiler +if(NOT MSVC) + global_append(KOKKOS_LINK_OPTIONS -DKOKKOS_DEPENDENCE) +endif() + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkos_configure_trilinos.cmake) + +if(Kokkos_ENABLE_TESTS) + find_package(GTest QUIET) +endif() + +# Include a set of Kokkos-specific wrapper functions that +# will either call raw CMake or TriBITS +# These are functions like KOKKOS_INCLUDE_DIRECTORIES +include(${KOKKOS_SRC_PATH}/cmake/kokkos_tribits.cmake) + +# Check the environment and set certain variables +# to allow platform-specific checks +include(${KOKKOS_SRC_PATH}/cmake/kokkos_check_env.cmake) + +include(${KOKKOS_SRC_PATH}/cmake/build_env_info.cmake) +check_git_setup() + +# The build environment setup goes in the following steps +# 1) Check all the enable options. This includes checking Kokkos_DEVICES +# 2) Check the compiler ID (type and version) +# 3) Check the CXX standard and select important CXX flags +# 4) Check for any third-party libraries (TPLs) like hwloc +# 5) Check if optimizing for a particular architecture and add arch-specific flags +kokkos_setup_build_environment() + +# Finish off the build +# 6) Recurse into subdirectories and configure individual libraries +# 7) Export and install targets + +option(BUILD_SHARED_LIBS "Build shared libraries" OFF) + +set(KOKKOS_COMPONENT_LIBRARIES kokkoscore kokkoscontainers kokkosalgorithms kokkossimd) +set_property(GLOBAL PROPERTY KOKKOS_INT_LIBRARIES kokkos ${KOKKOS_COMPONENT_LIBRARIES}) + +if(HAS_PARENT) + set(KOKKOS_HEADER_DIR "include/kokkos") + set(KOKKOS_IS_SUBDIRECTORY TRUE) +else() + set(KOKKOS_HEADER_DIR "${CMAKE_INSTALL_INCLUDEDIR}") + set(KOKKOS_IS_SUBDIRECTORY FALSE) +endif() + +#------------------------------------------------------------------------------ +# +# A) Forward declare the package so that certain options are also defined for +# subpackages + +#------------------------------------------------------------------------------ +# +# D) Process the subpackages (subdirectories) for Kokkos +# +kokkos_process_subpackages() + +#------------------------------------------------------------------------------ +# +# E) If Kokkos itself is enabled, process the Kokkos package +# + +kokkos_configure_core() + +if(NOT Kokkos_INSTALL_TESTING) + add_library(kokkos INTERFACE) + #Make sure in-tree projects can reference this as Kokkos:: + #to match the installed target names + add_library(Kokkos::kokkos ALIAS kokkos) + # all_libs target is required for TriBITS-compliance + add_library(Kokkos::all_libs ALIAS kokkos) + target_link_libraries(kokkos INTERFACE ${KOKKOS_COMPONENT_LIBRARIES}) + kokkos_internal_add_library_install(kokkos) +endif() +include(${KOKKOS_SRC_PATH}/cmake/kokkos_install.cmake) + +# nvcc_wrapper is Kokkos' wrapper for NVIDIA's NVCC CUDA compiler. +# Kokkos needs nvcc_wrapper in order to build. Other libraries and +# executables also need nvcc_wrapper. Thus, we need to install it. +# If the argument of DESTINATION is a relative path, CMake computes it +# as relative to ${CMAKE_INSTALL_PATH}. +# KOKKOS_INSTALL_ADDITIONAL_FILES will install nvcc wrapper and other generated +# files +kokkos_install_additional_files() + +# Finally - if we are a subproject - make sure the enabled devices are visible +if(HAS_PARENT) + foreach(DEV Kokkos_ENABLED_DEVICES) + #I would much rather not make these cache variables or global properties, but I can't + #make any guarantees on whether PARENT_SCOPE is good enough to make + #these variables visible where I need them + set(Kokkos_ENABLE_${DEV} ON PARENT_SCOPE) + set_property(GLOBAL PROPERTY Kokkos_ENABLE_${DEV} ON) + endforeach() +endif() diff --git a/bundled/kokkos-4.5.01/CONTRIBUTING.md b/bundled/kokkos-4.5.01/CONTRIBUTING.md new file mode 100644 index 0000000000..e97f8c4d89 --- /dev/null +++ b/bundled/kokkos-4.5.01/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing to Kokkos + +## Pull Requests +We actively welcome pull requests. +1. Fork the repo and create your branch from `develop`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. + +Before sending your patch for review, please try to ensure that it is formatted properly. We use clang-format version 16 for this. + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. + +## License +By contributing to Kokkos, you agree that your contributions will be licensed under the LICENSE file in the root directory of this source tree. diff --git a/bundled/kokkos-4.5.01/Copyright.txt b/bundled/kokkos-4.5.01/Copyright.txt new file mode 100644 index 0000000000..cbba3efc7b --- /dev/null +++ b/bundled/kokkos-4.5.01/Copyright.txt @@ -0,0 +1,8 @@ +************************************************************************ + + Kokkos v. 4.0 + Copyright (2022) National Technology & Engineering + Solutions of Sandia, LLC (NTESS). + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. diff --git a/bundled/kokkos-4.5.01/LICENSE b/bundled/kokkos-4.5.01/LICENSE new file mode 100644 index 0000000000..4d9d69d7c4 --- /dev/null +++ b/bundled/kokkos-4.5.01/LICENSE @@ -0,0 +1,228 @@ + ============================================================================== + Kokkos is under the Apache License v2.0 with LLVM Exceptions: + ============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS Apache 2.0 + + ---- LLVM Exceptions to the Apache 2.0 License ---- + + As an exception, if, as a result of your compiling your source code, portions + of this Software are embedded into an Object form of such source code, you + may redistribute such embedded portions in such Object form without complying + with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + + In addition, if you combine or link compiled forms of this Software with + software that is licensed under the GPLv2 ("Combined Software") and if a + court of competent jurisdiction determines that the patent provision (Section + 3), the indemnity provision (Section 9) or other Section of the License + conflicts with the conditions of the GPLv2, you may retroactively and + prospectively choose to deem waived or otherwise exclude such Section(s) of + the License, but only in their entirety and only with respect to the Combined + Software. + + ============================================================================== + Software from third parties included in Kokkos: + ============================================================================== + + Kokkos contains third party software which is under different license + terms. All such code will be identified clearly using at least one of two + mechanisms: + 1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or + 2) It will contain specific license and restriction terms at the top of every + file. + + + THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Questions? Contact: + Christian R. Trott (crtrott@sandia.gov) and + Damien T. Lebrun-Grandie (lebrungrandt@ornl.gov) + + ************************************************************************ diff --git a/bundled/kokkos-4.5.01/README.md b/bundled/kokkos-4.5.01/README.md new file mode 100644 index 0000000000..56159b35c2 --- /dev/null +++ b/bundled/kokkos-4.5.01/README.md @@ -0,0 +1,80 @@ +[![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4)](https://kokkos.org) + +# Kokkos: Core Libraries + +Kokkos Core implements a programming model in C++ for writing performance portable +applications targeting all major HPC platforms. For that purpose it provides +abstractions for both parallel execution of code and data management. +Kokkos is designed to target complex node architectures with N-level memory +hierarchies and multiple types of execution resources. It currently can use +CUDA, HIP, SYCL, HPX, OpenMP and C++ threads as backend programming models with several other +backends in development. + +**Kokkos Core is part of the [Kokkos C++ Performance Portability Programming Ecosystem](https://kokkos.org/about/abstract/).** + +Kokkos is a [Linux Foundation](https://linuxfoundation.org) project. + +## Learning about Kokkos + +To start learning about Kokkos: + +- [Kokkos Lectures](https://kokkos.org/kokkos-core-wiki/videolectures.html): they contain a mix of lecture videos and hands-on exercises covering all the important capabilities. + +- [Programming guide](https://kokkos.org/kokkos-core-wiki/programmingguide.html): contains in "narrative" form a technical description of the programming model, machine model, and the main building blocks like the Views and parallel dispatch. + +- [API reference](https://kokkos.org/kokkos-core-wiki/): organized by category, i.e., [core](https://kokkos.org/kokkos-core-wiki/API/core-index.html), [algorithms](https://kokkos.org/kokkos-core-wiki/API/algorithms-index.html) and [containers](https://kokkos.org/kokkos-core-wiki/API/containers-index.html) or, if you prefer, in [alphabetical order](https://kokkos.org/kokkos-core-wiki/API/alphabetical.html). + +- [Use cases and Examples](https://kokkos.org/kokkos-core-wiki/usecases.html): a serie of examples ranging from how to use Kokkos with MPI to Fortran interoperability. + +## Obtaining Kokkos + +The latest release of Kokkos can be obtained from the [GitHub releases page](https://github.com/kokkos/kokkos/releases/latest). + +The current release is [4.5.01](https://github.com/kokkos/kokkos/releases/tag/4.5.01). + +```bash +curl -OJ -L https://github.com/kokkos/kokkos/releases/download/4.5.01/kokkos-4.5.01.tar.gz +# Or with wget +wget https://github.com/kokkos/kokkos/releases/download/4.5.01/kokkos-4.5.01.tar.gz +``` + +To clone the latest development version of Kokkos from GitHub: + +```bash +git clone -b develop https://github.com/kokkos/kokkos.git +``` + +### Building Kokkos + +To build Kokkos, you will need to have a C++ compiler that supports C++17 or later. +All requirements including minimum and primary tested compiler versions can be found [here](https://kokkos.org/kokkos-core-wiki/requirements.html). + +Building and installation instructions are described [here](https://kokkos.org/kokkos-core-wiki/building.html). + +You can also install Kokkos using [Spack](https://spack.io/): `spack install kokkos`. [Available configuration options](https://packages.spack.io/package.html?name=kokkos) can be displayed using `spack info kokkos`. + +## For the complete documentation: [kokkos.org/kokkos-core-wiki/](https://kokkos.org/kokkos-core-wiki/) + +## Support + +For questions find us on Slack: https://kokkosteam.slack.com or open a GitHub issue. + +For non-public questions send an email to: *crtrott(at)sandia.gov* + +## Contributing + +Please see [this page](https://kokkos.org/kokkos-core-wiki/contributing.html) for details on how to contribute. + +## Citing Kokkos + +Please see the [following page](https://kokkos.org/kokkos-core-wiki/citation.html). + +## License + +[![License](https://img.shields.io/badge/License-Apache--2.0_WITH_LLVM--exception-blue)](https://spdx.org/licenses/LLVM-exception.html) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. + +The full license statement used in all headers is available [here](https://kokkos.org/kokkos-core-wiki/license.html) or +[here](https://github.com/kokkos/kokkos/blob/develop/LICENSE). diff --git a/bundled/kokkos-4.5.01/SECURITY.md b/bundled/kokkos-4.5.01/SECURITY.md new file mode 100644 index 0000000000..93cf6e3663 --- /dev/null +++ b/bundled/kokkos-4.5.01/SECURITY.md @@ -0,0 +1,12 @@ +# Reporting Security Issues + +To report a security issue, please email +[lebrungrandt@ornl.gov](mailto:lebrungrandt@ornl.gov) +and [crtrott@sandia.gov](mailto:crtrott@sandia.gov) +with a description of the issue, the steps you took to create the issue, +affected versions, and, if known, mitigations for the issue. + +Our vulnerability management team will respond within 5 working days of your +email. If the issue is confirmed as a vulnerability, we will open a +Security Advisory and acknowledge your contributions as part of it. This project +follows a 90 day disclosure timeline. diff --git a/bundled/kokkos-4.5.01/Spack.md b/bundled/kokkos-4.5.01/Spack.md new file mode 100644 index 0000000000..06c763a64e --- /dev/null +++ b/bundled/kokkos-4.5.01/Spack.md @@ -0,0 +1,268 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Kokkos Spack + +This gives instructions for using Spack to install Kokkos and developing packages that depend on Kokkos. + +## Getting Started + +Make sure you have downloaded [Spack](https://github.com/spack/spack). +The easiest way to configure the Spack environment is: +````bash +> source spack/share/spack/setup-env.sh +```` +with other scripts available for other shells. +You can display information about how to install packages with: +````bash +> spack info kokkos +```` +This will print all the information about how to install Kokkos with Spack. +For detailed instructions on how to use Spack, see the [User Manual](https://spack.readthedocs.io). + +## Setting Up Spack: Avoiding the Package Cascade +By default, Spack doesn't 'see' anything on your system - including things like CMake and CUDA. +This can be limited by adding a `packages.yaml` to your `$HOME/.spack` folder that includes CMake (and CUDA, if applicable). For example, your `packages.yaml` file could be: +````yaml +packages: + cuda: + buildable: false + externals: + - prefix: /opt/local/ppc64le-pwr8-nvidia/cuda/10.1.243 + spec: cuda@10.1.243 + - modules: + - cuda/10.1.243 + spec: cuda@10.1.243 + cmake: + buildable: false + externals: + - prefix: /opt/local/ppc64le/cmake/3.16.8 + spec: cmake@3.16.8 + - modules: + - cmake/3.16.8 + spec: cmake@3.16.8 +```` +The `modules` entry is only necessary on systems that require loading Modules (i.e. most DOE systems). +The `buildable` flag is useful to make sure Spack crashes if there is a path error, +rather than having a type-o and Spack rebuilding everything because `cmake` isn't found. +You can verify your environment is set up correctly by running `spack graph` or `spack spec`. +For example: +````bash +> spack graph kokkos +cuda +o kokkos +|\ +o | cuda + / +o cmake +```` +Without the existing CUDA and CMake being identified in `packages.yaml`, a (subset!) of the output would be: +````bash +o kokkos +|\ +| o cmake +| |\ +| | | |\ +| | | | | |\ +| | | | | | | |\ +| | | | | | | | | |\ +| | | | | | | o | | | libarchive +| | | | | | | |\ \ \ \ +| | | | | | | | | |\ \ \ \ +| | | | | | | | | | | | |_|/ +| | | | | | | | | | | |/| | +| | | | | | | | | | | | | o curl +| | |_|_|_|_|_|_|_|_|_|_|/| +| |/| | | |_|_|_|_|_|_|_|/ +| | | | |/| | | | | | | | +| | | | o | | | | | | | | openssl +| |/| | | | | | | | | | | +| | | | | | | | | | o | | libxml2 +| | |_|_|_|_|_|_|_|/| | | +| | | | | | | | | | |\ \ \ +| o | | | | | | | | | | | | zlib +| / / / / / / / / / / / / +| o | | | | | | | | | | | xz +| / / / / / / / / / / / +| o | | | | | | | | | | rhash +| / / / / / / / / / / +| | | | o | | | | | | nettle +| | | | |\ \ \ \ \ \ \ +| | | o | | | | | | | | libuv +| | | | o | | | | | | | autoconf +| | |_|/| | | | | | | | +| | | | |/ / / / / / / +| o | | | | | | | | | perl +| o | | | | | | | | | gdbm +| o | | | | | | | | | readline +```` + +## Configuring Kokkos as a Project Dependency +Say you have a project "SuperScience" which needs to use Kokkos. +In your `package.py` file, you would generally include something like: +````python +class SuperScience(CMakePackage): + ... + depends_on("kokkos") +```` +Often projects want to tweak behavior when using certain features, e.g. +````python + depends_on("kokkos+cuda", when="+cuda") +```` +if your project needs CUDA-specific logic to configure and build. +This illustrates the general principle in Spack of "flowing-up". +A user requests a feature in the final app: +````bash +> spack install superscience+cuda +```` +This flows upstream to the Kokkos dependency, causing the `kokkos+cuda` variant to build. +The downstream app (SuperScience) tells the upstream app (Kokkos) how to build. + +Because Kokkos is a performance portability library, it somewhat inverts this principle. +Kokkos "flows-down", telling your application how best to configure for performance. +Rather than a downstream app (SuperScience) telling the upstream (Kokkos) what variants to build, +a pre-built Kokkos should be telling the downstream app SuperScience what variants to use. +Kokkos works best when there is an "expert" configuration installed on your system. +Your build should simply request `-DKokkos_ROOT=` and configure appropriately based on the Kokkos it finds. + +Kokkos has many, many build variants. +Where possible, projects should only depend on a general Kokkos, not specific variants. +We recommend instead adding for each system you build on a Kokkos configuration to your `packages.yaml` file (usually found in `~/.spack` for specific users). +For a Xeon + Volta system, this could look like: +````yaml + kokkos: + variants: +cuda +openmp +cuda_lambda +wrapper ^cuda@10.1 cuda_arch=70 + compiler: [gcc@7.2.0] +```` +which gives the "best" Kokkos configuration as CUDA+OpenMP optimized for a Volta 70 architecture using CUDA 10.1. +It also enables support for CUDA Lambdas. +The `+wrapper` option tells Kokkos to build with the special `nvcc_wrapper` (more below). +Note here that we use the built-in `cuda_arch` variant of Spack to specify the archicture. +For a Haswell system, we use +````yaml + kokkos: + variants: +openmp std=14 target=haswell + compiler: [intel@18] +```` +which uses the built-in microarchitecture variants of Spack. +Consult the Spack documentation for more details of Spack microarchitectures +and CUDA architectures. +Spack does not currently provide an AMD GPU microarchitecture option. +If building for HIP or an AMD GPU, Kokkos provides an `amd_gpu_arch` similar to `cuda_arch`. +````yaml + kokkos: + variants: +hip amd_gpu_arch=vega900 +```` + +Without an optimal default in your `packages.yaml` file, it is highly likely that the default Kokkos configuration you get will not be what you want. +For example, CUDA is not enabled by default (there is no easy logic to conditionally activate this for CUDA-enabled systems). +If you don't specify a CUDA build variant in a `packages.yaml` and you build your Kokkos-dependent project: +````bash +> spack install superscience +```` +you may end up just getting the default Kokkos (i.e. Serial). +Before running `spack install ` we recommend running `spack spec ` to confirm your dependency tree is correct. +For example, with Kokkos Kernels: +````bash +kokkos-kernels@3.0%gcc@8.3.0~blas build_type=RelWithDebInfo ~cblas~complex_double~complex_float~cublas~cuda cuda_arch=none ~cusparse~diy+double execspace_cuda=auto execspace_openmp=auto execspace_serial=auto execspace_threads=auto ~float~lapack~lapacke+layoutleft~layoutright memspace_cudaspace=auto memspace_cudauvmspace=auto +memspace_hostspace~mkl+offset_int+offset_size_t~openmp+ordinal_int~ordinal_int64_t~serial~superlu arch=linux-rhel7-skylake_avx512 + ^cmake@3.16.2%gcc@8.3.0~doc+ncurses+openssl+ownlibs~qt arch=linux-rhel7-skylake_avx512 + ^kokkos@3.0%gcc@8.3.0~aggressive_vectorization~amdavx~armv80~armv81~armv8_thunderx~armv8_tx2~bdw~bgq build_type=RelWithDebInfo ~carrizo~compiler_warnings+cuda cuda_arch=none +cuda_lambda~cuda_ldg_intrinsic~cuda_relocatable_device_code~cuda_uvm~debug~debug_bounds_check~debug_dualview_modify_check~deprecated_code~diy~epyc~examples~explicit_instantiation~fiji~gfx901~hpx~hpx_async_dispatch~hsw~hwloc~kaveri~kepler30~kepler32~kepler35~kepler37~knc~knl~maxwell50~maxwell52~maxwell53~memkind~numactl+openmp~pascal60~pascal61~power7~power8~power9+profiling~profiling_load_print~pthread~qthread~rocm~ryzen~serial~skx~snb std=14 ~tests~turing75~vega+volta70~volta72+wrapper~wsm arch=linux-rhel7-skylake_avx512 + ^cuda@10.1%gcc@8.3.0 arch=linux-rhel7-skylake_avx512 + ^kokkos-nvcc-wrapper@old%gcc@8.3.0 build_type=RelWithDebInfo +mpi arch=linux-rhel7-skylake_avx512 + ^openmpi@4.0.2%gcc@8.3.0~cuda+cxx_exceptions fabrics=none ~java~legacylaunchers~memchecker patches=073477a76bba780c67c36e959cd3ee6910743e2735c7e76850ffba6791d498e4 ~pmi schedulers=none ~sqlite3~thread_multiple+vt arch=linux-rhel7-skylake_avx512 +```` +The output can be very verbose, but we can verify the expected `kokkos`: +````bash +kokkos@3.0%gcc@8.3.0~aggressive_vectorization~amdavx~armv80~armv81~armv8_thunderx~armv8_tx2~bdw~bgq build_type=RelWithDebInfo ~carrizo~compiler_warnings+cuda cuda_arch=none +cuda_lambda~cuda_ldg_intrinsic~cuda_relocatable_device_code~cuda_uvm~debug~debug_bounds_check~debug_dualview_modify_check~deprecated_code~diy~epyc~examples~explicit_instantiation~fiji~gfx901~hpx~hpx_async_dispatch~hsw~hwloc~kaveri~kepler30~kepler32~kepler35~kepler37~knc~knl~maxwell50~maxwell52~maxwell53~memkind~numactl+openmp~pascal60~pascal61~power7~power8~power9+profiling~profiling_load_print~pthread~qthread~rocm~ryzen~serial~skx~snb std=11 ~tests~turing75~vega+volta70~volta72+wrapper~wsm arch=linux-rhel7-skylake_avx512 +```` +We see that we do have `+volta70` and `+wrapper`, e.g. + +### Spack Environments +The encouraged way to use Spack is with Spack environments ([more details here](https://spack-tutorial.readthedocs.io/en/latest/tutorial_environments.html#dealing-with-many-specs-at-once)). +Rather than installing packages one-at-a-time, you add packages to an environment. +After adding all packages, you concretize and install them all. +Using environments, one can explicitly add a desired Kokkos for the environment, e.g. +````bash +> spack add kokkos +cuda +cuda_lambda +volta70 +> spack add my_project +my_variant +> ... +> spack install +```` +All packages within the environment will build against the CUDA-enabled Kokkos, +even if they only request a default Kokkos. + +## NVCC Wrapper +Kokkos is a C++ project, but often builds for the CUDA backend. +This is particularly problematic with CMake. At this point, `nvcc` does not accept all the flags that normally get passed to a C++ compiler. +Kokkos provides `nvcc_wrapper` that identifies correctly as a C++ compiler to CMake and accepts C++ flags, but uses `nvcc` as the underlying compiler. +`nvcc` itself also uses an underlying host compiler, e.g. GCC. + +In Spack, the underlying host compiler is specified as below, e.g.: +````bash +> spack install package %gcc@8.0.0 +```` +This is still valid for Kokkos. To use the special wrapper for CUDA builds, request a desired compiler and simply add the `+wrapper` variant. +````bash +> spack install kokkos +cuda +wrapper %gcc@7.2.0 +```` +Downstream projects depending on Kokkos need to override their compiler. +Kokkos provides the compiler in a `kokkos_cxx` variable, +which points to either `nvcc_wrapper` when needed or the regular compiler otherwise. +Spack projects already do this to use MPI compiler wrappers. +````python +def cmake_args(self): + options = [] + ... + options.append("-DCMAKE_CXX_COMPILER=%s" % self.spec["kokkos"].kokkos_cxx) + ... + return options +```` +Note: `nvcc_wrapper` works with the MPI compiler wrappers. +If building your project with MPI, do NOT set your compiler to `nvcc_wrapper`. +Instead set your compiler to `mpicxx` and `nvcc_wrapper` will be used under the hood. +````python +def cmake_args(self): + options = [] + ... + options.append("-DCMAKE_CXX_COMPILER=%s" % self.spec["mpi"].mpicxx) + ... + return options +```` +To accomplish this, `nvcc_wrapper` must depend on MPI (even though it uses no MPI). +This has the unfortunate consequence that Kokkos CUDA projects not using MPI will implicitly depend on MPI anyway. +This behavior is necessary for now, but will hopefully be removed later. +When using environments, if MPI is not needed, you can remove the MPI dependency with: +````bash +> spack add kokkos-nvcc-wrapper ~mpi +```` + +## Developing With Spack + +Spack has historically been much more suited to *deployment* of mature packages than active testing or developing. +However, recent features have improved support for development. +Future releases are likely to make this even easier and incorporate Git integration. +The most common commands will do a full build and install of the packages. +If doing development, you may wish to merely set up a build environment. +This allows you to modify the source and re-build. +In this case, you can stop after configuring. +Suppose you have Kokkos checkout in the folder `kokkos-src`: +````bash +> spack dev-build -d kokkos-src -u cmake kokkos@develop +wrapper +openmp +```` +This sets up a development environment for you in `kokkos-src` which you can use (Bash example shown): +Note: Always specify `develop` as the version when doing `dev-build`, except in rare cases. +You are usually developing a feature branch that will merge into `develop`, +hence you are making a new `develop` branch. + +````bash +> cd kokko-src +> source spack-build-env.txt +> cd spack-build +> make +```` +Before sourcing the Spack development environment, you may wish to save your current environment: +````bash +> declare -px > myenv.sh +```` +When done with Spack, you can then restore your original environment: +````bash +> source myenv.sh +```` diff --git a/bundled/kokkos-4.5.01/algorithms/CMakeLists.txt b/bundled/kokkos-4.5.01/algorithms/CMakeLists.txt new file mode 100644 index 0000000000..73ce9f7ec5 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/CMakeLists.txt @@ -0,0 +1,7 @@ +if(NOT Kokkos_INSTALL_TESTING) + add_subdirectory(src) +endif() +# FIXME_OPENACC: temporarily disabled due to unimplemented features +if(NOT ((KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) OR KOKKOS_ENABLE_OPENACC)) + kokkos_add_test_directories(unit_tests) +endif() diff --git a/bundled/kokkos-4.5.01/algorithms/src/CMakeLists.txt b/bundled/kokkos-4.5.01/algorithms/src/CMakeLists.txt new file mode 100644 index 0000000000..9f10b85e02 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/CMakeLists.txt @@ -0,0 +1,29 @@ +#I have to leave these here for tribits +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +#----------------------------------------------------------------------------- + +file(GLOB ALGO_HEADERS *.hpp) +file(GLOB ALGO_SOURCES *.cpp) +append_glob(ALGO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/std_algorithms/*.hpp) +append_glob(ALGO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/std_algorithms/impl/*.hpp) + +install( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" + DESTINATION ${KOKKOS_HEADER_DIR} + FILES_MATCHING + PATTERN "*.hpp" +) + +#----------------------------------------------------------------------------- + +# We have to pass the sources in here for Tribits +# These will get ignored for standalone CMake and a true interface library made +kokkos_add_interface_library(kokkosalgorithms NOINSTALLHEADERS ${ALGO_HEADERS} SOURCES ${ALGO_SOURCES}) +kokkos_lib_include_directories( + kokkosalgorithms ${KOKKOS_TOP_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} +) + +kokkos_link_tpl(kokkoscontainers PUBLIC ROCTHRUST) +kokkos_link_tpl(kokkoscore PUBLIC ONEDPL) diff --git a/bundled/kokkos-4.5.01/algorithms/src/KokkosAlgorithms_dummy.cpp b/bundled/kokkos-4.5.01/algorithms/src/KokkosAlgorithms_dummy.cpp new file mode 100644 index 0000000000..ddd6374483 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/KokkosAlgorithms_dummy.cpp @@ -0,0 +1,17 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +void KOKKOS_ALGORITHMS_SRC_DUMMY_PREVENT_LINK_ERROR() {} diff --git a/bundled/kokkos-4.5.01/algorithms/src/Kokkos_NestedSort.hpp b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_NestedSort.hpp new file mode 100644 index 0000000000..18e0674efe --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_NestedSort.hpp @@ -0,0 +1,30 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_NESTED_SORT_HPP_ +#define KOKKOS_NESTED_SORT_HPP_ +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +#define KOKKOS_IMPL_PUBLIC_INCLUDE +#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_NESTED_SORT +#endif + +#include "sorting/Kokkos_NestedSortPublicAPI.hpp" + +#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_NESTED_SORT +#undef KOKKOS_IMPL_PUBLIC_INCLUDE +#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_NESTED_SORT +#endif +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/Kokkos_Random.hpp b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_Random.hpp similarity index 92% rename from bundled/kokkos-3.7.00/algorithms/src/Kokkos_Random.hpp rename to bundled/kokkos-4.5.01/algorithms/src/Kokkos_Random.hpp index 1d85ffdfb9..b28ea4c2ca 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/Kokkos_Random.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_Random.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_RANDOM_HPP #define KOKKOS_RANDOM_HPP @@ -631,8 +603,7 @@ struct Random_XorShift1024_UseCArrayState : std::false_type {}; #endif #ifdef KOKKOS_ENABLE_HIP template <> -struct Random_XorShift1024_UseCArrayState - : std::false_type {}; +struct Random_XorShift1024_UseCArrayState : std::false_type {}; #endif #ifdef KOKKOS_ENABLE_OPENMPTARGET template <> @@ -644,7 +615,7 @@ template struct Random_UniqueIndex { using locks_view_type = View; KOKKOS_FUNCTION - static int get_state_idx(const locks_view_type) { + static int get_state_idx(const locks_view_type&) { KOKKOS_IF_ON_HOST( (return DeviceType::execution_space::impl_hardware_thread_id();)) @@ -657,7 +628,7 @@ struct Random_UniqueIndex { #if defined(KOKKOS_ENABLE_CUDA) #define KOKKOS_IMPL_EXECUTION_SPACE_CUDA_OR_HIP Kokkos::Cuda #elif defined(KOKKOS_ENABLE_HIP) -#define KOKKOS_IMPL_EXECUTION_SPACE_CUDA_OR_HIP Kokkos::Experimental::HIP +#define KOKKOS_IMPL_EXECUTION_SPACE_CUDA_OR_HIP Kokkos::HIP #endif template @@ -694,17 +665,16 @@ struct Random_UniqueIndex< #ifdef KOKKOS_ENABLE_SYCL template -struct Random_UniqueIndex< - Kokkos::Device> { +struct Random_UniqueIndex> { using locks_view_type = - View>; + View>; KOKKOS_FUNCTION static int get_state_idx(const locks_view_type& locks_) { auto item = sycl::ext::oneapi::experimental::this_nd_item<3>(); std::size_t threadIdx[3] = {item.get_local_id(2), item.get_local_id(1), item.get_local_id(0)}; std::size_t blockIdx[3] = {item.get_group(2), item.get_group(1), - item.get_group(0)}; + item.get_group(0)}; std::size_t blockDim[3] = {item.get_local_range(2), item.get_local_range(1), item.get_local_range(0)}; std::size_t gridDim[3] = { @@ -878,18 +848,17 @@ class Random_XorShift64 { return drand(end - start) + start; } - // Marsaglia polar method for drawing a standard normal distributed random + // Box-muller method for drawing a standard normal distributed random // number KOKKOS_INLINE_FUNCTION double normal() { - double S = 2.0; - double U; - while (S >= 1.0) { - U = 2.0 * drand() - 1.0; - const double V = 2.0 * drand() - 1.0; - S = U * U + V * V; - } - return U * std::sqrt(-2.0 * std::log(S) / S); + constexpr auto two_pi = 2 * Kokkos::numbers::pi_v; + + const double u = drand(); + const double v = drand(); + const double r = Kokkos::sqrt(-2.0 * Kokkos::log(u)); + const double theta = v * two_pi; + return r * Kokkos::cos(theta); } KOKKOS_INLINE_FUNCTION @@ -907,38 +876,32 @@ class Random_XorShift64_Pool { using execution_space = typename device_type::execution_space; using locks_type = View; using state_data_type = View; - locks_type locks_; - state_data_type state_; - int num_states_; - int padding_; + + locks_type locks_ = {}; + state_data_type state_ = {}; + int num_states_ = {}; + int padding_ = {}; public: using generator_type = Random_XorShift64; - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool() { - num_states_ = 0; - padding_ = 0; - } +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_DEFAULTED_FUNCTION Random_XorShift64_Pool() = default; + + KOKKOS_DEFAULTED_FUNCTION Random_XorShift64_Pool( + Random_XorShift64_Pool const&) = default; + + KOKKOS_DEFAULTED_FUNCTION Random_XorShift64_Pool& operator=( + Random_XorShift64_Pool const&) = default; +#else + Random_XorShift64_Pool() = default; +#endif Random_XorShift64_Pool(uint64_t seed) { num_states_ = 0; init(seed, execution_space().concurrency()); } - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool(const Random_XorShift64_Pool& src) - : locks_(src.locks_), state_(src.state_), num_states_(src.num_states_) {} - - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool operator=(const Random_XorShift64_Pool& src) { - locks_ = src.locks_; - state_ = src.state_; - num_states_ = src.num_states_; - padding_ = src.padding_; - return *this; - } - void init(uint64_t seed, int num_states) { if (seed == 0) seed = uint64_t(1318319); // I only want to pad on CPU like archs (less than 1000 threads). 64 is a @@ -976,8 +939,8 @@ class Random_XorShift64_Pool { deep_copy(locks_, h_lock); } - KOKKOS_INLINE_FUNCTION - Random_XorShift64 get_state() const { + KOKKOS_INLINE_FUNCTION Random_XorShift64 get_state() const { + KOKKOS_EXPECTS(num_states_ > 0); const int i = Impl::Random_UniqueIndex::get_state_idx(locks_); return Random_XorShift64(state_(i, 0), i); } @@ -991,6 +954,8 @@ class Random_XorShift64_Pool { KOKKOS_INLINE_FUNCTION void free_state(const Random_XorShift64& state) const { state_(state.state_idx_, 0) = state.state_; + // Release the lock only after the state has been updated in memory + Kokkos::memory_fence(); locks_(state.state_idx_, 0) = 0; } }; @@ -1127,18 +1092,17 @@ class Random_XorShift1024 { return drand(end - start) + start; } - // Marsaglia polar method for drawing a standard normal distributed random + // Box-muller method for drawing a standard normal distributed random // number KOKKOS_INLINE_FUNCTION double normal() { - double S = 2.0; - double U; - while (S >= 1.0) { - U = 2.0 * drand() - 1.0; - const double V = 2.0 * drand() - 1.0; - S = U * U + V * V; - } - return U * std::sqrt(-2.0 * std::log(S) / S); + constexpr auto two_pi = 2 * Kokkos::numbers::pi_v; + + const double u = drand(); + const double v = drand(); + const double r = Kokkos::sqrt(-2.0 * Kokkos::log(u)); + const double theta = v * two_pi; + return r * Kokkos::cos(theta); } KOKKOS_INLINE_FUNCTION @@ -1156,45 +1120,37 @@ class Random_XorShift1024_Pool { using execution_space = typename device_type::execution_space; using locks_type = View; using int_view_type = View; - using state_data_type = View; + using state_data_type = View; - locks_type locks_; - state_data_type state_; - int_view_type p_; - int num_states_; - int padding_; + locks_type locks_ = {}; + state_data_type state_ = {}; + int_view_type p_ = {}; + int num_states_ = {}; + int padding_ = {}; friend class Random_XorShift1024; public: using generator_type = Random_XorShift1024; - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool() { num_states_ = 0; } +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_DEFAULTED_FUNCTION Random_XorShift1024_Pool() = default; - inline Random_XorShift1024_Pool(uint64_t seed) { - num_states_ = 0; + KOKKOS_DEFAULTED_FUNCTION Random_XorShift1024_Pool( + Random_XorShift1024_Pool const&) = default; - init(seed, execution_space().concurrency()); - } + KOKKOS_DEFAULTED_FUNCTION Random_XorShift1024_Pool& operator=( + Random_XorShift1024_Pool const&) = default; +#else + Random_XorShift1024_Pool() = default; +#endif - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool(const Random_XorShift1024_Pool& src) - : locks_(src.locks_), - state_(src.state_), - p_(src.p_), - num_states_(src.num_states_) {} + Random_XorShift1024_Pool(uint64_t seed) { + num_states_ = 0; - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool operator=(const Random_XorShift1024_Pool& src) { - locks_ = src.locks_; - state_ = src.state_; - p_ = src.p_; - num_states_ = src.num_states_; - padding_ = src.padding_; - return *this; + init(seed, execution_space().concurrency()); } - inline void init(uint64_t seed, int num_states) { + void init(uint64_t seed, int num_states) { if (seed == 0) seed = uint64_t(1318319); // I only want to pad on CPU like archs (less than 1000 threads). 64 is a // magic number, or random number I just wanted something not too large and @@ -1237,6 +1193,7 @@ class Random_XorShift1024_Pool { KOKKOS_INLINE_FUNCTION Random_XorShift1024 get_state() const { + KOKKOS_EXPECTS(num_states_ > 0); const int i = Impl::Random_UniqueIndex::get_state_idx(locks_); return Random_XorShift1024(state_, p_(i, 0), i); }; @@ -1250,7 +1207,9 @@ class Random_XorShift1024_Pool { KOKKOS_INLINE_FUNCTION void free_state(const Random_XorShift1024& state) const { for (int i = 0; i < 16; i++) state_(state.state_idx_, i) = state.state_[i]; - p_(state.state_idx_, 0) = state.p_; + p_(state.state_idx_, 0) = state.p_; + // Release the lock only after the state has been updated in memory + Kokkos::memory_fence(); locks_(state.state_idx_, 0) = 0; } }; @@ -1556,7 +1515,7 @@ void fill_random(const ExecutionSpace& exec, ViewType a, RandomPool g, "Kokkos::fill_random", Kokkos::RangePolicy(exec, 0, (LDA + 127) / 128), Impl::fill_random_functor_begin_end( + ViewType::rank, IndexType>( a, g, begin, end)); } @@ -1583,13 +1542,23 @@ template void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type begin, typename ViewType::const_value_type end) { - fill_random(typename ViewType::execution_space{}, a, g, begin, end); + Kokkos::fence( + "fill_random: fence before since no execution space instance provided"); + typename ViewType::execution_space exec; + fill_random(exec, a, g, begin, end); + exec.fence( + "fill_random: fence after since no execution space instance provided"); } template void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type range) { - fill_random(typename ViewType::execution_space{}, a, g, 0, range); + Kokkos::fence( + "fill_random: fence before since no execution space instance provided"); + typename ViewType::execution_space exec; + fill_random(exec, a, g, 0, range); + exec.fence( + "fill_random: fence after since no execution space instance provided"); } } // namespace Kokkos diff --git a/bundled/kokkos-4.5.01/algorithms/src/Kokkos_Sort.hpp b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_Sort.hpp new file mode 100644 index 0000000000..136b4ec82d --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_Sort.hpp @@ -0,0 +1,33 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_SORT_HPP_ +#define KOKKOS_SORT_HPP_ +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +#define KOKKOS_IMPL_PUBLIC_INCLUDE +#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_SORT +#endif + +#include "sorting/Kokkos_BinSortPublicAPI.hpp" +#include "sorting/Kokkos_SortPublicAPI.hpp" +#include "sorting/Kokkos_SortByKeyPublicAPI.hpp" +#include "sorting/Kokkos_NestedSortPublicAPI.hpp" + +#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_SORT +#undef KOKKOS_IMPL_PUBLIC_INCLUDE +#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_SORT +#endif +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/Kokkos_StdAlgorithms.hpp b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_StdAlgorithms.hpp similarity index 70% rename from bundled/kokkos-3.7.00/algorithms/src/Kokkos_StdAlgorithms.hpp rename to bundled/kokkos-4.5.01/algorithms/src/Kokkos_StdAlgorithms.hpp index 3e0f731cf0..b532a774e1 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/Kokkos_StdAlgorithms.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/Kokkos_StdAlgorithms.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_HPP #define KOKKOS_STD_ALGORITHMS_HPP @@ -63,7 +35,6 @@ // following the std classification. // modifying ops -#include "std_algorithms/Kokkos_Swap.hpp" #include "std_algorithms/Kokkos_IterSwap.hpp" // non-modifying sequence diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp new file mode 100644 index 0000000000..8e7de32a07 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp @@ -0,0 +1,129 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_BIN_OPS_PUBLIC_API_HPP_ +#define KOKKOS_BIN_OPS_PUBLIC_API_HPP_ + +#include +#include + +namespace Kokkos { + +template +struct BinOp1D { + int max_bins_ = {}; + double mul_ = {}; + double min_ = {}; + +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_DEPRECATED BinOp1D() = default; +#else + BinOp1D() = delete; +#endif + + // Construct BinOp with number of bins, minimum value and maximum value + BinOp1D(int max_bins, typename KeyViewType::const_value_type min, + typename KeyViewType::const_value_type max) + : max_bins_(max_bins + 1), + // Cast to double to avoid possible overflow when using integer + mul_(static_cast(max_bins) / + (static_cast(max) - static_cast(min))), + min_(static_cast(min)) { + // For integral types the number of bins may be larger than the range + // in which case we can exactly have one unique value per bin + // and then don't need to sort bins. + if (std::is_integral::value && + (static_cast(max) - static_cast(min)) <= + static_cast(max_bins)) { + mul_ = 1.; + } + } + + // Determine bin index from key value + template + KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { + return static_cast(mul_ * (static_cast(keys(i)) - min_)); + } + + // Return maximum bin index + 1 + KOKKOS_INLINE_FUNCTION + int max_bins() const { return max_bins_; } + + // Compare to keys within a bin if true new_val will be put before old_val + template + KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, + iType2& i2) const { + return keys(i1) < keys(i2); + } +}; + +template +struct BinOp3D { + int max_bins_[3] = {}; + double mul_[3] = {}; + double min_[3] = {}; + +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_DEPRECATED BinOp3D() = default; +#else + BinOp3D() = delete; +#endif + + BinOp3D(int max_bins[], typename KeyViewType::const_value_type min[], + typename KeyViewType::const_value_type max[]) { + max_bins_[0] = max_bins[0]; + max_bins_[1] = max_bins[1]; + max_bins_[2] = max_bins[2]; + mul_[0] = static_cast(max_bins[0]) / + (static_cast(max[0]) - static_cast(min[0])); + mul_[1] = static_cast(max_bins[1]) / + (static_cast(max[1]) - static_cast(min[1])); + mul_[2] = static_cast(max_bins[2]) / + (static_cast(max[2]) - static_cast(min[2])); + min_[0] = static_cast(min[0]); + min_[1] = static_cast(min[1]); + min_[2] = static_cast(min[2]); + } + + template + KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { + return int((((int(mul_[0] * (keys(i, 0) - min_[0])) * max_bins_[1]) + + int(mul_[1] * (keys(i, 1) - min_[1]))) * + max_bins_[2]) + + int(mul_[2] * (keys(i, 2) - min_[2]))); + } + + KOKKOS_INLINE_FUNCTION + int max_bins() const { return max_bins_[0] * max_bins_[1] * max_bins_[2]; } + + template + KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, + iType2& i2) const { + if (keys(i1, 0) > keys(i2, 0)) + return true; + else if (keys(i1, 0) == keys(i2, 0)) { + if (keys(i1, 1) > keys(i2, 1)) + return true; + else if (keys(i1, 1) == keys(i2, 1)) { + if (keys(i1, 2) > keys(i2, 2)) return true; + } + } + return false; + } +}; + +} // namespace Kokkos +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp new file mode 100644 index 0000000000..f417b6b13b --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp @@ -0,0 +1,411 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_BIN_SORT_PUBLIC_API_HPP_ +#define KOKKOS_BIN_SORT_PUBLIC_API_HPP_ + +#include "Kokkos_BinOpsPublicAPI.hpp" +#include "impl/Kokkos_CopyOpsForBinSortImpl.hpp" +#include +#include + +namespace Kokkos { + +template +class BinSort { + public: + template + struct copy_functor { + using src_view_type = typename SrcViewType::const_type; + + using copy_op = Impl::CopyOp; + + DstViewType dst_values; + src_view_type src_values; + int dst_offset; + + copy_functor(DstViewType const& dst_values_, int const& dst_offset_, + SrcViewType const& src_values_) + : dst_values(dst_values_), + src_values(src_values_), + dst_offset(dst_offset_) {} + + KOKKOS_INLINE_FUNCTION + void operator()(const int& i) const { + copy_op::copy(dst_values, i + dst_offset, src_values, i); + } + }; + + template + struct copy_permute_functor { + // If a Kokkos::View then can generate constant random access + // otherwise can only use the constant type. + + using src_view_type = std::conditional_t< + Kokkos::is_view::value, + Kokkos::View= 230700) + , + Kokkos::MemoryTraits +#endif + >, + typename SrcViewType::const_type>; + + using perm_view_type = typename PermuteViewType::const_type; + + using copy_op = Impl::CopyOp; + + DstViewType dst_values; + perm_view_type sort_order; + src_view_type src_values; + int src_offset; + + copy_permute_functor(DstViewType const& dst_values_, + PermuteViewType const& sort_order_, + SrcViewType const& src_values_, int const& src_offset_) + : dst_values(dst_values_), + sort_order(sort_order_), + src_values(src_values_), + src_offset(src_offset_) {} + + KOKKOS_INLINE_FUNCTION + void operator()(const int& i) const { + copy_op::copy(dst_values, i, src_values, src_offset + sort_order(i)); + } + }; + + // Naming this alias "execution_space" would be problematic since it would be + // considered as execution space for the various functors which might use + // another execution space through sort() or create_permute_vector(). + using exec_space = typename Space::execution_space; + using bin_op_type = BinSortOp; + + struct bin_count_tag {}; + struct bin_offset_tag {}; + struct bin_binning_tag {}; + struct bin_sort_bins_tag {}; + + public: + using size_type = SizeType; + using value_type = size_type; + + using offset_type = Kokkos::View; + using bin_count_type = Kokkos::View; + + using const_key_view_type = typename KeyViewType::const_type; + + // If a Kokkos::View then can generate constant random access + // otherwise can only use the constant type. + + using const_rnd_key_view_type = std::conditional_t< + Kokkos::is_view::value, + Kokkos::View >, + const_key_view_type>; + + using non_const_key_scalar = typename KeyViewType::non_const_value_type; + using const_key_scalar = typename KeyViewType::const_value_type; + + using bin_count_atomic_type = + Kokkos::View >; + + private: + const_key_view_type keys; + const_rnd_key_view_type keys_rnd; + + public: + BinSortOp bin_op; + offset_type bin_offsets; + bin_count_atomic_type bin_count_atomic; + bin_count_type bin_count_const; + offset_type sort_order; + + int range_begin; + int range_end; + bool sort_within_bins; + + public: +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_DEPRECATED BinSort() = default; +#else + BinSort() = delete; +#endif + + //---------------------------------------- + // Constructor: takes the keys, the binning_operator and optionally whether to + // sort within bins (default false) + template + BinSort(const ExecutionSpace& exec, const_key_view_type keys_, + int range_begin_, int range_end_, BinSortOp bin_op_, + bool sort_within_bins_ = false) + : keys(keys_), + keys_rnd(keys_), + bin_op(bin_op_), + bin_offsets(), + bin_count_atomic(), + bin_count_const(), + sort_order(), + range_begin(range_begin_), + range_end(range_end_), + sort_within_bins(sort_within_bins_) { + static_assert( + Kokkos::SpaceAccessibility::accessible, + "The provided execution space must be able to access the memory space " + "BinSort was initialized with!"); + if (bin_op.max_bins() <= 0) + Kokkos::abort( + "The number of bins in the BinSortOp object must be greater than 0!"); + bin_count_atomic = Kokkos::View( + "Kokkos::SortImpl::BinSortFunctor::bin_count", bin_op.max_bins()); + bin_count_const = bin_count_atomic; + bin_offsets = + offset_type(view_alloc(exec, WithoutInitializing, + "Kokkos::SortImpl::BinSortFunctor::bin_offsets"), + bin_op.max_bins()); + sort_order = + offset_type(view_alloc(exec, WithoutInitializing, + "Kokkos::SortImpl::BinSortFunctor::sort_order"), + range_end - range_begin); + } + + BinSort(const_key_view_type keys_, int range_begin_, int range_end_, + BinSortOp bin_op_, bool sort_within_bins_ = false) + : BinSort(exec_space{}, keys_, range_begin_, range_end_, bin_op_, + sort_within_bins_) {} + + template + BinSort(const ExecutionSpace& exec, const_key_view_type keys_, + BinSortOp bin_op_, bool sort_within_bins_ = false) + : BinSort(exec, keys_, 0, keys_.extent(0), bin_op_, sort_within_bins_) {} + + BinSort(const_key_view_type keys_, BinSortOp bin_op_, + bool sort_within_bins_ = false) + : BinSort(exec_space{}, keys_, bin_op_, sort_within_bins_) {} + + //---------------------------------------- + // Create the permutation vector, the bin_offset array and the bin_count + // array. Can be called again if keys changed + template + void create_permute_vector(const ExecutionSpace& exec) { + static_assert( + Kokkos::SpaceAccessibility::accessible, + "The provided execution space must be able to access the memory space " + "BinSort was initialized with!"); + + const size_t len = range_end - range_begin; + Kokkos::parallel_for( + "Kokkos::Sort::BinCount", + Kokkos::RangePolicy(exec, 0, len), + *this); + Kokkos::parallel_scan("Kokkos::Sort::BinOffset", + Kokkos::RangePolicy( + exec, 0, bin_op.max_bins()), + *this); + + Kokkos::deep_copy(exec, bin_count_atomic, 0); + Kokkos::parallel_for( + "Kokkos::Sort::BinBinning", + Kokkos::RangePolicy(exec, 0, len), + *this); + + if (sort_within_bins) + Kokkos::parallel_for( + "Kokkos::Sort::BinSort", + Kokkos::RangePolicy( + exec, 0, bin_op.max_bins()), + *this); + } + + // Create the permutation vector, the bin_offset array and the bin_count + // array. Can be called again if keys changed + void create_permute_vector() { + Kokkos::fence("Kokkos::Binsort::create_permute_vector: before"); + exec_space e{}; + create_permute_vector(e); + e.fence("Kokkos::Binsort::create_permute_vector: after"); + } + + // Sort a subset of a view with respect to the first dimension using the + // permutation array + template + void sort(const ExecutionSpace& exec, ValuesViewType const& values, + int values_range_begin, int values_range_end) const { + if (values.extent(0) == 0) { + return; + } + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "The provided execution space must be able to access the memory space " + "BinSort was initialized with!"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename ValuesViewType::memory_space>::accessible, + "The provided execution space must be able to access the memory space " + "of the View argument!"); + + const size_t len = range_end - range_begin; + const size_t values_len = values_range_end - values_range_begin; + if (len != values_len) { + Kokkos::abort( + "BinSort::sort: values range length != permutation vector length"); + } + + using scratch_view_type = + Kokkos::View; + scratch_view_type sorted_values( + view_alloc(exec, WithoutInitializing, + "Kokkos::SortImpl::BinSortFunctor::sorted_values"), + values.rank_dynamic > 0 ? len : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 1 ? values.extent(1) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 2 ? values.extent(2) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 3 ? values.extent(3) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 4 ? values.extent(4) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 5 ? values.extent(5) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 6 ? values.extent(6) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 7 ? values.extent(7) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG); + + { + copy_permute_functor + functor(sorted_values, sort_order, values, + values_range_begin - range_begin); + + parallel_for("Kokkos::Sort::CopyPermute", + Kokkos::RangePolicy(exec, 0, len), functor); + } + + { + copy_functor functor( + values, range_begin, sorted_values); + + parallel_for("Kokkos::Sort::Copy", + Kokkos::RangePolicy(exec, 0, len), functor); + } + } + + // Sort a subset of a view with respect to the first dimension using the + // permutation array + template + void sort(ValuesViewType const& values, int values_range_begin, + int values_range_end) const { + Kokkos::fence("Kokkos::Binsort::sort: before"); + exec_space exec; + sort(exec, values, values_range_begin, values_range_end); + exec.fence("Kokkos::BinSort:sort: after"); + } + + template + void sort(ExecutionSpace const& exec, ValuesViewType const& values) const { + this->sort(exec, values, 0, /*values.extent(0)*/ range_end - range_begin); + } + + template + void sort(ValuesViewType const& values) const { + this->sort(values, 0, /*values.extent(0)*/ range_end - range_begin); + } + + // Get the permutation vector + KOKKOS_INLINE_FUNCTION + offset_type get_permute_vector() const { return sort_order; } + + // Get the start offsets for each bin + KOKKOS_INLINE_FUNCTION + offset_type get_bin_offsets() const { return bin_offsets; } + + // Get the count for each bin + KOKKOS_INLINE_FUNCTION + bin_count_type get_bin_count() const { return bin_count_const; } + + public: + KOKKOS_INLINE_FUNCTION + void operator()(const bin_count_tag& /*tag*/, const int i) const { + const int j = range_begin + i; + bin_count_atomic(bin_op.bin(keys, j))++; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const bin_offset_tag& /*tag*/, const int i, + value_type& offset, const bool& final) const { + if (final) { + bin_offsets(i) = offset; + } + offset += bin_count_const(i); + } + + KOKKOS_INLINE_FUNCTION + void operator()(const bin_binning_tag& /*tag*/, const int i) const { + const int j = range_begin + i; + const int bin = bin_op.bin(keys, j); + const int count = bin_count_atomic(bin)++; + + sort_order(bin_offsets(bin) + count) = j; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const bin_sort_bins_tag& /*tag*/, const int i) const { + auto bin_size = bin_count_const(i); + if (bin_size <= 1) return; + constexpr bool use_std_sort = + std::is_same_v; + int lower_bound = bin_offsets(i); + int upper_bound = lower_bound + bin_size; + // Switching to std::sort for more than 10 elements has been found + // reasonable experimentally. + if (use_std_sort && bin_size > 10) { + KOKKOS_IF_ON_HOST( + (std::sort(sort_order.data() + lower_bound, + sort_order.data() + upper_bound, + [this](int p, int q) { return bin_op(keys_rnd, p, q); });)) + } else { + for (int k = lower_bound + 1; k < upper_bound; ++k) { + int old_idx = sort_order(k); + int j = k - 1; + while (j >= lower_bound) { + int new_idx = sort_order(j); + if (!bin_op(keys_rnd, old_idx, new_idx)) break; + sort_order(j + 1) = new_idx; + --j; + } + sort_order(j + 1) = old_idx; + } + } + } +}; + +} // namespace Kokkos +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_NestedSortPublicAPI.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_NestedSortPublicAPI.hpp new file mode 100644 index 0000000000..dd468e0734 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_NestedSortPublicAPI.hpp @@ -0,0 +1,100 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_NESTED_SORT_PUBLIC_API_HPP_ +#define KOKKOS_NESTED_SORT_PUBLIC_API_HPP_ + +#include "impl/Kokkos_NestedSortImpl.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { + +template +KOKKOS_INLINE_FUNCTION void sort_team(const TeamMember& t, + const ViewType& view) { + Impl::sort_nested_impl(t, view, nullptr, + Experimental::Impl::StdAlgoLessThanBinaryPredicate< + typename ViewType::non_const_value_type>(), + Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_team(const TeamMember& t, const ViewType& view, + const Comparator& comp) { + Impl::sort_nested_impl(t, view, nullptr, comp, Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_by_key_team(const TeamMember& t, + const KeyViewType& keyView, + const ValueViewType& valueView) { + Impl::sort_nested_impl(t, keyView, valueView, + Experimental::Impl::StdAlgoLessThanBinaryPredicate< + typename KeyViewType::non_const_value_type>(), + Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_by_key_team(const TeamMember& t, + const KeyViewType& keyView, + const ValueViewType& valueView, + const Comparator& comp) { + Impl::sort_nested_impl(t, keyView, valueView, comp, + Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_thread(const TeamMember& t, + const ViewType& view) { + Impl::sort_nested_impl(t, view, nullptr, + Experimental::Impl::StdAlgoLessThanBinaryPredicate< + typename ViewType::non_const_value_type>(), + Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_thread(const TeamMember& t, + const ViewType& view, + const Comparator& comp) { + Impl::sort_nested_impl(t, view, nullptr, comp, Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_by_key_thread(const TeamMember& t, + const KeyViewType& keyView, + const ValueViewType& valueView) { + Impl::sort_nested_impl(t, keyView, valueView, + Experimental::Impl::StdAlgoLessThanBinaryPredicate< + typename KeyViewType::non_const_value_type>(), + Impl::NestedRange()); +} + +template +KOKKOS_INLINE_FUNCTION void sort_by_key_thread(const TeamMember& t, + const KeyViewType& keyView, + const ValueViewType& valueView, + const Comparator& comp) { + Impl::sort_nested_impl(t, keyView, valueView, comp, + Impl::NestedRange()); +} + +} // namespace Experimental +} // namespace Kokkos +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_SortByKeyPublicAPI.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_SortByKeyPublicAPI.hpp new file mode 100644 index 0000000000..fc73eccad6 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_SortByKeyPublicAPI.hpp @@ -0,0 +1,117 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_SORT_BY_KEY_PUBLIC_API_HPP_ +#define KOKKOS_SORT_BY_KEY_PUBLIC_API_HPP_ + +#include "./impl/Kokkos_SortByKeyImpl.hpp" +#include +#include + +namespace Kokkos::Experimental { + +// --------------------------------------------------------------- +// basic overloads +// --------------------------------------------------------------- + +template +void sort_by_key( + const ExecutionSpace& exec, + const Kokkos::View& keys, + const Kokkos::View& values) { + // constraints + using KeysType = Kokkos::View; + using ValuesType = Kokkos::View; + ::Kokkos::Impl::static_assert_is_admissible_to_kokkos_sort_by_key(keys); + ::Kokkos::Impl::static_assert_is_admissible_to_kokkos_sort_by_key(values); + + static_assert(SpaceAccessibility::accessible, + "Kokkos::sort: execution space instance is not able to access " + "the memory space of the keys View argument!"); + static_assert( + SpaceAccessibility::accessible, + "Kokkos::sort: execution space instance is not able to access " + "the memory space of the values View argument!"); + + static_assert(KeysType::static_extent(0) == 0 || + ValuesType::static_extent(0) == 0 || + KeysType::static_extent(0) == ValuesType::static_extent(0)); + if (values.size() != keys.size()) + Kokkos::abort((std::string("values and keys extents must be the same. The " + "values extent is ") + + std::to_string(values.size()) + ", and the keys extent is " + + std::to_string(keys.size()) + ".") + .c_str()); + + if (keys.extent(0) <= 1) { + return; + } + + ::Kokkos::Impl::sort_by_key_device_view_without_comparator(exec, keys, + values); +} + +// --------------------------------------------------------------- +// overloads supporting a custom comparator +// --------------------------------------------------------------- + +template +void sort_by_key( + const ExecutionSpace& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + const ComparatorType& comparator) { + // constraints + using KeysType = Kokkos::View; + using ValuesType = Kokkos::View; + ::Kokkos::Impl::static_assert_is_admissible_to_kokkos_sort_by_key(keys); + ::Kokkos::Impl::static_assert_is_admissible_to_kokkos_sort_by_key(values); + + static_assert(SpaceAccessibility::accessible, + "Kokkos::sort: execution space instance is not able to access " + "the memory space of the keys View argument!"); + static_assert( + SpaceAccessibility::accessible, + "Kokkos::sort: execution space instance is not able to access " + "the memory space of the values View argument!"); + + static_assert(KeysType::static_extent(0) == 0 || + ValuesType::static_extent(0) == 0 || + KeysType::static_extent(0) == ValuesType::static_extent(0)); + if (values.size() != keys.size()) + Kokkos::abort((std::string("values and keys extents must be the same. The " + "values extent is ") + + std::to_string(values.size()) + ", and the keys extent is " + + std::to_string(keys.size()) + ".") + .c_str()); + + if (keys.extent(0) <= 1) { + return; + } + + ::Kokkos::Impl::sort_by_key_device_view_with_comparator(exec, keys, values, + comparator); +} + +} // namespace Kokkos::Experimental +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp new file mode 100644 index 0000000000..20026c77e4 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp @@ -0,0 +1,204 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_SORT_PUBLIC_API_HPP_ +#define KOKKOS_SORT_PUBLIC_API_HPP_ + +#include "./impl/Kokkos_SortImpl.hpp" +#include +#include +#include + +namespace Kokkos { + +// --------------------------------------------------------------- +// basic overloads +// --------------------------------------------------------------- + +template +void sort(const ExecutionSpace& exec, + const Kokkos::View& view) { + // constraints + using ViewType = Kokkos::View; + using MemSpace = typename ViewType::memory_space; + static_assert( + ViewType::rank == 1 && + (std::is_same_v || + std::is_same_v || + std::is_same_v), + "Kokkos::sort without comparator: supports 1D Views with LayoutRight, " + "LayoutLeft or LayoutStride."); + + static_assert(SpaceAccessibility::accessible, + "Kokkos::sort: execution space instance is not able to access " + "the memory space of the " + "View argument!"); + + if (view.extent(0) <= 1) { + return; + } + + if constexpr (Impl::better_off_calling_std_sort_v) { + exec.fence("Kokkos::sort without comparator use std::sort"); + if (view.span_is_contiguous()) { + std::sort(view.data(), view.data() + view.size()); + } else { + auto first = ::Kokkos::Experimental::begin(view); + auto last = ::Kokkos::Experimental::end(view); + std::sort(first, last); + } + } else { + Impl::sort_device_view_without_comparator(exec, view); + } +} + +template +void sort(const Kokkos::View& view) { + using ViewType = Kokkos::View; + static_assert(ViewType::rank == 1, + "Kokkos::sort: currently only supports rank-1 Views."); + + Kokkos::fence("Kokkos::sort: before"); + + if (view.extent(0) <= 1) { + return; + } + + typename ViewType::execution_space exec; + sort(exec, view); + exec.fence("Kokkos::sort: fence after sorting"); +} + +// --------------------------------------------------------------- +// overloads supporting a custom comparator +// --------------------------------------------------------------- +template +void sort(const ExecutionSpace& exec, + const Kokkos::View& view, + const ComparatorType& comparator) { + // constraints + using ViewType = Kokkos::View; + using MemSpace = typename ViewType::memory_space; + static_assert( + ViewType::rank == 1 && + (std::is_same_v || + std::is_same_v || + std::is_same_v), + "Kokkos::sort with comparator: supports 1D Views with LayoutRight, " + "LayoutLeft or LayoutStride."); + + static_assert(SpaceAccessibility::accessible, + "Kokkos::sort: execution space instance is not able to access " + "the memory space of the View argument!"); + + if (view.extent(0) <= 1) { + return; + } + + if constexpr (Impl::better_off_calling_std_sort_v) { + exec.fence("Kokkos::sort with comparator use std::sort"); + if (view.span_is_contiguous()) { + std::sort(view.data(), view.data() + view.size(), comparator); + } else { + auto first = ::Kokkos::Experimental::begin(view); + auto last = ::Kokkos::Experimental::end(view); + std::sort(first, last, comparator); + } + } else { + Impl::sort_device_view_with_comparator(exec, view, comparator); + } +} + +template +void sort(const Kokkos::View& view, + const ComparatorType& comparator) { + using ViewType = Kokkos::View; + static_assert( + ViewType::rank == 1 && + (std::is_same_v || + std::is_same_v || + std::is_same_v), + "Kokkos::sort with comparator: supports 1D Views with LayoutRight, " + "LayoutLeft or LayoutStride."); + + Kokkos::fence("Kokkos::sort with comparator: before"); + + if (view.extent(0) <= 1) { + return; + } + + typename ViewType::execution_space exec; + sort(exec, view, comparator); + exec.fence("Kokkos::sort with comparator: fence after sorting"); +} + +// --------------------------------------------------------------- +// overloads for sorting a view with a subrange +// specified via integers begin, end +// --------------------------------------------------------------- + +template +std::enable_if_t::value> sort( + const ExecutionSpace& exec, ViewType view, size_t const begin, + size_t const end) { + // view must be rank-1 because the Impl::min_max_functor + // used below only works for rank-1 views for now + static_assert(ViewType::rank == 1, + "Kokkos::sort: currently only supports rank-1 Views."); + + if (view.extent(0) <= 1) { + return; + } + + using range_policy = Kokkos::RangePolicy; + using CompType = BinOp1D; + + Kokkos::MinMaxScalar result; + Kokkos::MinMax reducer(result); + + parallel_reduce("Kokkos::Sort::FindExtent", range_policy(exec, begin, end), + Impl::min_max_functor(view), reducer); + + if (result.min_val == result.max_val) return; + + BinSort bin_sort( + exec, view, begin, end, + CompType((end - begin) / 2, result.min_val, result.max_val), true); + + bin_sort.create_permute_vector(exec); + bin_sort.sort(exec, view, begin, end); +} + +template +void sort(ViewType view, size_t const begin, size_t const end) { + // same constraints as the overload above which this gets dispatched to + static_assert(ViewType::rank == 1, + "Kokkos::sort: currently only supports rank-1 Views."); + + Kokkos::fence("Kokkos::sort: before"); + + if (view.extent(0) <= 1) { + return; + } + + typename ViewType::execution_space exec; + sort(exec, view, begin, end); + exec.fence("Kokkos::Sort: fence after sorting"); +} + +} // namespace Kokkos +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_CopyOpsForBinSortImpl.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_CopyOpsForBinSortImpl.hpp new file mode 100644 index 0000000000..07f5926d82 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_CopyOpsForBinSortImpl.hpp @@ -0,0 +1,61 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_COPY_OPS_FOR_BINSORT_IMPL_HPP_ +#define KOKKOS_COPY_OPS_FOR_BINSORT_IMPL_HPP_ + +#include +#include + +namespace Kokkos { +namespace Impl { + +template +struct CopyOp; + +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + dst(i_dst) = src(i_src); + } +}; + +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + for (int j = 0; j < (int)dst.extent(1); j++) dst(i_dst, j) = src(i_src, j); + } +}; + +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + for (int j = 0; j < dst.extent(1); j++) + for (int k = 0; k < dst.extent(2); k++) + dst(i_dst, j, k) = src(i_src, j, k); + } +}; + +} // namespace Impl +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_NestedSortImpl.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_NestedSortImpl.hpp new file mode 100644 index 0000000000..2fe58272d9 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_NestedSortImpl.hpp @@ -0,0 +1,114 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_NESTED_SORT_IMPL_HPP_ +#define KOKKOS_NESTED_SORT_IMPL_HPP_ + +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +// true for TeamVectorRange, false for ThreadVectorRange +template +struct NestedRange {}; + +// Specialization for team-level +template <> +struct NestedRange { + template + KOKKOS_FUNCTION static auto create(const TeamMember& t, SizeType len) { + return Kokkos::TeamVectorRange(t, len); + } + template + KOKKOS_FUNCTION static void barrier(const TeamMember& t) { + t.team_barrier(); + } +}; + +// Specialization for thread-level +template <> +struct NestedRange { + template + KOKKOS_FUNCTION static auto create(const TeamMember& t, SizeType len) { + return Kokkos::ThreadVectorRange(t, len); + } + // Barrier is no-op, as vector lanes of a thread are implicitly synchronized + // after parallel region + template + KOKKOS_FUNCTION static void barrier(const TeamMember&) {} +}; + +// When just doing sort (not sort_by_key), use nullptr_t for ValueViewType. +// This only takes the NestedRange instance for template arg deduction. +template +KOKKOS_INLINE_FUNCTION void sort_nested_impl( + const TeamMember& t, const KeyViewType& keyView, + [[maybe_unused]] const ValueViewType& valueView, const Comparator& comp, + const NestedRange) { + using SizeType = typename KeyViewType::size_type; + using KeyType = typename KeyViewType::non_const_value_type; + using Range = NestedRange; + SizeType n = keyView.extent(0); + SizeType npot = 1; + SizeType levels = 0; + // FIXME: ceiling power-of-two is a common thing to need - make it a utility + while (npot < n) { + levels++; + npot <<= 1; + } + for (SizeType i = 0; i < levels; i++) { + for (SizeType j = 0; j <= i; j++) { + // n/2 pairs of items are compared in parallel + Kokkos::parallel_for(Range::create(t, npot / 2), [=](const SizeType k) { + // How big are the brown/pink boxes? + // (Terminology comes from Wikipedia diagram) + // https://commons.wikimedia.org/wiki/File:BitonicSort.svg#/media/File:BitonicSort.svg + SizeType boxSize = SizeType(2) << (i - j); + // Which box contains this thread? + SizeType boxID = k >> (i - j); // k * 2 / boxSize; + SizeType boxStart = boxID << (1 + i - j); // boxID * boxSize + SizeType boxOffset = k - (boxStart >> 1); // k - boxID * boxSize / 2; + SizeType elem1 = boxStart + boxOffset; + // In first phase (j == 0, brown box): within a box, compare with the + // opposite value in the box. + // In later phases (j > 0, pink box): within a box, compare with fixed + // distance (boxSize / 2) apart. + SizeType elem2 = (j == 0) ? (boxStart + boxSize - 1 - boxOffset) + : (elem1 + boxSize / 2); + if (elem2 < n) { + KeyType key1 = keyView(elem1); + KeyType key2 = keyView(elem2); + if (comp(key2, key1)) { + keyView(elem1) = key2; + keyView(elem2) = key1; + if constexpr (!std::is_same_v) { + Kokkos::kokkos_swap(valueView(elem1), valueView(elem2)); + } + } + } + }); + Range::barrier(t); + } + } +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp new file mode 100644 index 0000000000..2a8f761d9b --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp @@ -0,0 +1,422 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_SORT_BY_KEY_FREE_FUNCS_IMPL_HPP_ +#define KOKKOS_SORT_BY_KEY_FREE_FUNCS_IMPL_HPP_ + +#include + +#if defined(KOKKOS_ENABLE_CUDA) + +// Workaround for `Instruction 'shfl' without '.sync' is not supported on +// .target sm_70 and higher from PTX ISA version 6.4`. +// Also see https://github.com/NVIDIA/cub/pull/170. +#if !defined(CUB_USE_COOPERATIVE_GROUPS) +#define CUB_USE_COOPERATIVE_GROUPS +#endif + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshadow" +#pragma GCC diagnostic ignored "-Wsuggest-override" + +#if defined(KOKKOS_COMPILER_CLANG) +// Some versions of Clang fail to compile Thrust, failing with errors like +// this: +// /thrust/system/cuda/detail/core/agent_launcher.h:557:11: +// error: use of undeclared identifier 'va_printf' +// The exact combination of versions for Clang and Thrust (or CUDA) for this +// failure was not investigated, however even very recent version combination +// (Clang 10.0.0 and Cuda 10.0) demonstrated failure. +// +// Defining _CubLog here locally allows us to avoid that code path, however +// disabling some debugging diagnostics +#pragma push_macro("_CubLog") +#ifdef _CubLog +#undef _CubLog +#endif +#define _CubLog +#include +#include +#pragma pop_macro("_CubLog") +#else +#include +#include +#endif + +#pragma GCC diagnostic pop + +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +#include +#include +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) && \ + (ONEDPL_VERSION_MAJOR > 2022 || \ + (ONEDPL_VERSION_MAJOR == 2022 && ONEDPL_VERSION_MINOR >= 2)) +#define KOKKOS_ONEDPL_HAS_SORT_BY_KEY +#include +#include +#endif + +namespace Kokkos::Impl { + +template +constexpr inline bool is_admissible_to_kokkos_sort_by_key = + ::Kokkos::is_view::value && T::rank() == 1 && + (std::is_same_v || + std::is_same_v || + std::is_same_v); + +template +KOKKOS_INLINE_FUNCTION constexpr void +static_assert_is_admissible_to_kokkos_sort_by_key(const ViewType& /* view */) { + static_assert(is_admissible_to_kokkos_sort_by_key, + "Kokkos::sort_by_key only accepts 1D values View with " + "LayoutRight, LayoutLeft or LayoutStride."); +} + +// For the fallback implementation for sort_by_key using Kokkos::sort, we need +// to consider if Kokkos::sort defers to the fallback implementation that copies +// the array to the host and uses std::sort, see +// copy_to_host_run_stdsort_copy_back() in impl/Kokkos_SortImpl.hpp. If +// sort_on_device_v is true, we assume that std::sort doesn't copy data. +// Otherwise, we manually copy all data to the host and provide Kokkos::sort +// with a host execution space. +template +inline constexpr bool sort_on_device_v = false; + +#if defined(KOKKOS_ENABLE_CUDA) +template +inline constexpr bool sort_on_device_v = true; + +template +void sort_by_key_cudathrust( + const Kokkos::Cuda& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + MaybeComparator&&... maybeComparator) { + const auto policy = thrust::cuda::par.on(exec.cuda_stream()); + auto keys_first = ::Kokkos::Experimental::begin(keys); + auto keys_last = ::Kokkos::Experimental::end(keys); + auto values_first = ::Kokkos::Experimental::begin(values); + thrust::sort_by_key(policy, keys_first, keys_last, values_first, + std::forward(maybeComparator)...); +} +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +template +inline constexpr bool sort_on_device_v = true; + +template +void sort_by_key_rocthrust( + const Kokkos::HIP& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + MaybeComparator&&... maybeComparator) { + const auto policy = thrust::hip::par.on(exec.hip_stream()); + auto keys_first = ::Kokkos::Experimental::begin(keys); + auto keys_last = ::Kokkos::Experimental::end(keys); + auto values_first = ::Kokkos::Experimental::begin(values); + thrust::sort_by_key(policy, keys_first, keys_last, values_first, + std::forward(maybeComparator)...); +} +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +template +inline constexpr bool sort_on_device_v = + std::is_same_v || + std::is_same_v; + +#ifdef KOKKOS_ONEDPL_HAS_SORT_BY_KEY +template +void sort_by_key_onedpl( + const Kokkos::SYCL& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + MaybeComparator&&... maybeComparator) { + if (keys.stride(0) != 1 && values.stride(0) != 1) { + Kokkos::abort( + "SYCL sort_by_key only supports rank-1 Views with stride(0) = 1."); + } + + // Can't use Experimental::begin/end here since the oneDPL then assumes that + // the data is on the host. + auto queue = exec.sycl_queue(); + auto policy = oneapi::dpl::execution::make_device_policy(queue); + const int n = keys.extent(0); + oneapi::dpl::sort_by_key(policy, keys.data(), keys.data() + n, values.data(), + std::forward(maybeComparator)...); +} +#endif +#endif + +template +void applyPermutation(const ExecutionSpace& space, + const PermutationView& permutation, + const ViewType& view) { + static_assert(std::is_integral_v); + + auto view_copy = Kokkos::create_mirror( + Kokkos::view_alloc(space, typename ExecutionSpace::memory_space{}, + Kokkos::WithoutInitializing), + view); + Kokkos::deep_copy(space, view_copy, view); + Kokkos::parallel_for( + "Kokkos::sort_by_key_via_sort::permute_" + view.label(), + Kokkos::RangePolicy(space, 0, view.extent(0)), + KOKKOS_LAMBDA(int i) { view(i) = view_copy(permutation(i)); }); +} + +// FIXME_NVCC: nvcc has trouble compiling lambdas inside a function with +// variadic templates (sort_by_key_via_sort). Switch to using functors instead. +template +struct IotaFunctor { + Permute _permute; + KOKKOS_FUNCTION void operator()(int i) const { _permute(i) = i; } +}; +template +struct LessFunctor { + Keys _keys; + KOKKOS_FUNCTION bool operator()(int i, int j) const { + return _keys(i) < _keys(j); + } +}; + +// FIXME_NVCC+MSVC: We can't use a lambda instead of a functor which gave us +// "For this host platform/dialect, an extended lambda cannot be defined inside +// the 'if' or 'else' block of a constexpr if statement" +template +struct KeyComparisonFunctor { + Keys m_keys; + Comparator m_comparator; + KOKKOS_FUNCTION bool operator()(int i, int j) const { + return m_comparator(m_keys(i), m_keys(j)); + } +}; + +template +void sort_by_key_via_sort( + const ExecutionSpace& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + MaybeComparator&&... maybeComparator) { + static_assert(sizeof...(MaybeComparator) <= 1); + + auto const n = keys.size(); + + Kokkos::View permute( + Kokkos::view_alloc(exec, Kokkos::WithoutInitializing, + "Kokkos::sort_by_key_via_sort::permute"), + n); + + // iota + Kokkos::parallel_for("Kokkos::sort_by_key_via_sort::iota", + Kokkos::RangePolicy(exec, 0, n), + IotaFunctor{permute}); + + using Layout = + typename Kokkos::View::array_layout; + if constexpr (!sort_on_device_v) { + auto host_keys = Kokkos::create_mirror_view( + Kokkos::view_alloc(Kokkos::HostSpace{}, Kokkos::WithoutInitializing), + keys); + auto host_permute = Kokkos::create_mirror_view( + Kokkos::view_alloc(Kokkos::HostSpace{}, Kokkos::WithoutInitializing), + permute); + Kokkos::deep_copy(exec, host_keys, keys); + Kokkos::deep_copy(exec, host_permute, permute); + + exec.fence("Kokkos::Impl::sort_by_key_via_sort: before host sort"); + Kokkos::DefaultHostExecutionSpace host_exec; + + if constexpr (sizeof...(MaybeComparator) == 0) { + Kokkos::sort(host_exec, host_permute, + LessFunctor{host_keys}); + } else { + auto keys_comparator = + std::get<0>(std::tuple(maybeComparator...)); + Kokkos::sort( + host_exec, host_permute, + KeyComparisonFunctor{ + host_keys, keys_comparator}); + } + host_exec.fence("Kokkos::Impl::sort_by_key_via_sort: after host sort"); + Kokkos::deep_copy(exec, permute, host_permute); + } else { +#ifdef KOKKOS_ENABLE_SYCL + auto* raw_keys_in_comparator = keys.data(); + auto stride = keys.stride(0); + if constexpr (sizeof...(MaybeComparator) == 0) { + Kokkos::sort( + exec, permute, KOKKOS_LAMBDA(int i, int j) { + return raw_keys_in_comparator[i * stride] < + raw_keys_in_comparator[j * stride]; + }); + } else { + auto keys_comparator = + std::get<0>(std::tuple(maybeComparator...)); + Kokkos::sort( + exec, permute, KOKKOS_LAMBDA(int i, int j) { + return keys_comparator(raw_keys_in_comparator[i * stride], + raw_keys_in_comparator[j * stride]); + }); + } +#else + if constexpr (sizeof...(MaybeComparator) == 0) { + Kokkos::sort(exec, permute, LessFunctor{keys}); + } else { + auto keys_comparator = + std::get<0>(std::tuple(maybeComparator...)); + Kokkos::sort( + exec, permute, + KeyComparisonFunctor{ + keys, keys_comparator}); + } +#endif + } + + applyPermutation(exec, permute, keys); + applyPermutation(exec, permute, values); +} + +// ------------------------------------------------------ +// +// specialize cases for sorting by key without comparator +// +// ------------------------------------------------------ + +#if defined(KOKKOS_ENABLE_CUDA) +template +void sort_by_key_device_view_without_comparator( + const Kokkos::Cuda& exec, + const Kokkos::View& keys, + const Kokkos::View& values) { + sort_by_key_cudathrust(exec, keys, values); +} +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +template +void sort_by_key_device_view_without_comparator( + const Kokkos::HIP& exec, + const Kokkos::View& keys, + const Kokkos::View& values) { + sort_by_key_rocthrust(exec, keys, values); +} +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +template +void sort_by_key_device_view_without_comparator( + const Kokkos::SYCL& exec, + const Kokkos::View& keys, + const Kokkos::View& values) { +#ifdef KOKKOS_ONEDPL_HAS_SORT_BY_KEY + if (keys.stride(0) == 1 && values.stride(0) == 1) + sort_by_key_onedpl(exec, keys, values); + else +#endif + sort_by_key_via_sort(exec, keys, values); +} +#endif + +// fallback case +template +std::enable_if_t::value> +sort_by_key_device_view_without_comparator( + const ExecutionSpace& exec, + const Kokkos::View& keys, + const Kokkos::View& values) { + sort_by_key_via_sort(exec, keys, values); +} + +// --------------------------------------------------- +// +// specialize cases for sorting by key with comparator +// +// --------------------------------------------------- + +#if defined(KOKKOS_ENABLE_CUDA) +template +void sort_by_key_device_view_with_comparator( + const Kokkos::Cuda& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + const ComparatorType& comparator) { + sort_by_key_cudathrust(exec, keys, values, comparator); +} +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +template +void sort_by_key_device_view_with_comparator( + const Kokkos::HIP& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + const ComparatorType& comparator) { + sort_by_key_rocthrust(exec, keys, values, comparator); +} +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +template +void sort_by_key_device_view_with_comparator( + const Kokkos::SYCL& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + const ComparatorType& comparator) { +#ifdef KOKKOS_ONEDPL_HAS_SORT_BY_KEY + if (keys.stride(0) == 1 && values.stride(0) == 1) + sort_by_key_onedpl(exec, keys, values, comparator); + else +#endif + sort_by_key_via_sort(exec, keys, values, comparator); +} +#endif + +// fallback case +template +std::enable_if_t::value> +sort_by_key_device_view_with_comparator( + const ExecutionSpace& exec, + const Kokkos::View& keys, + const Kokkos::View& values, + const ComparatorType& comparator) { + sort_by_key_via_sort(exec, keys, values, comparator); +} + +#undef KOKKOS_ONEDPL_HAS_SORT_BY_KEY + +} // namespace Kokkos::Impl +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp new file mode 100644 index 0000000000..734ce450f6 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp @@ -0,0 +1,426 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_SORT_FREE_FUNCS_IMPL_HPP_ +#define KOKKOS_SORT_FREE_FUNCS_IMPL_HPP_ + +#include "../Kokkos_BinOpsPublicAPI.hpp" +#include "../Kokkos_BinSortPublicAPI.hpp" +#include +#include +#include + +#if defined(KOKKOS_ENABLE_CUDA) + +// Workaround for `Instruction 'shfl' without '.sync' is not supported on +// .target sm_70 and higher from PTX ISA version 6.4`. +// Also see https://github.com/NVIDIA/cub/pull/170. +#if !defined(CUB_USE_COOPERATIVE_GROUPS) +#define CUB_USE_COOPERATIVE_GROUPS +#endif + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshadow" +#pragma GCC diagnostic ignored "-Wsuggest-override" + +#if defined(KOKKOS_COMPILER_CLANG) +// Some versions of Clang fail to compile Thrust, failing with errors like +// this: +// /thrust/system/cuda/detail/core/agent_launcher.h:557:11: +// error: use of undeclared identifier 'va_printf' +// The exact combination of versions for Clang and Thrust (or CUDA) for this +// failure was not investigated, however even very recent version combination +// (Clang 10.0.0 and Cuda 10.0) demonstrated failure. +// +// Defining _CubLog here locally allows us to avoid that code path, however +// disabling some debugging diagnostics +#pragma push_macro("_CubLog") +#ifdef _CubLog +#undef _CubLog +#endif +#define _CubLog +#include +#include +#pragma pop_macro("_CubLog") +#else +#include +#include +#endif + +#pragma GCC diagnostic pop + +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +#include +#include +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +#include +#include +#endif + +namespace Kokkos { +namespace Impl { + +template +struct better_off_calling_std_sort : std::false_type {}; + +#if defined KOKKOS_ENABLE_SERIAL +template <> +struct better_off_calling_std_sort : std::true_type {}; +#endif + +#if defined KOKKOS_ENABLE_OPENMP +template <> +struct better_off_calling_std_sort : std::true_type {}; +#endif + +#if defined KOKKOS_ENABLE_THREADS +template <> +struct better_off_calling_std_sort : std::true_type {}; +#endif + +#if defined KOKKOS_ENABLE_HPX +template <> +struct better_off_calling_std_sort : std::true_type { +}; +#endif + +template +inline constexpr bool better_off_calling_std_sort_v = + better_off_calling_std_sort::value; + +template +struct min_max_functor { + using minmax_scalar = + Kokkos::MinMaxScalar; + + ViewType view; + min_max_functor(const ViewType& view_) : view(view_) {} + + KOKKOS_INLINE_FUNCTION + void operator()(const size_t& i, minmax_scalar& minmax) const { + if (view(i) < minmax.min_val) minmax.min_val = view(i); + if (view(i) > minmax.max_val) minmax.max_val = view(i); + } +}; + +template +void sort_via_binsort(const ExecutionSpace& exec, + const Kokkos::View& view) { + // Although we are using BinSort below, which could work on rank-2 views, + // for now view must be rank-1 because the min_max_functor + // used below only works for rank-1 views + using ViewType = Kokkos::View; + static_assert(ViewType::rank == 1, + "Kokkos::sort: currently only supports rank-1 Views."); + + if (view.extent(0) <= 1) { + return; + } + + Kokkos::MinMaxScalar result; + Kokkos::MinMax reducer(result); + parallel_reduce("Kokkos::Sort::FindExtent", + Kokkos::RangePolicy( + exec, 0, view.extent(0)), + min_max_functor(view), reducer); + if (result.min_val == result.max_val) return; + // For integral types the number of bins may be larger than the range + // in which case we can exactly have one unique value per bin + // and then don't need to sort bins. + bool sort_in_bins = true; + // TODO: figure out better max_bins then this ... + int64_t max_bins = view.extent(0) / 2; + if (std::is_integral_v) { + // Cast to double to avoid possible overflow when using integer + auto const max_val = static_cast(result.max_val); + auto const min_val = static_cast(result.min_val); + // using 10M as the cutoff for special behavior (roughly 40MB for the count + // array) + if ((max_val - min_val) < 10000000) { + max_bins = max_val - min_val + 1; + sort_in_bins = false; + } + } + if (std::is_floating_point_v) { + KOKKOS_ASSERT(std::isfinite(static_cast(result.max_val) - + static_cast(result.min_val))); + } + + using CompType = BinOp1D; + BinSort bin_sort( + view, CompType(max_bins, result.min_val, result.max_val), sort_in_bins); + bin_sort.create_permute_vector(exec); + bin_sort.sort(exec, view); +} + +#if defined(KOKKOS_ENABLE_CUDA) +template +void sort_cudathrust(const Cuda& space, + const Kokkos::View& view, + MaybeComparator&&... maybeComparator) { + using ViewType = Kokkos::View; + static_assert(ViewType::rank == 1, + "Kokkos::sort: currently only supports rank-1 Views."); + + if (view.extent(0) <= 1) { + return; + } + const auto exec = thrust::cuda::par.on(space.cuda_stream()); + auto first = ::Kokkos::Experimental::begin(view); + auto last = ::Kokkos::Experimental::end(view); + thrust::sort(exec, first, last, + std::forward(maybeComparator)...); +} +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +template +void sort_rocthrust(const HIP& space, + const Kokkos::View& view, + MaybeComparator&&... maybeComparator) { + using ViewType = Kokkos::View; + static_assert(ViewType::rank == 1, + "Kokkos::sort: currently only supports rank-1 Views."); + + if (view.extent(0) <= 1) { + return; + } + const auto exec = thrust::hip::par.on(space.hip_stream()); + auto first = ::Kokkos::Experimental::begin(view); + auto last = ::Kokkos::Experimental::end(view); + thrust::sort(exec, first, last, + std::forward(maybeComparator)...); +} +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +template +void sort_onedpl(const Kokkos::SYCL& space, + const Kokkos::View& view, + MaybeComparator&&... maybeComparator) { + using ViewType = Kokkos::View; + static_assert(SpaceAccessibility::accessible, + "SYCL execution space is not able to access the memory space " + "of the View argument!"); + + static_assert( + (ViewType::rank == 1) && + (std::is_same_v || + std::is_same_v || + std::is_same_v), + "SYCL sort only supports contiguous rank-1 Views with LayoutLeft, " + "LayoutRight or LayoutStride" + "For the latter, this means the View must have stride(0) = 1, enforced " + "at runtime."); + + if (view.stride(0) != 1) { + Kokkos::abort("SYCL sort only supports rank-1 Views with stride(0) = 1."); + } + + if (view.extent(0) <= 1) { + return; + } + + // Can't use Experimental::begin/end here since the oneDPL then assumes that + // the data is on the host. + auto queue = space.sycl_queue(); + auto policy = oneapi::dpl::execution::make_device_policy(queue); + const int n = view.extent(0); + oneapi::dpl::sort(policy, view.data(), view.data() + n, + std::forward(maybeComparator)...); +} +#endif + +template +void copy_to_host_run_stdsort_copy_back( + const ExecutionSpace& exec, + const Kokkos::View& view, + MaybeComparator&&... maybeComparator) { + namespace KE = ::Kokkos::Experimental; + + using ViewType = Kokkos::View; + using layout = typename ViewType::array_layout; + if constexpr (std::is_same_v) { + // for strided views we cannot just deep_copy from device to host, + // so we need to do a few more jumps + using view_value_type = typename ViewType::non_const_value_type; + using view_exespace = typename ViewType::execution_space; + using view_deep_copyable_t = Kokkos::View; + view_deep_copyable_t view_dc("view_dc", view.extent(0)); + KE::copy(exec, view, view_dc); + + // run sort on the mirror of view_dc + auto mv_h = create_mirror_view_and_copy(Kokkos::HostSpace(), view_dc); + if (view.span_is_contiguous()) { + std::sort(mv_h.data(), mv_h.data() + mv_h.size(), + std::forward(maybeComparator)...); + } else { + auto first = KE::begin(mv_h); + auto last = KE::end(mv_h); + std::sort(first, last, std::forward(maybeComparator)...); + } + Kokkos::deep_copy(exec, view_dc, mv_h); + + // copy back to argument view + KE::copy(exec, KE::cbegin(view_dc), KE::cend(view_dc), KE::begin(view)); + } else { + auto view_h = create_mirror_view_and_copy(Kokkos::HostSpace(), view); + if (view.span_is_contiguous()) { + std::sort(view_h.data(), view_h.data() + view_h.size(), + std::forward(maybeComparator)...); + } else { + auto first = KE::begin(view_h); + auto last = KE::end(view_h); + std::sort(first, last, std::forward(maybeComparator)...); + } + Kokkos::deep_copy(exec, view, view_h); + } +} + +// -------------------------------------------------- +// +// specialize cases for sorting without comparator +// +// -------------------------------------------------- + +#if defined(KOKKOS_ENABLE_CUDA) +template +void sort_device_view_without_comparator( + const Cuda& exec, const Kokkos::View& view) { + sort_cudathrust(exec, view); +} +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +template +void sort_device_view_without_comparator( + const HIP& exec, const Kokkos::View& view) { + sort_rocthrust(exec, view); +} +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +template +void sort_device_view_without_comparator( + const Kokkos::SYCL& exec, + const Kokkos::View& view) { + using ViewType = Kokkos::View; + static_assert( + (ViewType::rank == 1) && + (std::is_same_v || + std::is_same_v || + std::is_same_v), + "sort_device_view_without_comparator: supports rank-1 Views " + "with LayoutLeft, LayoutRight or LayoutStride"); + + if (view.stride(0) == 1) { + sort_onedpl(exec, view); + } else { + copy_to_host_run_stdsort_copy_back(exec, view); + } +} +#endif + +// fallback case +template +std::enable_if_t::value> +sort_device_view_without_comparator( + const ExecutionSpace& exec, + const Kokkos::View& view) { + sort_via_binsort(exec, view); +} + +// -------------------------------------------------- +// +// specialize cases for sorting with comparator +// +// -------------------------------------------------- + +#if defined(KOKKOS_ENABLE_CUDA) +template +void sort_device_view_with_comparator( + const Cuda& exec, const Kokkos::View& view, + const ComparatorType& comparator) { + sort_cudathrust(exec, view, comparator); +} +#endif + +#if defined(KOKKOS_ENABLE_ROCTHRUST) +template +void sort_device_view_with_comparator( + const HIP& exec, const Kokkos::View& view, + const ComparatorType& comparator) { + sort_rocthrust(exec, view, comparator); +} +#endif + +#if defined(KOKKOS_ENABLE_ONEDPL) +template +void sort_device_view_with_comparator( + const Kokkos::SYCL& exec, const Kokkos::View& view, + const ComparatorType& comparator) { + using ViewType = Kokkos::View; + static_assert( + (ViewType::rank == 1) && + (std::is_same_v || + std::is_same_v || + std::is_same_v), + "sort_device_view_with_comparator: supports rank-1 Views " + "with LayoutLeft, LayoutRight or LayoutStride"); + + if (view.stride(0) == 1) { + sort_onedpl(exec, view, comparator); + } else { + copy_to_host_run_stdsort_copy_back(exec, view, comparator); + } +} +#endif + +template +std::enable_if_t::value> +sort_device_view_with_comparator( + const ExecutionSpace& exec, + const Kokkos::View& view, + const ComparatorType& comparator) { + // This is a fallback case if a more specialized overload does not exist: + // for now, this fallback copies data to host, runs std::sort + // and then copies data back. Potentially, this can later be changed + // with a better solution like our own quicksort on device or similar. + +// Note with HIP unified memory this code path is still the right thing to do +// if we end up here when RocThrust is not enabled. +// The create_mirror_view_and_copy will do the right thing (no copy). +#ifndef KOKKOS_IMPL_HIP_UNIFIED_MEMORY + using ViewType = Kokkos::View; + using MemSpace = typename ViewType::memory_space; + static_assert(!SpaceAccessibility::accessible, + "Impl::sort_device_view_with_comparator: should not be called " + "on a view that is already accessible on the host"); +#endif + + copy_to_host_run_stdsort_copy_back(exec, view, comparator); +} + +} // namespace Impl +} // namespace Kokkos +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AdjacentDifference.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AdjacentDifference.hpp new file mode 100644 index 0000000000..f254686dba --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AdjacentDifference.hpp @@ -0,0 +1,263 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_HPP +#define KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_HPP + +#include "impl/Kokkos_AdjacentDifference.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIteratorType, + typename OutputIteratorType, + std::enable_if_t::value && + ::Kokkos::is_execution_space::value, + int> = 0> +OutputIteratorType adjacent_difference(const ExecutionSpace& ex, + InputIteratorType first_from, + InputIteratorType last_from, + OutputIteratorType first_dest) { + using value_type1 = typename InputIteratorType::value_type; + using value_type2 = typename OutputIteratorType::value_type; + using binary_op = + Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; + return Impl::adjacent_difference_exespace_impl( + "Kokkos::adjacent_difference_iterator_api", ex, first_from, last_from, + first_dest, binary_op()); +} + +template < + typename ExecutionSpace, typename InputIteratorType, + typename OutputIteratorType, typename BinaryOp, + std::enable_if_t::value && + ::Kokkos::is_execution_space::value, + int> = 0> +OutputIteratorType adjacent_difference(const ExecutionSpace& ex, + InputIteratorType first_from, + InputIteratorType last_from, + OutputIteratorType first_dest, + BinaryOp bin_op) { + return Impl::adjacent_difference_exespace_impl( + "Kokkos::adjacent_difference_iterator_api", ex, first_from, last_from, + first_dest, bin_op); +} + +template < + typename ExecutionSpace, typename InputIteratorType, + typename OutputIteratorType, + std::enable_if_t::value && + ::Kokkos::is_execution_space::value, + int> = 0> +OutputIteratorType adjacent_difference(const std::string& label, + const ExecutionSpace& ex, + InputIteratorType first_from, + InputIteratorType last_from, + OutputIteratorType first_dest) { + using value_type1 = typename InputIteratorType::value_type; + using value_type2 = typename OutputIteratorType::value_type; + using binary_op = + Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; + return Impl::adjacent_difference_exespace_impl( + label, ex, first_from, last_from, first_dest, binary_op()); +} + +template < + typename ExecutionSpace, typename InputIteratorType, + typename OutputIteratorType, typename BinaryOp, + std::enable_if_t::value && + ::Kokkos::is_execution_space::value, + int> = 0> +OutputIteratorType adjacent_difference(const std::string& label, + const ExecutionSpace& ex, + InputIteratorType first_from, + InputIteratorType last_from, + OutputIteratorType first_dest, + BinaryOp bin_op) { + return Impl::adjacent_difference_exespace_impl(label, ex, first_from, + last_from, first_dest, bin_op); +} + +template ::value, + int> = 0> +auto adjacent_difference( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + using view_type1 = ::Kokkos::View; + using view_type2 = ::Kokkos::View; + using value_type1 = typename view_type1::value_type; + using value_type2 = typename view_type2::value_type; + using binary_op = + Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; + return Impl::adjacent_difference_exespace_impl( + "Kokkos::adjacent_difference_view_api", ex, KE::cbegin(view_from), + KE::cend(view_from), KE::begin(view_dest), binary_op()); +} + +template ::value, + int> = 0> +auto adjacent_difference( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp bin_op) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + return Impl::adjacent_difference_exespace_impl( + "Kokkos::adjacent_difference_view_api", ex, KE::cbegin(view_from), + KE::cend(view_from), KE::begin(view_dest), bin_op); +} + +template ::value, + int> = 0> +auto adjacent_difference( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + using view_type1 = ::Kokkos::View; + using view_type2 = ::Kokkos::View; + using value_type1 = typename view_type1::value_type; + using value_type2 = typename view_type2::value_type; + using binary_op = + Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; + + return Impl::adjacent_difference_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op()); +} + +template ::value, + int> = 0> +auto adjacent_difference( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp bin_op) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + return Impl::adjacent_difference_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), bin_op); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template ::value && + ::Kokkos::is_team_handle::value, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType adjacent_difference( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest) { + using value_type1 = typename InputIteratorType::value_type; + using value_type2 = typename OutputIteratorType::value_type; + using binary_op = + Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; + return Impl::adjacent_difference_team_impl(teamHandle, first_from, last_from, + first_dest, binary_op()); +} + +template ::value && + ::Kokkos::is_team_handle::value, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType +adjacent_difference(const TeamHandleType& teamHandle, + InputIteratorType first_from, InputIteratorType last_from, + OutputIteratorType first_dest, BinaryOp bin_op) { + return Impl::adjacent_difference_team_impl(teamHandle, first_from, last_from, + first_dest, bin_op); +} + +template < + typename TeamHandleType, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto adjacent_difference( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + using view_type1 = ::Kokkos::View; + using view_type2 = ::Kokkos::View; + using value_type1 = typename view_type1::value_type; + using value_type2 = typename view_type2::value_type; + using binary_op = + Impl::StdAdjacentDifferenceDefaultBinaryOpFunctor; + return Impl::adjacent_difference_team_impl(teamHandle, KE::cbegin(view_from), + KE::cend(view_from), + KE::begin(view_dest), binary_op()); +} + +template < + typename TeamHandleType, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOp, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto adjacent_difference( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp bin_op) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + return Impl::adjacent_difference_team_impl(teamHandle, KE::cbegin(view_from), + KE::cend(view_from), + KE::begin(view_dest), bin_op); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AdjacentFind.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AdjacentFind.hpp new file mode 100644 index 0000000000..ac476ca5bf --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AdjacentFind.hpp @@ -0,0 +1,169 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_HPP +#define KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_HPP + +#include "impl/Kokkos_AdjacentFind.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set1 +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t, int> = 0> +IteratorType adjacent_find(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::adjacent_find_exespace_impl( + "Kokkos::adjacent_find_iterator_api_default", ex, first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t, int> = 0> +IteratorType adjacent_find(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::adjacent_find_exespace_impl(label, ex, first, last); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t, int> = 0> +auto adjacent_find(const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::adjacent_find_exespace_impl( + "Kokkos::adjacent_find_view_api_default", ex, KE::begin(v), KE::end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t, int> = 0> +auto adjacent_find(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::adjacent_find_exespace_impl(label, ex, KE::begin(v), KE::end(v)); +} + +// overload set2 +template < + typename ExecutionSpace, typename IteratorType, + typename BinaryPredicateType, + std::enable_if_t, int> = 0> +IteratorType adjacent_find(const ExecutionSpace& ex, IteratorType first, + IteratorType last, BinaryPredicateType pred) { + return Impl::adjacent_find_exespace_impl( + "Kokkos::adjacent_find_iterator_api_default", ex, first, last, pred); +} + +template < + typename ExecutionSpace, typename IteratorType, + typename BinaryPredicateType, + std::enable_if_t, int> = 0> +IteratorType adjacent_find(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + BinaryPredicateType pred) { + return Impl::adjacent_find_exespace_impl(label, ex, first, last, pred); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename BinaryPredicateType, + std::enable_if_t, int> = 0> +auto adjacent_find(const ExecutionSpace& ex, + const ::Kokkos::View& v, + BinaryPredicateType pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::adjacent_find_exespace_impl( + "Kokkos::adjacent_find_view_api_default", ex, KE::begin(v), KE::end(v), + pred); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename BinaryPredicateType, + std::enable_if_t, int> = 0> +auto adjacent_find(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + BinaryPredicateType pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::adjacent_find_exespace_impl(label, ex, KE::begin(v), KE::end(v), + pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set1 +template , int> = 0> +KOKKOS_FUNCTION IteratorType adjacent_find(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last) { + return Impl::adjacent_find_team_impl(teamHandle, first, last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto adjacent_find( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::adjacent_find_team_impl(teamHandle, KE::begin(v), KE::end(v)); +} + +// overload set2 +template , int> = 0> +KOKKOS_FUNCTION IteratorType adjacent_find(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last, + BinaryPredicateType pred) { + return Impl::adjacent_find_team_impl(teamHandle, first, last, pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto adjacent_find( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + BinaryPredicateType pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::adjacent_find_team_impl(teamHandle, KE::begin(v), KE::end(v), + pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AllOf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AllOf.hpp new file mode 100644 index 0000000000..d6ed4c4a7e --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AllOf.hpp @@ -0,0 +1,104 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ALL_OF_HPP +#define KOKKOS_STD_ALGORITHMS_ALL_OF_HPP + +#include "impl/Kokkos_AllOfAnyOfNoneOf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool all_of(const ExecutionSpace& ex, InputIterator first, InputIterator last, + Predicate predicate) { + return Impl::all_of_exespace_impl("Kokkos::all_of_iterator_api_default", ex, + first, last, predicate); +} + +template < + typename ExecutionSpace, typename InputIterator, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool all_of(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, Predicate predicate) { + return Impl::all_of_exespace_impl(label, ex, first, last, predicate); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool all_of(const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::all_of_exespace_impl("Kokkos::all_of_view_api_default", ex, + KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool all_of(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::all_of_exespace_impl(label, ex, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION bool all_of(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + Predicate predicate) { + return Impl::all_of_team_impl(teamHandle, first, last, predicate); +} + +template , int> = 0> +KOKKOS_FUNCTION bool all_of(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::all_of_team_impl(teamHandle, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AnyOf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AnyOf.hpp new file mode 100644 index 0000000000..82356e6598 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_AnyOf.hpp @@ -0,0 +1,104 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ANY_OF_HPP +#define KOKKOS_STD_ALGORITHMS_ANY_OF_HPP + +#include "impl/Kokkos_AllOfAnyOfNoneOf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool any_of(const ExecutionSpace& ex, InputIterator first, InputIterator last, + Predicate predicate) { + return Impl::any_of_exespace_impl("Kokkos::any_of_view_api_default", ex, + first, last, predicate); +} + +template < + typename ExecutionSpace, typename InputIterator, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool any_of(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, Predicate predicate) { + return Impl::any_of_exespace_impl(label, ex, first, last, predicate); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool any_of(const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::any_of_exespace_impl("Kokkos::any_of_view_api_default", ex, + KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool any_of(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::any_of_exespace_impl(label, ex, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION bool any_of(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + Predicate predicate) { + return Impl::any_of_team_impl(teamHandle, first, last, predicate); +} + +template , int> = 0> +KOKKOS_FUNCTION bool any_of(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::any_of_team_impl(teamHandle, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_BeginEnd.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_BeginEnd.hpp similarity index 55% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_BeginEnd.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_BeginEnd.hpp index 5449196192..3836d4bf62 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/Kokkos_BeginEnd.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_BeginEnd.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_BEGIN_END_HPP #define KOKKOS_BEGIN_END_HPP diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Copy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Copy.hpp new file mode 100644 index 0000000000..c5406c72b0 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Copy.hpp @@ -0,0 +1,108 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_COPY_HPP + +#include "impl/Kokkos_CopyCopyN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator copy(const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first) { + return Impl::copy_exespace_impl("Kokkos::copy_iterator_api_default", ex, + first, last, d_first); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first) { + return Impl::copy_exespace_impl(label, ex, first, last, d_first); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + namespace KE = ::Kokkos::Experimental; + return Impl::copy_exespace_impl("Kokkos::copy_view_api_default", ex, + KE::cbegin(source), KE::cend(source), + KE::begin(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + namespace KE = ::Kokkos::Experimental; + return Impl::copy_exespace_impl(label, ex, KE::cbegin(source), + KE::cend(source), KE::begin(dest)); +} + +// +// overload set accepting team handle +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator copy(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + OutputIterator d_first) { + return Impl::copy_team_impl(teamHandle, first, last, d_first); +} + +template , int> = 0> +KOKKOS_FUNCTION auto copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + namespace KE = ::Kokkos::Experimental; + return Impl::copy_team_impl(teamHandle, KE::cbegin(source), KE::cend(source), + KE::begin(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyBackward.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyBackward.hpp new file mode 100644 index 0000000000..82071a9362 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyBackward.hpp @@ -0,0 +1,106 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_HPP +#define KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_HPP + +#include "impl/Kokkos_CopyBackward.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType2 copy_backward(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 d_last) { + return Impl::copy_backward_exespace_impl( + "Kokkos::copy_backward_iterator_api_default", ex, first, last, d_last); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType2 copy_backward(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 d_last) { + return Impl::copy_backward_exespace_impl(label, ex, first, last, d_last); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy_backward(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::copy_backward_exespace_impl( + "Kokkos::copy_backward_view_api_default", ex, cbegin(source), + cend(source), end(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy_backward(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::copy_backward_exespace_impl(label, ex, cbegin(source), + cend(source), end(dest)); +} + +// +// overload set accepting team handle +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType2 copy_backward(const TeamHandleType& teamHandle, + IteratorType1 first, + IteratorType1 last, + IteratorType2 d_last) { + return Impl::copy_backward_team_impl(teamHandle, first, last, d_last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto copy_backward( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::copy_backward_team_impl(teamHandle, cbegin(source), cend(source), + end(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyIf.hpp new file mode 100644 index 0000000000..599fde5737 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyIf.hpp @@ -0,0 +1,112 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COPY_IF_HPP +#define KOKKOS_STD_ALGORITHMS_COPY_IF_HPP + +#include "impl/Kokkos_CopyIf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator copy_if(const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first, + Predicate pred) { + return Impl::copy_if_exespace_impl("Kokkos::copy_if_iterator_api_default", ex, + first, last, d_first, std::move(pred)); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator copy_if(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first, Predicate pred) { + return Impl::copy_if_exespace_impl(label, ex, first, last, d_first, + std::move(pred)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy_if(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + Predicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::copy_if_exespace_impl("Kokkos::copy_if_view_api_default", ex, + cbegin(source), cend(source), begin(dest), + std::move(pred)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + Predicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::copy_if_exespace_impl(label, ex, cbegin(source), cend(source), + begin(dest), std::move(pred)); +} + +// +// overload set accepting team handle +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator copy_if(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + OutputIterator d_first, Predicate pred) { + return Impl::copy_if_team_impl(teamHandle, first, last, d_first, + std::move(pred)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto copy_if( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, Predicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::copy_if_team_impl(teamHandle, cbegin(source), cend(source), + begin(dest), std::move(pred)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyN.hpp new file mode 100644 index 0000000000..637d8d4cbc --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CopyN.hpp @@ -0,0 +1,108 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COPY_N_HPP +#define KOKKOS_STD_ALGORITHMS_COPY_N_HPP + +#include "impl/Kokkos_CopyCopyN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename Size, + typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator copy_n(const ExecutionSpace& ex, InputIterator first, Size count, + OutputIterator result) { + return Impl::copy_n_exespace_impl("Kokkos::copy_n_iterator_api_default", ex, + first, count, result); +} + +template < + typename ExecutionSpace, typename InputIterator, typename Size, + typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator copy_n(const std::string& label, const ExecutionSpace& ex, + InputIterator first, Size count, OutputIterator result) { + return Impl::copy_n_exespace_impl(label, ex, first, count, result); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename Size, typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy_n(const ExecutionSpace& ex, + const ::Kokkos::View& source, Size count, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + namespace KE = ::Kokkos::Experimental; + return Impl::copy_n_exespace_impl("Kokkos::copy_n_view_api_default", ex, + KE::cbegin(source), count, KE::begin(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename Size, typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto copy_n(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, Size count, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + namespace KE = ::Kokkos::Experimental; + return Impl::copy_n_exespace_impl(label, ex, KE::cbegin(source), count, + KE::begin(dest)); +} + +// +// overload set accepting team handle +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator copy_n(const TeamHandleType& teamHandle, + InputIterator first, Size count, + OutputIterator result) { + return Impl::copy_n_team_impl(teamHandle, first, count, result); +} + +template , int> = 0> +KOKKOS_FUNCTION auto copy_n( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, Size count, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + namespace KE = ::Kokkos::Experimental; + return Impl::copy_n_team_impl(teamHandle, KE::cbegin(source), count, + KE::begin(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Count.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Count.hpp new file mode 100644 index 0000000000..f179e88bab --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Count.hpp @@ -0,0 +1,106 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COUNT_HPP +#define KOKKOS_STD_ALGORITHMS_COUNT_HPP + +#include "impl/Kokkos_CountCountIf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +typename IteratorType::difference_type count(const ExecutionSpace& ex, + IteratorType first, + IteratorType last, + const T& value) { + return Impl::count_exespace_impl("Kokkos::count_iterator_api_default", ex, + first, last, value); +} + +template < + typename ExecutionSpace, typename IteratorType, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +typename IteratorType::difference_type count(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, + IteratorType last, + const T& value) { + return Impl::count_exespace_impl(label, ex, first, last, value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto count(const ExecutionSpace& ex, + const ::Kokkos::View& v, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::count_exespace_impl("Kokkos::count_view_api_default", ex, + KE::cbegin(v), KE::cend(v), value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto count(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::count_exespace_impl(label, ex, KE::cbegin(v), KE::cend(v), + value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +template , int> = 0> +KOKKOS_FUNCTION typename IteratorType::difference_type count( + const TeamHandleType& teamHandle, IteratorType first, IteratorType last, + const T& value) { + return Impl::count_team_impl(teamHandle, first, last, value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto count(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::count_team_impl(teamHandle, KE::cbegin(v), KE::cend(v), value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CountIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CountIf.hpp new file mode 100644 index 0000000000..967cf75e7a --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_CountIf.hpp @@ -0,0 +1,109 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COUNT_IF_HPP +#define KOKKOS_STD_ALGORITHMS_COUNT_IF_HPP + +#include "impl/Kokkos_CountCountIf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +typename IteratorType::difference_type count_if(const ExecutionSpace& ex, + IteratorType first, + IteratorType last, + Predicate predicate) { + return Impl::count_if_exespace_impl("Kokkos::count_if_iterator_api_default", + ex, first, last, std::move(predicate)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +typename IteratorType::difference_type count_if(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, + IteratorType last, + Predicate predicate) { + return Impl::count_if_exespace_impl(label, ex, first, last, + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto count_if(const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::count_if_exespace_impl("Kokkos::count_if_view_api_default", ex, + KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto count_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::count_if_exespace_impl(label, ex, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +// +// overload set accepting team handle +// +template , int> = 0> +KOKKOS_FUNCTION typename IteratorType::difference_type count_if( + const TeamHandleType& teamHandle, IteratorType first, IteratorType last, + Predicate predicate) { + return Impl::count_if_team_impl(teamHandle, first, last, + std::move(predicate)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto count_if(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::count_if_team_impl(teamHandle, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Distance.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Distance.hpp new file mode 100644 index 0000000000..8605606307 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Distance.hpp @@ -0,0 +1,41 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_DISTANCE_HPP +#define KOKKOS_STD_ALGORITHMS_DISTANCE_HPP + +#include "impl/Kokkos_Constraints.hpp" +#include "impl/Kokkos_RandomAccessIterator.hpp" + +namespace Kokkos { +namespace Experimental { + +template +KOKKOS_INLINE_FUNCTION constexpr typename IteratorType::difference_type +distance(IteratorType first, IteratorType last) { + static_assert( + ::Kokkos::Experimental::Impl::are_random_access_iterators< + IteratorType>::value, + "Kokkos::Experimental::distance: only implemented for random access " + "iterators."); + + return last - first; +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Equal.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Equal.hpp new file mode 100644 index 0000000000..593c42f87e --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Equal.hpp @@ -0,0 +1,285 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_EQUAL_HPP +#define KOKKOS_STD_ALGORITHMS_EQUAL_HPP + +#include "impl/Kokkos_Equal.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template && + Kokkos::is_execution_space_v, + int> = 0> +bool equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2) { + return Impl::equal_exespace_impl("Kokkos::equal_iterator_api_default", ex, + first1, last1, first2); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, IteratorType2 first2) { + return Impl::equal_exespace_impl(label, ex, first1, last1, first2); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, BinaryPredicateType predicate) { + return Impl::equal_exespace_impl("Kokkos::equal_iterator_api_default", ex, + first1, last1, first2, std::move(predicate)); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, + BinaryPredicateType predicate) { + return Impl::equal_exespace_impl(label, ex, first1, last1, first2, + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool equal(const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::equal_exespace_impl("Kokkos::equal_view_api_default", ex, + KE::cbegin(view1), KE::cend(view1), + KE::cbegin(view2)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool equal(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::equal_exespace_impl(label, ex, KE::cbegin(view1), + KE::cend(view1), KE::cbegin(view2)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool equal(const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + BinaryPredicateType predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::equal_exespace_impl("Kokkos::equal_view_api_default", ex, + KE::cbegin(view1), KE::cend(view1), + KE::cbegin(view2), std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool equal(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + BinaryPredicateType predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::equal_exespace_impl(label, ex, KE::cbegin(view1), + KE::cend(view1), KE::cbegin(view2), + std::move(predicate)); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2) { + return Impl::equal_exespace_impl("Kokkos::equal_iterator_api_default", ex, + first1, last1, first2, last2); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, + IteratorType2 last2) { + return Impl::equal_exespace_impl(label, ex, first1, last1, first2, last2); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType predicate) { + return Impl::equal_exespace_impl("Kokkos::equal_iterator_api_default", ex, + first1, last1, first2, last2, + std::move(predicate)); +} + +template && ::Kokkos:: + is_execution_space_v, + int> = 0> +bool equal(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, + IteratorType2 last2, BinaryPredicateType predicate) { + return Impl::equal_exespace_impl(label, ex, first1, last1, first2, last2, + std::move(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template && ::Kokkos:: + is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION bool equal(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2) { + return Impl::equal_team_impl(teamHandle, first1, last1, first2); +} + +template && ::Kokkos:: + is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION bool equal(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, + BinaryPredicateType predicate) { + return Impl::equal_team_impl(teamHandle, first1, last1, first2, + std::move(predicate)); +} + +template , int> = 0> +KOKKOS_FUNCTION bool equal( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::equal_team_impl(teamHandle, KE::cbegin(view1), KE::cend(view1), + KE::cbegin(view2)); +} + +template , int> = 0> +KOKKOS_FUNCTION bool equal( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + BinaryPredicateType predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::equal_team_impl(teamHandle, KE::cbegin(view1), KE::cend(view1), + KE::cbegin(view2), std::move(predicate)); +} + +template && ::Kokkos:: + is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION bool equal(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2) { + return Impl::equal_team_impl(teamHandle, first1, last1, first2, last2); +} + +template && ::Kokkos:: + is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION bool equal(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType predicate) { + return Impl::equal_team_impl(teamHandle, first1, last1, first2, last2, + std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ExclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ExclusiveScan.hpp new file mode 100644 index 0000000000..ee3a105126 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ExclusiveScan.hpp @@ -0,0 +1,269 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_HPP +#define KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_HPP + +#include "impl/Kokkos_ExclusiveScan.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1 +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType exclusive_scan(const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + ValueType init_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::exclusive_scan_default_op_exespace_impl( + "Kokkos::exclusive_scan_default_functors_iterator_api", ex, first, last, + first_dest, std::move(init_value)); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType exclusive_scan(const std::string& label, + const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + ValueType init_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::exclusive_scan_default_op_exespace_impl( + label, ex, first, last, first_dest, std::move(init_value)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto exclusive_scan(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::exclusive_scan_default_op_exespace_impl( + "Kokkos::exclusive_scan_default_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + std::move(init_value)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto exclusive_scan(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::exclusive_scan_default_op_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), std::move(init_value)); +} + +// overload set 2 +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType exclusive_scan(const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + ValueType init_value, BinaryOpType bop) { + Impl::static_assert_is_not_openmptarget(ex); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::exclusive_scan_custom_op_exespace_impl( + "Kokkos::exclusive_scan_custom_functors_iterator_api", ex, first, last, + first_dest, std::move(init_value), bop); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType exclusive_scan(const std::string& label, + const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + ValueType init_value, BinaryOpType bop) { + Impl::static_assert_is_not_openmptarget(ex); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::exclusive_scan_custom_op_exespace_impl( + label, ex, first, last, first_dest, std::move(init_value), bop); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + typename BinaryOpType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto exclusive_scan(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value, BinaryOpType bop) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::exclusive_scan_custom_op_exespace_impl( + "Kokkos::exclusive_scan_custom_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + std::move(init_value), bop); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + typename BinaryOpType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto exclusive_scan(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value, BinaryOpType bop) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::exclusive_scan_custom_op_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), std::move(init_value), bop); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1 +template && + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType +exclusive_scan(const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + ValueType init_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::exclusive_scan_default_op_team_impl( + teamHandle, first, last, first_dest, std::move(init_value)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto exclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::exclusive_scan_default_op_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), std::move(init_value)); +} + +// overload set 2 +template && + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType +exclusive_scan(const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + ValueType init_value, BinaryOpType bop) { + Impl::static_assert_is_not_openmptarget(teamHandle); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::exclusive_scan_custom_op_team_impl( + teamHandle, first, last, first_dest, std::move(init_value), bop); +} + +template , int> = 0> +KOKKOS_FUNCTION auto exclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value, BinaryOpType bop) { + Impl::static_assert_is_not_openmptarget(teamHandle); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::exclusive_scan_custom_op_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), std::move(init_value), bop); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Fill.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Fill.hpp new file mode 100644 index 0000000000..6d805ba1be --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Fill.hpp @@ -0,0 +1,92 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FILL_HPP +#define KOKKOS_STD_ALGORITHMS_FILL_HPP + +#include "impl/Kokkos_FillFillN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void fill(const ExecutionSpace& ex, IteratorType first, IteratorType last, + const T& value) { + Impl::fill_exespace_impl("Kokkos::fill_iterator_api_default", ex, first, last, + value); +} + +template < + typename ExecutionSpace, typename IteratorType, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void fill(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, const T& value) { + Impl::fill_exespace_impl(label, ex, first, last, value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void fill(const ExecutionSpace& ex, + const ::Kokkos::View& view, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::fill_exespace_impl("Kokkos::fill_view_api_default", ex, begin(view), + end(view), value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void fill(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::fill_exespace_impl(label, ex, begin(view), end(view), value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION void fill(const TeamHandleType& th, IteratorType first, + IteratorType last, const T& value) { + Impl::fill_team_impl(th, first, last, value); +} + +template , int> = 0> +KOKKOS_FUNCTION void fill(const TeamHandleType& th, + const ::Kokkos::View& view, + const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::fill_team_impl(th, begin(view), end(view), value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FillN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FillN.hpp new file mode 100644 index 0000000000..66b8cd66cc --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FillN.hpp @@ -0,0 +1,97 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FILL_N_HPP +#define KOKKOS_STD_ALGORITHMS_FILL_N_HPP + +#include "impl/Kokkos_FillFillN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +template < + typename ExecutionSpace, typename IteratorType, typename SizeType, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType fill_n(const ExecutionSpace& ex, IteratorType first, SizeType n, + const T& value) { + return Impl::fill_n_exespace_impl("Kokkos::fill_n_iterator_api_default", ex, + first, n, value); +} + +template < + typename ExecutionSpace, typename IteratorType, typename SizeType, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType fill_n(const std::string& label, const ExecutionSpace& ex, + IteratorType first, SizeType n, const T& value) { + return Impl::fill_n_exespace_impl(label, ex, first, n, value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename SizeType, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto fill_n(const ExecutionSpace& ex, + const ::Kokkos::View& view, SizeType n, + const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::fill_n_exespace_impl("Kokkos::fill_n_view_api_default", ex, + begin(view), n, value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename SizeType, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto fill_n(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, SizeType n, + const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::fill_n_exespace_impl(label, ex, begin(view), n, value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType fill_n(const TeamHandleType& th, + IteratorType first, SizeType n, + const T& value) { + return Impl::fill_n_team_impl(th, first, n, value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto fill_n(const TeamHandleType& th, + const ::Kokkos::View& view, + SizeType n, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::fill_n_team_impl(th, begin(view), n, value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Find.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Find.hpp new file mode 100644 index 0000000000..e5e2b0e2b0 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Find.hpp @@ -0,0 +1,101 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FIND_HPP +#define KOKKOS_STD_ALGORITHMS_FIND_HPP + +#include "impl/Kokkos_FindIfOrNot.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +InputIterator find(const ExecutionSpace& ex, InputIterator first, + InputIterator last, const T& value) { + return Impl::find_exespace_impl("Kokkos::find_iterator_api_default", ex, + first, last, value); +} + +template < + typename ExecutionSpace, typename InputIterator, typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +InputIterator find(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, const T& value) { + return Impl::find_exespace_impl(label, ex, first, last, value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find(const ExecutionSpace& ex, + const ::Kokkos::View& view, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_exespace_impl("Kokkos::find_view_api_default", ex, + KE::begin(view), KE::end(view), value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename T, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_exespace_impl(label, ex, KE::begin(view), KE::end(view), + value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION InputIterator find(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + const T& value) { + return Impl::find_team_impl(teamHandle, first, last, value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto find(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const T& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_team_impl(teamHandle, KE::begin(view), KE::end(view), + value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindEnd.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindEnd.hpp new file mode 100644 index 0000000000..a4ec735fd5 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindEnd.hpp @@ -0,0 +1,205 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FIND_END_HPP +#define KOKKOS_STD_ALGORITHMS_FIND_END_HPP + +#include "impl/Kokkos_FindEnd.hpp" +#include "Kokkos_Equal.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1: no binary predicate passed +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_end(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last) { + return Impl::find_end_exespace_impl("Kokkos::find_end_iterator_api_default", + ex, first, last, s_first, s_last); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_end(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last) { + return Impl::find_end_exespace_impl(label, ex, first, last, s_first, s_last); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_end(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_end_exespace_impl("Kokkos::find_end_view_api_default", ex, + KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_end(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_end_exespace_impl(label, ex, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view)); +} + +// overload set 2: binary predicate passed +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_end(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last, const BinaryPredicateType& pred) { + return Impl::find_end_exespace_impl("Kokkos::find_end_iterator_api_default", + ex, first, last, s_first, s_last, pred); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_end(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::find_end_exespace_impl(label, ex, first, last, s_first, s_last, + pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_end(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_end_exespace_impl("Kokkos::find_end_view_api_default", ex, + KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view), pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_end(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_end_exespace_impl(label, ex, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view), pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1: no binary predicate passed +template , int> = 0> +KOKKOS_FUNCTION IteratorType1 find_end(const TeamHandleType& teamHandle, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last) { + return Impl::find_end_team_impl(teamHandle, first, last, s_first, s_last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto find_end( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_end_team_impl(teamHandle, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view)); +} + +// overload set 2: binary predicate passed +template , int> = 0> + +KOKKOS_FUNCTION IteratorType1 find_end(const TeamHandleType& teamHandle, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::find_end_team_impl(teamHandle, first, last, s_first, s_last, + pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto find_end( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_end_team_impl(teamHandle, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view), pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindFirstOf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindFirstOf.hpp new file mode 100644 index 0000000000..341a70e2f2 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindFirstOf.hpp @@ -0,0 +1,215 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FIND_FIRST_OF_HPP +#define KOKKOS_STD_ALGORITHMS_FIND_FIRST_OF_HPP + +#include "impl/Kokkos_FindFirstOf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1: no binary predicate passed +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_first_of(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last) { + return Impl::find_first_of_exespace_impl( + "Kokkos::find_first_of_iterator_api_default", ex, first, last, s_first, + s_last); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_first_of(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last) { + return Impl::find_first_of_exespace_impl(label, ex, first, last, s_first, + s_last); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_first_of(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_first_of_exespace_impl( + "Kokkos::find_first_of_view_api_default", ex, KE::begin(view), + KE::end(view), KE::begin(s_view), KE::end(s_view)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_first_of(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_first_of_exespace_impl(label, ex, KE::begin(view), + KE::end(view), KE::begin(s_view), + KE::end(s_view)); +} + +// overload set 2: binary predicate passed +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_first_of(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::find_first_of_exespace_impl( + "Kokkos::find_first_of_iterator_api_default", ex, first, last, s_first, + s_last, pred); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 find_first_of(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::find_first_of_exespace_impl(label, ex, first, last, s_first, + s_last, pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_first_of(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_first_of_exespace_impl( + "Kokkos::find_first_of_view_api_default", ex, KE::begin(view), + KE::end(view), KE::begin(s_view), KE::end(s_view), pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto find_first_of(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_first_of_exespace_impl(label, ex, KE::begin(view), + KE::end(view), KE::begin(s_view), + KE::end(s_view), pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1: no binary predicate passed +template , int> = 0> +KOKKOS_FUNCTION IteratorType1 find_first_of(const TeamHandleType& teamHandle, + IteratorType1 first, + IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last) { + return Impl::find_first_of_team_impl(teamHandle, first, last, s_first, + s_last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto find_first_of( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_first_of_team_impl(teamHandle, KE::begin(view), + KE::end(view), KE::begin(s_view), + KE::end(s_view)); +} + +// overload set 2: binary predicate passed +template , int> = 0> + +KOKKOS_FUNCTION IteratorType1 find_first_of(const TeamHandleType& teamHandle, + IteratorType1 first, + IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::find_first_of_team_impl(teamHandle, first, last, s_first, s_last, + pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto find_first_of( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_first_of_team_impl(teamHandle, KE::begin(view), + KE::end(view), KE::begin(s_view), + KE::end(s_view), pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindIf.hpp new file mode 100644 index 0000000000..283fab7617 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindIf.hpp @@ -0,0 +1,107 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FIND_IF_HPP +#define KOKKOS_STD_ALGORITHMS_FIND_IF_HPP + +#include "impl/Kokkos_FindIfOrNot.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType find_if(const ExecutionSpace& ex, IteratorType first, + IteratorType last, PredicateType predicate) { + return Impl::find_if_or_not_exespace_impl( + "Kokkos::find_if_iterator_api_default", ex, first, last, + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType find_if(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + PredicateType predicate) { + return Impl::find_if_or_not_exespace_impl(label, ex, first, last, + std::move(predicate)); +} + +template ::value, + int> = 0> +auto find_if(const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::find_if_or_not_exespace_impl( + "Kokkos::find_if_view_api_default", ex, KE::begin(v), KE::end(v), + std::move(predicate)); +} + +template ::value, + int> = 0> +auto find_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::find_if_or_not_exespace_impl( + label, ex, KE::begin(v), KE::end(v), std::move(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType find_if(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + PredicateType predicate) { + return Impl::find_if_or_not_team_impl(teamHandle, first, last, + std::move(predicate)); +} + +template < + typename TeamHandleType, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto find_if(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + namespace KE = ::Kokkos::Experimental; + return Impl::find_if_or_not_team_impl(teamHandle, KE::begin(v), + KE::end(v), std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindIfNot.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindIfNot.hpp new file mode 100644 index 0000000000..5e17a6f539 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_FindIfNot.hpp @@ -0,0 +1,109 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FIND_IF_NOT_HPP +#define KOKKOS_STD_ALGORITHMS_FIND_IF_NOT_HPP + +#include "impl/Kokkos_FindIfOrNot.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType find_if_not(const ExecutionSpace& ex, IteratorType first, + IteratorType last, Predicate predicate) { + return Impl::find_if_or_not_exespace_impl( + "Kokkos::find_if_not_iterator_api_default", ex, first, last, + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType find_if_not(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + Predicate predicate) { + return Impl::find_if_or_not_exespace_impl(label, ex, first, last, + std::move(predicate)); +} + +template ::value, + int> = 0> +auto find_if_not(const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_if_or_not_exespace_impl( + "Kokkos::find_if_not_view_api_default", ex, KE::begin(v), KE::end(v), + std::move(predicate)); +} + +template ::value, + int> = 0> +auto find_if_not(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_if_or_not_exespace_impl( + label, ex, KE::begin(v), KE::end(v), std::move(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType find_if_not(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + Predicate predicate) { + return Impl::find_if_or_not_team_impl(teamHandle, first, last, + std::move(predicate)); +} + +template < + typename TeamHandleType, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto find_if_not( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::find_if_or_not_team_impl( + teamHandle, KE::begin(v), KE::end(v), std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ForEach.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ForEach.hpp new file mode 100644 index 0000000000..05969be463 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ForEach.hpp @@ -0,0 +1,104 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FOR_EACH_HPP +#define KOKKOS_STD_ALGORITHMS_FOR_EACH_HPP + +#include "impl/Kokkos_ForEachForEachN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + class ExecutionSpace, class IteratorType, class UnaryFunctorType, + std::enable_if_t, int> = 0> +void for_each(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, UnaryFunctorType functor) { + Impl::for_each_exespace_impl(label, ex, first, last, std::move(functor)); +} + +template < + class ExecutionSpace, class IteratorType, class UnaryFunctorType, + std::enable_if_t, int> = 0> +void for_each(const ExecutionSpace& ex, IteratorType first, IteratorType last, + UnaryFunctorType functor) { + Impl::for_each_exespace_impl("Kokkos::for_each_iterator_api_default", ex, + first, last, std::move(functor)); +} + +template < + class ExecutionSpace, class DataType, class... Properties, + class UnaryFunctorType, + std::enable_if_t, int> = 0> +void for_each(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + UnaryFunctorType functor) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + Impl::for_each_exespace_impl(label, ex, KE::begin(v), KE::end(v), + std::move(functor)); +} + +template < + class ExecutionSpace, class DataType, class... Properties, + class UnaryFunctorType, + std::enable_if_t, int> = 0> +void for_each(const ExecutionSpace& ex, + const ::Kokkos::View& v, + UnaryFunctorType functor) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + Impl::for_each_exespace_impl("Kokkos::for_each_view_api_default", ex, + KE::begin(v), KE::end(v), std::move(functor)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +template , int> = 0> +KOKKOS_FUNCTION void for_each(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + UnaryFunctorType functor) { + Impl::for_each_team_impl(teamHandle, first, last, std::move(functor)); +} + +template , int> = 0> +KOKKOS_FUNCTION void for_each(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + UnaryFunctorType functor) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + Impl::for_each_team_impl(teamHandle, KE::begin(v), KE::end(v), + std::move(functor)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ForEachN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ForEachN.hpp new file mode 100644 index 0000000000..e6fbcad891 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ForEachN.hpp @@ -0,0 +1,112 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FOR_EACH_N_HPP +#define KOKKOS_STD_ALGORITHMS_FOR_EACH_N_HPP + +#include "impl/Kokkos_ForEachForEachN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + class ExecutionSpace, class IteratorType, class SizeType, + class UnaryFunctorType, + std::enable_if_t, int> = 0> +IteratorType for_each_n(const std::string& label, const ExecutionSpace& ex, + IteratorType first, SizeType n, + UnaryFunctorType functor) { + return Impl::for_each_n_exespace_impl(label, ex, first, n, + std::move(functor)); +} + +template < + class ExecutionSpace, class IteratorType, class SizeType, + class UnaryFunctorType, + std::enable_if_t, int> = 0> +IteratorType for_each_n(const ExecutionSpace& ex, IteratorType first, + SizeType n, UnaryFunctorType functor) { + return Impl::for_each_n_exespace_impl( + "Kokkos::for_each_n_iterator_api_default", ex, first, n, + std::move(functor)); +} + +template < + class ExecutionSpace, class DataType, class... Properties, class SizeType, + class UnaryFunctorType, + std::enable_if_t, int> = 0> +auto for_each_n(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, SizeType n, + UnaryFunctorType functor) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::for_each_n_exespace_impl(label, ex, KE::begin(v), n, + std::move(functor)); +} + +template < + class ExecutionSpace, class DataType, class... Properties, class SizeType, + class UnaryFunctorType, + std::enable_if_t, int> = 0> +auto for_each_n(const ExecutionSpace& ex, + const ::Kokkos::View& v, SizeType n, + UnaryFunctorType functor) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::for_each_n_exespace_impl("Kokkos::for_each_n_view_api_default", + ex, KE::begin(v), n, + std::move(functor)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +template , int> = 0> +KOKKOS_FUNCTION IteratorType for_each_n(const TeamHandleType& teamHandle, + IteratorType first, SizeType n, + UnaryFunctorType functor) { + return Impl::for_each_n_team_impl(teamHandle, first, n, std::move(functor)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto for_each_n( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, SizeType n, + UnaryFunctorType functor) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::for_each_n_team_impl(teamHandle, KE::begin(v), n, + std::move(functor)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Generate.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Generate.hpp new file mode 100644 index 0000000000..a3295084ee --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Generate.hpp @@ -0,0 +1,93 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_GENERATE_HPP +#define KOKKOS_STD_ALGORITHMS_GENERATE_HPP + +#include "impl/Kokkos_GenerateGenerateN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template , int> = 0> +void generate(const ExecutionSpace& ex, IteratorType first, IteratorType last, + Generator g) { + Impl::generate_exespace_impl("Kokkos::generate_iterator_api_default", ex, + first, last, std::move(g)); +} + +template , int> = 0> +void generate(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, Generator g) { + Impl::generate_exespace_impl(label, ex, first, last, std::move(g)); +} + +template , int> = 0> +void generate(const ExecutionSpace& ex, + const ::Kokkos::View& view, + Generator g) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + Impl::generate_exespace_impl("Kokkos::generate_view_api_default", ex, + begin(view), end(view), std::move(g)); +} + +template , int> = 0> +void generate(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + Generator g) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + Impl::generate_exespace_impl(label, ex, begin(view), end(view), std::move(g)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION void generate(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + Generator g) { + Impl::generate_team_impl(teamHandle, first, last, std::move(g)); +} + +template , int> = 0> +KOKKOS_FUNCTION void generate( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, Generator g) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::generate_team_impl(teamHandle, begin(view), end(view), std::move(g)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_GenerateN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_GenerateN.hpp new file mode 100644 index 0000000000..e480062c23 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_GenerateN.hpp @@ -0,0 +1,100 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_GENERATE_N_HPP +#define KOKKOS_STD_ALGORITHMS_GENERATE_N_HPP + +#include "impl/Kokkos_GenerateGenerateN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template , int> = 0> +IteratorType generate_n(const ExecutionSpace& ex, IteratorType first, + Size count, Generator g) { + return Impl::generate_n_exespace_impl( + "Kokkos::generate_n_iterator_api_default", ex, first, count, + std::move(g)); +} + +template , int> = 0> +IteratorType generate_n(const std::string& label, const ExecutionSpace& ex, + IteratorType first, Size count, Generator g) { + return Impl::generate_n_exespace_impl(label, ex, first, count, std::move(g)); +} + +template , int> = 0> +auto generate_n(const ExecutionSpace& ex, + const ::Kokkos::View& view, Size count, + Generator g) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::generate_n_exespace_impl("Kokkos::generate_n_view_api_default", + ex, begin(view), count, std::move(g)); +} + +template , int> = 0> +auto generate_n(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, Size count, + Generator g) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::generate_n_exespace_impl(label, ex, begin(view), count, + std::move(g)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType generate_n(const TeamHandleType& teamHandle, + IteratorType first, Size count, + Generator g) { + return Impl::generate_n_team_impl(teamHandle, first, count, std::move(g)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto generate_n( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, Size count, + Generator g) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::generate_n_team_impl(teamHandle, begin(view), count, + std::move(g)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_InclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_InclusiveScan.hpp new file mode 100644 index 0000000000..a0e540b5e7 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_InclusiveScan.hpp @@ -0,0 +1,342 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_INCLUSIVE_SCAN_HPP +#define KOKKOS_STD_ALGORITHMS_INCLUSIVE_SCAN_HPP + +#include "impl/Kokkos_InclusiveScan.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1 +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType inclusive_scan(const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest) { + return Impl::inclusive_scan_default_op_exespace_impl( + "Kokkos::inclusive_scan_default_functors_iterator_api", ex, first, last, + first_dest); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType inclusive_scan(const std::string& label, + const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest) { + return Impl::inclusive_scan_default_op_exespace_impl(label, ex, first, last, + first_dest); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto inclusive_scan( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_default_op_exespace_impl( + "Kokkos::inclusive_scan_default_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto inclusive_scan( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_default_op_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest)); +} + +// overload set 2 (accepting custom binary op) +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType inclusive_scan(const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + BinaryOp binary_op) { + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + "Kokkos::inclusive_scan_custom_functors_iterator_api", ex, first, last, + first_dest, binary_op); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType inclusive_scan( + const std::string& label, const ExecutionSpace& ex, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, BinaryOp binary_op) { + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + label, ex, first, last, first_dest, binary_op); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOp, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto inclusive_scan(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp binary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + "Kokkos::inclusive_scan_custom_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + binary_op); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOp, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto inclusive_scan(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp binary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op); +} + +// overload set 3 +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType inclusive_scan(const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + BinaryOp binary_op, ValueType init_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + "Kokkos::inclusive_scan_custom_functors_iterator_api", ex, first, last, + first_dest, binary_op, std::move(init_value)); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType inclusive_scan(const std::string& label, + const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + BinaryOp binary_op, ValueType init_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + label, ex, first, last, first_dest, binary_op, std::move(init_value)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOp, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto inclusive_scan(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp binary_op, ValueType init_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + "Kokkos::inclusive_scan_custom_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + binary_op, std::move(init_value)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOp, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto inclusive_scan(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp binary_op, ValueType init_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_custom_binary_op_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op, std::move(init_value)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1 +template && :: + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType +inclusive_scan(const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest) { + return Impl::inclusive_scan_default_op_team_impl(teamHandle, first, last, + first_dest); +} + +template , int> = 0> +KOKKOS_FUNCTION auto inclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_default_op_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest)); +} + +// overload set 2 (accepting custom binary op) +template && :: + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType inclusive_scan( + const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, BinaryOp binary_op) { + return Impl::inclusive_scan_custom_binary_op_team_impl( + teamHandle, first, last, first_dest, binary_op); +} + +template , int> = 0> +KOKKOS_FUNCTION auto inclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp binary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_custom_binary_op_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op); +} + +// overload set 3 +template && :: + Kokkos::is_team_handle_v, + int> = 0> + +KOKKOS_FUNCTION OutputIteratorType +inclusive_scan(const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + BinaryOp binary_op, ValueType init_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::inclusive_scan_custom_binary_op_team_impl( + teamHandle, first, last, first_dest, binary_op, std::move(init_value)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto inclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOp binary_op, ValueType init_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + return Impl::inclusive_scan_custom_binary_op_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op, std::move(init_value)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsPartitioned.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsPartitioned.hpp new file mode 100644 index 0000000000..42f20bc4ec --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsPartitioned.hpp @@ -0,0 +1,103 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_IS_PARTITIONED_HPP +#define KOKKOS_STD_ALGORITHMS_IS_PARTITIONED_HPP + +#include "impl/Kokkos_IsPartitioned.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_partitioned(const ExecutionSpace& ex, IteratorType first, + IteratorType last, PredicateType p) { + return Impl::is_partitioned_exespace_impl( + "Kokkos::is_partitioned_iterator_api_default", ex, first, last, + std::move(p)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_partitioned(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, PredicateType p) { + return Impl::is_partitioned_exespace_impl(label, ex, first, last, + std::move(p)); +} + +template < + typename ExecutionSpace, typename PredicateType, typename DataType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_partitioned(const ExecutionSpace& ex, + const ::Kokkos::View& v, + PredicateType p) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::is_partitioned_exespace_impl( + "Kokkos::is_partitioned_view_api_default", ex, cbegin(v), cend(v), + std::move(p)); +} + +template < + typename ExecutionSpace, typename PredicateType, typename DataType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_partitioned(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + PredicateType p) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::is_partitioned_exespace_impl(label, ex, cbegin(v), cend(v), + std::move(p)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION bool is_partitioned(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + PredicateType p) { + return Impl::is_partitioned_team_impl(teamHandle, first, last, std::move(p)); +} + +template , int> = 0> +KOKKOS_FUNCTION bool is_partitioned( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, PredicateType p) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + return Impl::is_partitioned_team_impl(teamHandle, cbegin(v), cend(v), + std::move(p)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsSorted.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsSorted.hpp new file mode 100644 index 0000000000..2c676c3ff3 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsSorted.hpp @@ -0,0 +1,171 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_HPP +#define KOKKOS_STD_ALGORITHMS_IS_SORTED_HPP + +#include "impl/Kokkos_IsSorted.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::is_sorted_exespace_impl("Kokkos::is_sorted_iterator_api_default", + ex, first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::is_sorted_exespace_impl(label, ex, first, last); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_exespace_impl("Kokkos::is_sorted_view_api_default", ex, + KE::cbegin(view), KE::cend(view)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_exespace_impl(label, ex, KE::cbegin(view), + KE::cend(view)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const ExecutionSpace& ex, IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + return Impl::is_sorted_exespace_impl("Kokkos::is_sorted_iterator_api_default", + ex, first, last, std::move(comp)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + return Impl::is_sorted_exespace_impl(label, ex, first, last, std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const ExecutionSpace& ex, + const ::Kokkos::View& view, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_not_openmptarget(ex); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_exespace_impl("Kokkos::is_sorted_view_api_default", ex, + KE::cbegin(view), KE::cend(view), + std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool is_sorted(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_not_openmptarget(ex); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_exespace_impl(label, ex, KE::cbegin(view), + KE::cend(view), std::move(comp)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION bool is_sorted(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last) { + return Impl::is_sorted_team_impl(teamHandle, first, last); +} + +template , int> = 0> +KOKKOS_FUNCTION bool is_sorted( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_team_impl(teamHandle, KE::cbegin(view), + KE::cend(view)); +} + +template , int> = 0> +KOKKOS_FUNCTION bool is_sorted(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(teamHandle); + return Impl::is_sorted_team_impl(teamHandle, first, last, std::move(comp)); +} + +template , int> = 0> +KOKKOS_FUNCTION bool is_sorted( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_not_openmptarget(teamHandle); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_team_impl(teamHandle, KE::cbegin(view), KE::cend(view), + std::move(comp)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsSortedUntil.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsSortedUntil.hpp new file mode 100644 index 0000000000..96a17b6785 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IsSortedUntil.hpp @@ -0,0 +1,177 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_HPP +#define KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_HPP + +#include "impl/Kokkos_IsSortedUntil.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType is_sorted_until(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::is_sorted_until_exespace_impl( + "Kokkos::is_sorted_until_iterator_api_default", ex, first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType is_sorted_until(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::is_sorted_until_exespace_impl(label, ex, first, last); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto is_sorted_until(const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_until_exespace_impl( + "Kokkos::is_sorted_until_view_api_default", ex, KE::begin(view), + KE::end(view)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto is_sorted_until(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_until_exespace_impl(label, ex, KE::begin(view), + KE::end(view)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType is_sorted_until(const ExecutionSpace& ex, IteratorType first, + IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + return Impl::is_sorted_until_exespace_impl( + "Kokkos::is_sorted_until_iterator_api_default", ex, first, last, + std::move(comp)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType is_sorted_until(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::is_sorted_until_exespace_impl(label, ex, first, last, + std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto is_sorted_until(const ExecutionSpace& ex, + const ::Kokkos::View& view, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_not_openmptarget(ex); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_until_exespace_impl( + "Kokkos::is_sorted_until_view_api_default", ex, KE::begin(view), + KE::end(view), std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto is_sorted_until(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_not_openmptarget(ex); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_until_exespace_impl(label, ex, KE::begin(view), + KE::end(view), std::move(comp)); +} + +// +// overload set accepting team handle +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType is_sorted_until(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last) { + return Impl::is_sorted_until_team_impl(teamHandle, first, last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto is_sorted_until( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_until_team_impl(teamHandle, KE::begin(view), + KE::end(view)); +} + +template , int> = 0> +KOKKOS_FUNCTION IteratorType is_sorted_until(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(teamHandle); + return Impl::is_sorted_until_team_impl(teamHandle, first, last, + std::move(comp)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto is_sorted_until( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_not_openmptarget(teamHandle); + + namespace KE = ::Kokkos::Experimental; + return Impl::is_sorted_until_team_impl(teamHandle, KE::begin(view), + KE::end(view), std::move(comp)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IterSwap.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IterSwap.hpp new file mode 100644 index 0000000000..5bb2d1039d --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_IterSwap.hpp @@ -0,0 +1,73 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ITER_SWAP_HPP +#define KOKKOS_STD_ALGORITHMS_ITER_SWAP_HPP + +#include +#include "impl/Kokkos_Constraints.hpp" + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdIterSwapFunctor { + IteratorType1 m_a; + IteratorType2 m_b; + + KOKKOS_FUNCTION + void operator()(int i) const { + (void)i; + ::Kokkos::kokkos_swap(*m_a, *m_b); + } + + KOKKOS_FUNCTION + StdIterSwapFunctor(IteratorType1 _a, IteratorType2 _b) + : m_a(std::move(_a)), m_b(std::move(_b)) {} +}; + +template +void iter_swap_impl(IteratorType1 a, IteratorType2 b) { + // is there a better way to do this maybe? + ::Kokkos::parallel_for( + 1, StdIterSwapFunctor(a, b)); + Kokkos::DefaultExecutionSpace().fence( + "Kokkos::iter_swap: fence after operation"); +} +} // namespace Impl +//---------------------------------------------------------------------------- + +// iter_swap +template +void iter_swap(IteratorType1 a, IteratorType2 b) { + Impl::iter_swap_impl(a, b); +} + +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +template +KOKKOS_DEPRECATED_WITH_COMMENT("Use Kokkos::kokkos_swap instead!") +KOKKOS_FUNCTION + void swap(T& a, T& b) noexcept(::Kokkos::kokkos_swap(std::declval(), + std::declval())) { + ::Kokkos::kokkos_swap(a, b); +} +#endif + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_LexicographicalCompare.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_LexicographicalCompare.hpp new file mode 100644 index 0000000000..e13479c370 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_LexicographicalCompare.hpp @@ -0,0 +1,206 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_LEXICOGRAPHICAL_COMPARE_HPP +#define KOKKOS_STD_ALGORITHMS_LEXICOGRAPHICAL_COMPARE_HPP + +#include "impl/Kokkos_LexicographicalCompare.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + std::enable_if_t, int> = 0> +bool lexicographical_compare(const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, + IteratorType2 last2) { + return Impl::lexicographical_compare_exespace_impl( + "Kokkos::lexicographical_compare_iterator_api_default", ex, first1, last1, + first2, last2); +} + +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + std::enable_if_t, int> = 0> +bool lexicographical_compare(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2) { + return Impl::lexicographical_compare_exespace_impl(label, ex, first1, last1, + first2, last2); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, + std::enable_if_t, int> = 0> +bool lexicographical_compare( + const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::lexicographical_compare_exespace_impl( + "Kokkos::lexicographical_compare_view_api_default", ex, KE::cbegin(view1), + KE::cend(view1), KE::cbegin(view2), KE::cend(view2)); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, + std::enable_if_t, int> = 0> +bool lexicographical_compare( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::lexicographical_compare_exespace_impl( + label, ex, KE::cbegin(view1), KE::cend(view1), KE::cbegin(view2), + KE::cend(view2)); +} + +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + class ComparatorType, + std::enable_if_t, int> = 0> +bool lexicographical_compare(const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, + IteratorType2 last2, ComparatorType comp) { + return Impl::lexicographical_compare_exespace_impl( + "Kokkos::lexicographical_compare_iterator_api_default", ex, first1, last1, + first2, last2, comp); +} + +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + class ComparatorType, + std::enable_if_t, int> = 0> +bool lexicographical_compare(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + ComparatorType comp) { + return Impl::lexicographical_compare_exespace_impl(label, ex, first1, last1, + first2, last2, comp); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, class ComparatorType, + std::enable_if_t, int> = 0> +bool lexicographical_compare( + const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::lexicographical_compare_exespace_impl( + "Kokkos::lexicographical_compare_view_api_default", ex, KE::cbegin(view1), + KE::cend(view1), KE::cbegin(view2), KE::cend(view2), comp); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, class ComparatorType, + std::enable_if_t, int> = 0> +bool lexicographical_compare( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::lexicographical_compare_exespace_impl( + label, ex, KE::cbegin(view1), KE::cend(view1), KE::cbegin(view2), + KE::cend(view2), comp); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION bool lexicographical_compare(const TeamHandleType& teamHandle, + IteratorType1 first1, + IteratorType1 last1, + IteratorType2 first2, + IteratorType2 last2) { + return Impl::lexicographical_compare_team_impl(teamHandle, first1, last1, + first2, last2); +} + +template , int> = 0> +KOKKOS_FUNCTION bool lexicographical_compare( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::lexicographical_compare_team_impl( + teamHandle, KE::cbegin(view1), KE::cend(view1), KE::cbegin(view2), + KE::cend(view2)); +} + +template , int> = 0> +KOKKOS_FUNCTION bool lexicographical_compare( + const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, ComparatorType comp) { + return Impl::lexicographical_compare_team_impl(teamHandle, first1, last1, + first2, last2, comp); +} + +template , int> = 0> +KOKKOS_FUNCTION bool lexicographical_compare( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::lexicographical_compare_team_impl( + teamHandle, KE::cbegin(view1), KE::cend(view1), KE::cbegin(view2), + KE::cend(view2), comp); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MaxElement.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MaxElement.hpp new file mode 100644 index 0000000000..d16bac5bfc --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MaxElement.hpp @@ -0,0 +1,171 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_MAX_ELEMENT_HPP +#define KOKKOS_STD_ALGORITHMS_MAX_ELEMENT_HPP + +#include "impl/Kokkos_MinMaxMinmaxElement.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::min_or_max_element_exespace_impl( + "Kokkos::max_element_iterator_api_default", ex, first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::min_or_max_element_exespace_impl(label, ex, first, + last); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const ExecutionSpace& ex, IteratorType first, + IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + "Kokkos::max_element_iterator_api_default", ex, first, last, + std::move(comp)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + label, ex, first, last, std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::min_or_max_element_exespace_impl( + "Kokkos::max_element_view_api_default", ex, begin(v), end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::min_or_max_element_exespace_impl(label, ex, + begin(v), end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename ComparatorType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const ExecutionSpace& ex, + const ::Kokkos::View& v, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + "Kokkos::max_element_view_api_default", ex, begin(v), end(v), + std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename ComparatorType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto max_element(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + label, ex, begin(v), end(v), std::move(comp)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION auto max_element(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last) { + return Impl::min_or_max_element_team_impl(teamHandle, first, + last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto max_element( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::min_or_max_element_team_impl(teamHandle, begin(v), + end(v)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto max_element(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(teamHandle); + return Impl::min_or_max_element_team_impl( + teamHandle, first, last, std::move(comp)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto max_element( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(teamHandle); + return Impl::min_or_max_element_team_impl( + teamHandle, begin(v), end(v), std::move(comp)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MinElement.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MinElement.hpp new file mode 100644 index 0000000000..2a53fce3e2 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MinElement.hpp @@ -0,0 +1,171 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_MIN_ELEMENT_HPP +#define KOKKOS_STD_ALGORITHMS_MIN_ELEMENT_HPP + +#include "impl/Kokkos_MinMaxMinmaxElement.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::min_or_max_element_exespace_impl( + "Kokkos::min_element_iterator_api_default", ex, first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::min_or_max_element_exespace_impl(label, ex, first, + last); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const ExecutionSpace& ex, IteratorType first, + IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + "Kokkos::min_element_iterator_api_default", ex, first, last, + std::move(comp)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + label, ex, first, last, std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::min_or_max_element_exespace_impl( + "Kokkos::min_element_view_api_default", ex, begin(v), end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename ComparatorType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const ExecutionSpace& ex, + const ::Kokkos::View& v, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + "Kokkos::min_element_view_api_default", ex, begin(v), end(v), + std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::min_or_max_element_exespace_impl(label, ex, + begin(v), end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename ComparatorType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto min_element(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::min_or_max_element_exespace_impl( + label, ex, begin(v), end(v), std::move(comp)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION auto min_element(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last) { + return Impl::min_or_max_element_team_impl(teamHandle, first, + last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto min_element( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::min_or_max_element_team_impl(teamHandle, begin(v), + end(v)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto min_element(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(teamHandle); + return Impl::min_or_max_element_team_impl( + teamHandle, first, last, std::move(comp)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto min_element( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(teamHandle); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + return Impl::min_or_max_element_team_impl( + teamHandle, begin(v), end(v), std::move(comp)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MinMaxElement.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MinMaxElement.hpp new file mode 100644 index 0000000000..c3a1f73ef6 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MinMaxElement.hpp @@ -0,0 +1,174 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_MINMAX_ELEMENT_HPP +#define KOKKOS_STD_ALGORITHMS_MINMAX_ELEMENT_HPP + +#include "impl/Kokkos_MinMaxMinmaxElement.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::minmax_element_exespace_impl( + "Kokkos::minmax_element_iterator_api_default", ex, first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::minmax_element_exespace_impl(label, ex, + first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const ExecutionSpace& ex, IteratorType first, + IteratorType last, ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::minmax_element_exespace_impl( + "Kokkos::minmax_element_iterator_api_default", ex, first, last, + std::move(comp)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename ComparatorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::minmax_element_exespace_impl( + label, ex, first, last, std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::minmax_element_exespace_impl( + "Kokkos::minmax_element_view_api_default", ex, begin(v), end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::minmax_element_exespace_impl( + label, ex, begin(v), end(v)); +} + +template < + typename ExecutionSpace, typename DataType, typename ComparatorType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const ExecutionSpace& ex, + const ::Kokkos::View& v, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::minmax_element_exespace_impl( + "Kokkos::minmax_element_view_api_default", ex, begin(v), end(v), + std::move(comp)); +} + +template < + typename ExecutionSpace, typename DataType, typename ComparatorType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto minmax_element(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::minmax_element_exespace_impl( + label, ex, begin(v), end(v), std::move(comp)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION auto minmax_element(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last) { + return Impl::minmax_element_team_impl(teamHandle, first, + last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto minmax_element(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ComparatorType comp) { + Impl::static_assert_is_not_openmptarget(teamHandle); + + return Impl::minmax_element_team_impl( + teamHandle, first, last, std::move(comp)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto minmax_element( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + return Impl::minmax_element_team_impl(teamHandle, + begin(v), end(v)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto minmax_element( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, ComparatorType comp) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + Impl::static_assert_is_not_openmptarget(teamHandle); + + return Impl::minmax_element_team_impl( + teamHandle, begin(v), end(v), std::move(comp)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Mismatch.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Mismatch.hpp new file mode 100644 index 0000000000..090afe69e3 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Mismatch.hpp @@ -0,0 +1,209 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_MISMATCH_HPP +#define KOKKOS_STD_ALGORITHMS_MISMATCH_HPP + +#include "impl/Kokkos_Mismatch.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// FIXME: add mismatch overloads accepting 3 iterators. +// An overload consistent with other algorithms: +// +// auto mismatch(const ExecSpace& ex, It1 first1, It1 last1, It2 first2) {...} +// +// makes API ambiguous (with the overload accepting views). + +// +// overload set accepting execution space +// +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + std::enable_if_t, int> = 0> +::Kokkos::pair mismatch(const ExecutionSpace& ex, + IteratorType1 first1, + IteratorType1 last1, + IteratorType2 first2, + IteratorType2 last2) { + return Impl::mismatch_exespace_impl("Kokkos::mismatch_iterator_api_default", + ex, first1, last1, first2, last2); +} + +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + class BinaryPredicateType, + std::enable_if_t, int> = 0> +::Kokkos::pair mismatch( + const ExecutionSpace& ex, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType&& predicate) { + return Impl::mismatch_exespace_impl( + "Kokkos::mismatch_iterator_api_default", ex, first1, last1, first2, last2, + std::forward(predicate)); +} + +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + std::enable_if_t, int> = 0> +::Kokkos::pair mismatch( + const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, IteratorType2 last2) { + return Impl::mismatch_exespace_impl(label, ex, first1, last1, first2, last2); +} + +template < + class ExecutionSpace, class IteratorType1, class IteratorType2, + class BinaryPredicateType, + std::enable_if_t, int> = 0> +::Kokkos::pair mismatch( + const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType&& predicate) { + return Impl::mismatch_exespace_impl( + label, ex, first1, last1, first2, last2, + std::forward(predicate)); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, + std::enable_if_t, int> = 0> +auto mismatch(const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::mismatch_exespace_impl("Kokkos::mismatch_view_api_default", ex, + KE::begin(view1), KE::end(view1), + KE::begin(view2), KE::end(view2)); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, class BinaryPredicateType, + std::enable_if_t, int> = 0> +auto mismatch(const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + BinaryPredicateType&& predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::mismatch_exespace_impl( + "Kokkos::mismatch_view_api_default", ex, KE::begin(view1), KE::end(view1), + KE::begin(view2), KE::end(view2), + std::forward(predicate)); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, + std::enable_if_t, int> = 0> +auto mismatch(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::mismatch_exespace_impl(label, ex, KE::begin(view1), + KE::end(view1), KE::begin(view2), + KE::end(view2)); +} + +template < + class ExecutionSpace, class DataType1, class... Properties1, + class DataType2, class... Properties2, class BinaryPredicateType, + std::enable_if_t, int> = 0> +auto mismatch(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + BinaryPredicateType&& predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::mismatch_exespace_impl( + label, ex, KE::begin(view1), KE::end(view1), KE::begin(view2), + KE::end(view2), std::forward(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION ::Kokkos::pair mismatch( + const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2) { + return Impl::mismatch_team_impl(teamHandle, first1, last1, first2, last2); +} + +template , int> = 0> +KOKKOS_FUNCTION ::Kokkos::pair mismatch( + const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType&& predicate) { + return Impl::mismatch_team_impl(teamHandle, first1, last1, first2, last2, + std::forward(predicate)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto mismatch( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::mismatch_team_impl(teamHandle, KE::begin(view1), KE::end(view1), + KE::begin(view2), KE::end(view2)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto mismatch( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view1, + const ::Kokkos::View& view2, + BinaryPredicateType&& predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view2); + + namespace KE = ::Kokkos::Experimental; + return Impl::mismatch_team_impl(teamHandle, KE::begin(view1), KE::end(view1), + KE::begin(view2), KE::end(view2), + std::forward(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Move.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Move.hpp new file mode 100644 index 0000000000..ac308ea184 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Move.hpp @@ -0,0 +1,106 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_MOVE_HPP +#define KOKKOS_STD_ALGORITHMS_MOVE_HPP + +#include "impl/Kokkos_Move.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator move(const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first) { + return Impl::move_exespace_impl("Kokkos::move_iterator_api_default", ex, + first, last, d_first); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator move(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first) { + return Impl::move_exespace_impl(label, ex, first, last, d_first); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto move(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::move_exespace_impl("Kokkos::move_view_api_default", ex, + begin(source), end(source), begin(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto move(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::move_exespace_impl(label, ex, begin(source), end(source), + begin(dest)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator move(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + OutputIterator d_first) { + return Impl::move_team_impl(teamHandle, first, last, d_first); +} + +template , int> = 0> +KOKKOS_FUNCTION auto move( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::move_team_impl(teamHandle, begin(source), end(source), + begin(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MoveBackward.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MoveBackward.hpp new file mode 100644 index 0000000000..2789ab2179 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_MoveBackward.hpp @@ -0,0 +1,108 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_MOVE_BACKWARD_HPP +#define KOKKOS_STD_ALGORITHMS_MOVE_BACKWARD_HPP + +#include "impl/Kokkos_MoveBackward.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType2 move_backward(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 d_last) { + return Impl::move_backward_exespace_impl( + "Kokkos::move_backward_iterator_api_default", ex, first, last, d_last); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto move_backward(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::move_backward_exespace_impl( + "Kokkos::move_backward_view_api_default", ex, begin(source), end(source), + end(dest)); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType2 move_backward(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 d_last) { + return Impl::move_backward_exespace_impl(label, ex, first, last, d_last); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto move_backward(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::move_backward_exespace_impl(label, ex, begin(source), + end(source), end(dest)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType2 move_backward(const TeamHandleType& teamHandle, + IteratorType1 first, + IteratorType1 last, + IteratorType2 d_last) { + return Impl::move_backward_team_impl(teamHandle, first, last, d_last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto move_backward( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::move_backward_team_impl(teamHandle, begin(source), end(source), + end(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_NoneOf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_NoneOf.hpp new file mode 100644 index 0000000000..f7baab3fc0 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_NoneOf.hpp @@ -0,0 +1,105 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_NONE_OF_HPP +#define KOKKOS_STD_ALGORITHMS_NONE_OF_HPP + +#include "impl/Kokkos_AllOfAnyOfNoneOf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool none_of(const ExecutionSpace& ex, IteratorType first, IteratorType last, + Predicate predicate) { + return Impl::none_of_exespace_impl("Kokkos::none_of_iterator_api_default", ex, + first, last, predicate); +} + +template < + typename ExecutionSpace, typename IteratorType, typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool none_of(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, Predicate predicate) { + return Impl::none_of_exespace_impl(label, ex, first, last, predicate); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool none_of(const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::none_of_exespace_impl("Kokkos::none_of_view_api_default", ex, + KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename Predicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +bool none_of(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::none_of_exespace_impl(label, ex, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template +KOKKOS_FUNCTION + std::enable_if_t<::Kokkos::is_team_handle::value, bool> + none_of(const TeamHandleType& teamHandle, IteratorType first, + IteratorType last, Predicate predicate) { + return Impl::none_of_team_impl(teamHandle, first, last, predicate); +} + +template +KOKKOS_FUNCTION + std::enable_if_t<::Kokkos::is_team_handle::value, bool> + none_of(const TeamHandleType& teamHandle, + const ::Kokkos::View& v, + Predicate predicate) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + + namespace KE = ::Kokkos::Experimental; + return Impl::none_of_team_impl(teamHandle, KE::cbegin(v), KE::cend(v), + std::move(predicate)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_PartitionCopy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_PartitionCopy.hpp new file mode 100644 index 0000000000..a1feee8d6d --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_PartitionCopy.hpp @@ -0,0 +1,128 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_PARTITION_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_PARTITION_COPY_HPP + +#include "impl/Kokkos_PartitionCopy.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIteratorType, + typename OutputIteratorTrueType, typename OutputIteratorFalseType, + typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +::Kokkos::pair partition_copy( + const ExecutionSpace& ex, InputIteratorType from_first, + InputIteratorType from_last, OutputIteratorTrueType to_first_true, + OutputIteratorFalseType to_first_false, PredicateType p) { + return Impl::partition_copy_exespace_impl( + "Kokkos::partition_copy_iterator_api_default", ex, from_first, from_last, + to_first_true, to_first_false, std::move(p)); +} + +template < + typename ExecutionSpace, typename InputIteratorType, + typename OutputIteratorTrueType, typename OutputIteratorFalseType, + typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +::Kokkos::pair partition_copy( + const std::string& label, const ExecutionSpace& ex, + InputIteratorType from_first, InputIteratorType from_last, + OutputIteratorTrueType to_first_true, + OutputIteratorFalseType to_first_false, PredicateType p) { + return Impl::partition_copy_exespace_impl(label, ex, from_first, from_last, + to_first_true, to_first_false, + std::move(p)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename DataType3, + typename... Properties3, typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto partition_copy( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest_true, + const ::Kokkos::View& view_dest_false, + PredicateType p) { + return Impl::partition_copy_exespace_impl( + "Kokkos::partition_copy_view_api_default", ex, cbegin(view_from), + cend(view_from), begin(view_dest_true), begin(view_dest_false), + std::move(p)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename DataType3, + typename... Properties3, typename PredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto partition_copy( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest_true, + const ::Kokkos::View& view_dest_false, + PredicateType p) { + return Impl::partition_copy_exespace_impl( + label, ex, cbegin(view_from), cend(view_from), begin(view_dest_true), + begin(view_dest_false), std::move(p)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION ::Kokkos::pair +partition_copy(const TeamHandleType& teamHandle, InputIteratorType from_first, + InputIteratorType from_last, + OutputIteratorTrueType to_first_true, + OutputIteratorFalseType to_first_false, PredicateType p) { + return Impl::partition_copy_team_impl(teamHandle, from_first, from_last, + to_first_true, to_first_false, + std::move(p)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto partition_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest_true, + const ::Kokkos::View& view_dest_false, + PredicateType p) { + return Impl::partition_copy_team_impl(teamHandle, cbegin(view_from), + cend(view_from), begin(view_dest_true), + begin(view_dest_false), std::move(p)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_PartitionPoint.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_PartitionPoint.hpp new file mode 100644 index 0000000000..60cbeeda87 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_PartitionPoint.hpp @@ -0,0 +1,103 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_PARTITION_POINT_HPP +#define KOKKOS_STD_ALGORITHMS_PARTITION_POINT_HPP + +#include "impl/Kokkos_PartitionPoint.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType partition_point(const ExecutionSpace& ex, IteratorType first, + IteratorType last, UnaryPredicate p) { + return Impl::partition_point_exespace_impl( + "Kokkos::partitioned_point_iterator_api_default", ex, first, last, + std::move(p)); +} + +template < + typename ExecutionSpace, typename IteratorType, typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType partition_point(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + UnaryPredicate p) { + return Impl::partition_point_exespace_impl(label, ex, first, last, + std::move(p)); +} + +template < + typename ExecutionSpace, typename UnaryPredicate, typename DataType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto partition_point(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& v, + UnaryPredicate p) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + return Impl::partition_point_exespace_impl(label, ex, begin(v), end(v), + std::move(p)); +} + +template < + typename ExecutionSpace, typename UnaryPredicate, typename DataType, + typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto partition_point(const ExecutionSpace& ex, + const ::Kokkos::View& v, + UnaryPredicate p) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + return Impl::partition_point_exespace_impl( + "Kokkos::partition_point_view_api_default", ex, begin(v), end(v), + std::move(p)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType partition_point(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last, + UnaryPredicate p) { + return Impl::partition_point_team_impl(teamHandle, first, last, std::move(p)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto partition_point( + const TeamHandleType& teamHandle, + const ::Kokkos::View& v, UnaryPredicate p) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(v); + return Impl::partition_point_team_impl(teamHandle, begin(v), end(v), + std::move(p)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Reduce.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Reduce.hpp new file mode 100644 index 0000000000..ea7e55ca61 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Reduce.hpp @@ -0,0 +1,326 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REDUCE_HPP +#define KOKKOS_STD_ALGORITHMS_REDUCE_HPP + +#include "impl/Kokkos_Reduce.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// +// overload set 1 +// +template ::value, + int> = 0> +typename IteratorType::value_type reduce(const ExecutionSpace& ex, + IteratorType first, + IteratorType last) { + return Impl::reduce_default_functors_exespace_impl( + "Kokkos::reduce_default_functors_iterator_api", ex, first, last, + typename IteratorType::value_type()); +} + +template ::value, + int> = 0> +typename IteratorType::value_type reduce(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, + IteratorType last) { + return Impl::reduce_default_functors_exespace_impl( + label, ex, first, last, typename IteratorType::value_type()); +} + +template ::value, + int> = 0> +auto reduce(const ExecutionSpace& ex, + const ::Kokkos::View& view) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + using view_type = ::Kokkos::View; + using value_type = typename view_type::value_type; + + return Impl::reduce_default_functors_exespace_impl( + "Kokkos::reduce_default_functors_view_api", ex, KE::cbegin(view), + KE::cend(view), value_type()); +} + +template ::value, + int> = 0> +auto reduce(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + using view_type = ::Kokkos::View; + using value_type = typename view_type::value_type; + + return Impl::reduce_default_functors_exespace_impl( + label, ex, KE::cbegin(view), KE::cend(view), value_type()); +} + +// +// overload set2: +// +template ::value, + int> = 0> +ValueType reduce(const ExecutionSpace& ex, IteratorType first, + IteratorType last, ValueType init_reduction_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::reduce_default_functors_exespace_impl( + "Kokkos::reduce_default_functors_iterator_api", ex, first, last, + init_reduction_value); +} + +template ::value, + int> = 0> +ValueType reduce(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + ValueType init_reduction_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::reduce_default_functors_exespace_impl(label, ex, first, last, + init_reduction_value); +} + +template ::value, + int> = 0> +ValueType reduce(const ExecutionSpace& ex, + const ::Kokkos::View& view, + ValueType init_reduction_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::reduce_default_functors_exespace_impl( + "Kokkos::reduce_default_functors_view_api", ex, KE::cbegin(view), + KE::cend(view), init_reduction_value); +} + +template ::value, + int> = 0> +ValueType reduce(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + ValueType init_reduction_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::reduce_default_functors_exespace_impl( + label, ex, KE::cbegin(view), KE::cend(view), init_reduction_value); +} + +// +// overload set 3 +// +template ::value, + int> = 0> +ValueType reduce(const ExecutionSpace& ex, IteratorType first, + IteratorType last, ValueType init_reduction_value, + BinaryOp joiner) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::reduce_custom_functors_exespace_impl( + "Kokkos::reduce_default_functors_iterator_api", ex, first, last, + init_reduction_value, joiner); +} + +template ::value, + int> = 0> +ValueType reduce(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + ValueType init_reduction_value, BinaryOp joiner) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::reduce_custom_functors_exespace_impl( + label, ex, first, last, init_reduction_value, joiner); +} + +template ::value, + int> = 0> +ValueType reduce(const ExecutionSpace& ex, + const ::Kokkos::View& view, + ValueType init_reduction_value, BinaryOp joiner) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::reduce_custom_functors_exespace_impl( + "Kokkos::reduce_custom_functors_view_api", ex, KE::cbegin(view), + KE::cend(view), init_reduction_value, joiner); +} + +template ::value, + int> = 0> +ValueType reduce(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + ValueType init_reduction_value, BinaryOp joiner) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::reduce_custom_functors_exespace_impl( + label, ex, KE::cbegin(view), KE::cend(view), init_reduction_value, + joiner); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// +// overload set 1 +// +template < + typename TeamHandleType, typename IteratorType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION typename IteratorType::value_type reduce( + const TeamHandleType& teamHandle, IteratorType first, IteratorType last) { + return Impl::reduce_default_functors_team_impl( + teamHandle, first, last, typename IteratorType::value_type()); +} + +template < + typename TeamHandleType, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto reduce( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + using view_type = ::Kokkos::View; + using value_type = typename view_type::value_type; + + return Impl::reduce_default_functors_team_impl(teamHandle, KE::cbegin(view), + KE::cend(view), value_type()); +} + +// +// overload set2: +// +template < + typename TeamHandleType, typename IteratorType, typename ValueType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType reduce(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ValueType init_reduction_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::reduce_default_functors_team_impl(teamHandle, first, last, + init_reduction_value); +} + +template < + typename TeamHandleType, typename DataType, typename... Properties, + typename ValueType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType +reduce(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + ValueType init_reduction_value) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::reduce_default_functors_team_impl( + teamHandle, KE::cbegin(view), KE::cend(view), init_reduction_value); +} + +// +// overload set 3 +// +template < + typename TeamHandleType, typename IteratorType, typename ValueType, + typename BinaryOp, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType reduce(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ValueType init_reduction_value, + BinaryOp joiner) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::reduce_custom_functors_team_impl(teamHandle, first, last, + init_reduction_value, joiner); +} + +template < + typename TeamHandleType, typename DataType, typename... Properties, + typename ValueType, typename BinaryOp, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType +reduce(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + ValueType init_reduction_value, BinaryOp joiner) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::reduce_custom_functors_team_impl(teamHandle, KE::cbegin(view), + KE::cend(view), + init_reduction_value, joiner); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Remove.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Remove.hpp new file mode 100644 index 0000000000..8a429d8d51 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Remove.hpp @@ -0,0 +1,99 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_HPP +#define KOKKOS_STD_ALGORITHMS_REMOVE_HPP + +#include "impl/Kokkos_RemoveAllVariants.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename Iterator, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +Iterator remove(const ExecutionSpace& ex, Iterator first, Iterator last, + const ValueType& value) { + return Impl::remove_exespace_impl("Kokkos::remove_iterator_api_default", ex, + first, last, value); +} + +template < + typename ExecutionSpace, typename Iterator, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +Iterator remove(const std::string& label, const ExecutionSpace& ex, + Iterator first, Iterator last, const ValueType& value) { + return Impl::remove_exespace_impl(label, ex, first, last, value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::remove_exespace_impl("Kokkos::remove_iterator_api_default", ex, + ::Kokkos::Experimental::begin(view), + ::Kokkos::Experimental::end(view), value); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::remove_exespace_impl(label, ex, + ::Kokkos::Experimental::begin(view), + ::Kokkos::Experimental::end(view), value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION Iterator remove(const TeamHandleType& teamHandle, + Iterator first, Iterator last, + const ValueType& value) { + return Impl::remove_team_impl(teamHandle, first, last, value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto remove(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::remove_team_impl(teamHandle, ::Kokkos::Experimental::begin(view), + ::Kokkos::Experimental::end(view), value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveCopy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveCopy.hpp new file mode 100644 index 0000000000..4b8fa9fe07 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveCopy.hpp @@ -0,0 +1,124 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_REMOVE_COPY_HPP + +#include "impl/Kokkos_RemoveAllVariants.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator remove_copy(const ExecutionSpace& ex, InputIterator first_from, + InputIterator last_from, OutputIterator first_dest, + const ValueType& value) { + return Impl::remove_copy_exespace_impl( + "Kokkos::remove_copy_iterator_api_default", ex, first_from, last_from, + first_dest, value); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator remove_copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first_from, InputIterator last_from, + OutputIterator first_dest, const ValueType& value) { + return Impl::remove_copy_exespace_impl(label, ex, first_from, last_from, + first_dest, value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove_copy(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + return Impl::remove_copy_exespace_impl( + "Kokkos::remove_copy_iterator_api_default", ex, + ::Kokkos::Experimental::cbegin(view_from), + ::Kokkos::Experimental::cend(view_from), + ::Kokkos::Experimental::begin(view_dest), value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove_copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + return Impl::remove_copy_exespace_impl( + label, ex, ::Kokkos::Experimental::cbegin(view_from), + ::Kokkos::Experimental::cend(view_from), + ::Kokkos::Experimental::begin(view_dest), value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator remove_copy(const TeamHandleType& teamHandle, + InputIterator first_from, + InputIterator last_from, + OutputIterator first_dest, + const ValueType& value) { + return Impl::remove_copy_team_impl(teamHandle, first_from, last_from, + first_dest, value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto remove_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + return Impl::remove_copy_team_impl( + teamHandle, ::Kokkos::Experimental::cbegin(view_from), + ::Kokkos::Experimental::cend(view_from), + ::Kokkos::Experimental::begin(view_dest), value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveCopyIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveCopyIf.hpp new file mode 100644 index 0000000000..45e2b54bb6 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveCopyIf.hpp @@ -0,0 +1,127 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_COPY_IF_HPP +#define KOKKOS_STD_ALGORITHMS_REMOVE_COPY_IF_HPP + +#include "impl/Kokkos_RemoveAllVariants.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator remove_copy_if(const ExecutionSpace& ex, + InputIterator first_from, InputIterator last_from, + OutputIterator first_dest, + const UnaryPredicate& pred) { + return Impl::remove_copy_if_exespace_impl( + "Kokkos::remove_copy_if_iterator_api_default", ex, first_from, last_from, + first_dest, pred); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator remove_copy_if(const std::string& label, + const ExecutionSpace& ex, + InputIterator first_from, InputIterator last_from, + OutputIterator first_dest, + const UnaryPredicate& pred) { + return Impl::remove_copy_if_exespace_impl(label, ex, first_from, last_from, + first_dest, pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove_copy_if(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const UnaryPredicate& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + return Impl::remove_copy_if_exespace_impl( + "Kokkos::remove_copy_if_iterator_api_default", ex, + ::Kokkos::Experimental::cbegin(view_from), + ::Kokkos::Experimental::cend(view_from), + ::Kokkos::Experimental::begin(view_dest), pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove_copy_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const UnaryPredicate& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + return Impl::remove_copy_if_exespace_impl( + label, ex, ::Kokkos::Experimental::cbegin(view_from), + ::Kokkos::Experimental::cend(view_from), + ::Kokkos::Experimental::begin(view_dest), pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator remove_copy_if(const TeamHandleType& teamHandle, + InputIterator first_from, + InputIterator last_from, + OutputIterator first_dest, + const UnaryPredicate& pred) { + return Impl::remove_copy_if_team_impl(teamHandle, first_from, last_from, + first_dest, pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto remove_copy_if( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const UnaryPredicate& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + + return Impl::remove_copy_if_team_impl( + teamHandle, ::Kokkos::Experimental::cbegin(view_from), + ::Kokkos::Experimental::cend(view_from), + ::Kokkos::Experimental::begin(view_dest), pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveIf.hpp new file mode 100644 index 0000000000..38461a37f2 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RemoveIf.hpp @@ -0,0 +1,102 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REMOVE_IF_HPP +#define KOKKOS_STD_ALGORITHMS_REMOVE_IF_HPP + +#include "impl/Kokkos_RemoveAllVariants.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename Iterator, typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +Iterator remove_if(const ExecutionSpace& ex, Iterator first, Iterator last, + UnaryPredicate pred) { + return Impl::remove_if_exespace_impl("Kokkos::remove_if_iterator_api_default", + ex, first, last, pred); +} + +template < + typename ExecutionSpace, typename Iterator, typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +Iterator remove_if(const std::string& label, const ExecutionSpace& ex, + Iterator first, Iterator last, UnaryPredicate pred) { + return Impl::remove_if_exespace_impl(label, ex, first, last, pred); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove_if(const ExecutionSpace& ex, + const ::Kokkos::View& view, + UnaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::remove_if_exespace_impl("Kokkos::remove_if_iterator_api_default", + ex, ::Kokkos::Experimental::begin(view), + ::Kokkos::Experimental::end(view), pred); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename UnaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto remove_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + UnaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::remove_if_exespace_impl(label, ex, + ::Kokkos::Experimental::begin(view), + ::Kokkos::Experimental::end(view), pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION Iterator remove_if(const TeamHandleType& teamHandle, + Iterator first, Iterator last, + UnaryPredicate pred) { + return Impl::remove_if_team_impl(teamHandle, first, last, pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto remove_if( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, UnaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::remove_if_team_impl(teamHandle, + ::Kokkos::Experimental::begin(view), + ::Kokkos::Experimental::end(view), pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Replace.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Replace.hpp new file mode 100644 index 0000000000..29afc4f0c2 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Replace.hpp @@ -0,0 +1,102 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_HPP +#define KOKKOS_STD_ALGORITHMS_REPLACE_HPP + +#include "impl/Kokkos_Replace.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename Iterator, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace(const ExecutionSpace& ex, Iterator first, Iterator last, + const ValueType& old_value, const ValueType& new_value) { + Impl::replace_exespace_impl("Kokkos::replace_iterator_api", ex, first, last, + old_value, new_value); +} + +template < + typename ExecutionSpace, typename Iterator, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace(const std::string& label, const ExecutionSpace& ex, Iterator first, + Iterator last, const ValueType& old_value, + const ValueType& new_value) { + Impl::replace_exespace_impl(label, ex, first, last, old_value, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ValueType& old_value, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + Impl::replace_exespace_impl("Kokkos::replace_view_api", ex, KE::begin(view), + KE::end(view), old_value, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ValueType& old_value, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + Impl::replace_exespace_impl(label, ex, KE::begin(view), KE::end(view), + old_value, new_value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION void replace(const TeamHandleType& teamHandle, Iterator first, + Iterator last, const ValueType& old_value, + const ValueType& new_value) { + Impl::replace_team_impl(teamHandle, first, last, old_value, new_value); +} + +template , int> = 0> +KOKKOS_FUNCTION void replace( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ValueType& old_value, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + Impl::replace_team_impl(teamHandle, KE::begin(view), KE::end(view), old_value, + new_value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceCopy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceCopy.hpp new file mode 100644 index 0000000000..04d5767e89 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceCopy.hpp @@ -0,0 +1,124 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_REPLACE_COPY_HPP + +#include "impl/Kokkos_ReplaceCopy.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator replace_copy(const ExecutionSpace& ex, InputIterator first_from, + InputIterator last_from, OutputIterator first_dest, + const ValueType& old_value, + const ValueType& new_value) { + return Impl::replace_copy_exespace_impl("Kokkos::replace_copy_iterator_api", + ex, first_from, last_from, first_dest, + old_value, new_value); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator replace_copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first_from, InputIterator last_from, + OutputIterator first_dest, + const ValueType& old_value, + const ValueType& new_value) { + return Impl::replace_copy_exespace_impl(label, ex, first_from, last_from, + first_dest, old_value, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto replace_copy(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const ValueType& old_value, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::replace_copy_exespace_impl( + "Kokkos::replace_copy_view_api", ex, KE::cbegin(view_from), + KE::cend(view_from), KE::begin(view_dest), old_value, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto replace_copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const ValueType& old_value, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::replace_copy_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), old_value, new_value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator replace_copy(const TeamHandleType& teamHandle, + InputIterator first_from, + InputIterator last_from, + OutputIterator first_dest, + const ValueType& old_value, + const ValueType& new_value) { + return Impl::replace_copy_team_impl(teamHandle, first_from, last_from, + first_dest, old_value, new_value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto replace_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + const ValueType& old_value, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::replace_copy_team_impl(teamHandle, KE::cbegin(view_from), + KE::cend(view_from), KE::begin(view_dest), + old_value, new_value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceCopyIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceCopyIf.hpp new file mode 100644 index 0000000000..b87163f194 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceCopyIf.hpp @@ -0,0 +1,129 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IF_HPP +#define KOKKOS_STD_ALGORITHMS_REPLACE_COPY_IF_HPP + +#include "impl/Kokkos_ReplaceCopyIf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename PredicateType, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator replace_copy_if(const ExecutionSpace& ex, + InputIterator first_from, + InputIterator last_from, + OutputIterator first_dest, PredicateType pred, + const ValueType& new_value) { + return Impl::replace_copy_if_exespace_impl( + "Kokkos::replace_copy_if_iterator_api", ex, first_from, last_from, + first_dest, pred, new_value); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename PredicateType, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator replace_copy_if(const std::string& label, + const ExecutionSpace& ex, + InputIterator first_from, + InputIterator last_from, + OutputIterator first_dest, PredicateType pred, + const ValueType& new_value) { + return Impl::replace_copy_if_exespace_impl(label, ex, first_from, last_from, + first_dest, pred, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename PredicateType, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto replace_copy_if(const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + PredicateType pred, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::replace_copy_if_exespace_impl( + "Kokkos::replace_copy_if_view_api", ex, KE::cbegin(view_from), + KE::cend(view_from), KE::begin(view_dest), pred, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename PredicateType, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto replace_copy_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + PredicateType pred, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::replace_copy_if_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), pred, new_value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator replace_copy_if(const TeamHandleType& teamHandle, + InputIterator first_from, + InputIterator last_from, + OutputIterator first_dest, + PredicateType pred, + const ValueType& new_value) { + return Impl::replace_copy_if_team_impl(teamHandle, first_from, last_from, + first_dest, pred, new_value); +} + +template , int> = 0> +KOKKOS_FUNCTION auto replace_copy_if( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + PredicateType pred, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::replace_copy_if_team_impl(teamHandle, KE::cbegin(view_from), + KE::cend(view_from), + KE::begin(view_dest), pred, new_value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceIf.hpp new file mode 100644 index 0000000000..73af1f16f0 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReplaceIf.hpp @@ -0,0 +1,107 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REPLACE_IF_HPP +#define KOKKOS_STD_ALGORITHMS_REPLACE_IF_HPP + +#include "impl/Kokkos_ReplaceIf.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename Predicate, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace_if(const ExecutionSpace& ex, InputIterator first, + InputIterator last, Predicate pred, + const ValueType& new_value) { + Impl::replace_if_exespace_impl("Kokkos::replace_if_iterator_api", ex, first, + last, pred, new_value); +} + +template < + typename ExecutionSpace, typename InputIterator, typename Predicate, + typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace_if(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, Predicate pred, + const ValueType& new_value) { + Impl::replace_if_exespace_impl(label, ex, first, last, pred, new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename Predicate, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace_if(const ExecutionSpace& ex, + const ::Kokkos::View& view, + Predicate pred, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + Impl::replace_if_exespace_impl("Kokkos::replace_if_view_api", ex, + KE::begin(view), KE::end(view), pred, + new_value); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename Predicate, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void replace_if(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + Predicate pred, const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + Impl::replace_if_exespace_impl(label, ex, KE::begin(view), KE::end(view), + pred, new_value); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION void replace_if(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + Predicate pred, const ValueType& new_value) { + Impl::replace_if_team_impl(teamHandle, first, last, pred, new_value); +} + +template , int> = 0> +KOKKOS_FUNCTION void replace_if( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, Predicate pred, + const ValueType& new_value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + Impl::replace_if_team_impl(teamHandle, KE::begin(view), KE::end(view), pred, + new_value); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Reverse.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Reverse.hpp new file mode 100644 index 0000000000..a0786d3a2e --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Reverse.hpp @@ -0,0 +1,92 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REVERSE_HPP +#define KOKKOS_STD_ALGORITHMS_REVERSE_HPP + +#include "impl/Kokkos_Reverse.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void reverse(const ExecutionSpace& ex, InputIterator first, + InputIterator last) { + return Impl::reverse_exespace_impl("Kokkos::reverse_iterator_api_default", ex, + first, last); +} + +template < + typename ExecutionSpace, typename InputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void reverse(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last) { + return Impl::reverse_exespace_impl(label, ex, first, last); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void reverse(const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + return Impl::reverse_exespace_impl("Kokkos::reverse_view_api_default", ex, + KE::begin(view), KE::end(view)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +void reverse(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + return Impl::reverse_exespace_impl(label, ex, KE::begin(view), KE::end(view)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION void reverse(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last) { + return Impl::reverse_team_impl(teamHandle, first, last); +} + +template , int> = 0> +KOKKOS_FUNCTION void reverse( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + namespace KE = ::Kokkos::Experimental; + return Impl::reverse_team_impl(teamHandle, KE::begin(view), KE::end(view)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReverseCopy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReverseCopy.hpp new file mode 100644 index 0000000000..66f39c4eaa --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ReverseCopy.hpp @@ -0,0 +1,108 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_REVERSE_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_REVERSE_COPY_HPP + +#include "impl/Kokkos_ReverseCopy.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator reverse_copy(const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first) { + return Impl::reverse_copy_exespace_impl( + "Kokkos::reverse_copy_iterator_api_default", ex, first, last, d_first); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator reverse_copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first) { + return Impl::reverse_copy_exespace_impl(label, ex, first, last, d_first); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto reverse_copy(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::reverse_copy_exespace_impl( + "Kokkos::reverse_copy_view_api_default", ex, cbegin(source), cend(source), + begin(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto reverse_copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::reverse_copy_exespace_impl(label, ex, cbegin(source), + cend(source), begin(dest)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator reverse_copy(const TeamHandleType& teamHandle, + InputIterator first, + InputIterator last, + OutputIterator d_first) { + return Impl::reverse_copy_team_impl(teamHandle, first, last, d_first); +} + +template , int> = 0> +KOKKOS_FUNCTION auto reverse_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::reverse_copy_team_impl(teamHandle, cbegin(source), cend(source), + begin(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Rotate.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Rotate.hpp new file mode 100644 index 0000000000..aff04b47d6 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Rotate.hpp @@ -0,0 +1,96 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ROTATE_HPP +#define KOKKOS_STD_ALGORITHMS_ROTATE_HPP + +#include "impl/Kokkos_Rotate.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType rotate(const ExecutionSpace& ex, IteratorType first, + IteratorType n_first, IteratorType last) { + return Impl::rotate_exespace_impl("Kokkos::rotate_iterator_api_default", ex, + first, n_first, last); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType rotate(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType n_first, + IteratorType last) { + return Impl::rotate_exespace_impl(label, ex, first, n_first, last); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto rotate(const ExecutionSpace& ex, + const ::Kokkos::View& view, + std::size_t n_location) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::rotate_exespace_impl("Kokkos::rotate_view_api_default", ex, + begin(view), begin(view) + n_location, + end(view)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto rotate(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + std::size_t n_location) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::rotate_exespace_impl(label, ex, begin(view), + begin(view) + n_location, end(view)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType rotate(const TeamHandleType& teamHandle, + IteratorType first, IteratorType n_first, + IteratorType last) { + return Impl::rotate_team_impl(teamHandle, first, n_first, last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto rotate(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + std::size_t n_location) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::rotate_team_impl(teamHandle, begin(view), + begin(view) + n_location, end(view)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RotateCopy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RotateCopy.hpp new file mode 100644 index 0000000000..cce37fccfa --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_RotateCopy.hpp @@ -0,0 +1,117 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ROTATE_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_ROTATE_COPY_HPP + +#include "impl/Kokkos_RotateCopy.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator rotate_copy(const ExecutionSpace& ex, InputIterator first, + InputIterator n_first, InputIterator last, + OutputIterator d_first) { + return Impl::rotate_copy_exespace_impl( + "Kokkos::rotate_copy_iterator_api_default", ex, first, n_first, last, + d_first); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator rotate_copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator n_first, + InputIterator last, OutputIterator d_first) { + return Impl::rotate_copy_exespace_impl(label, ex, first, n_first, last, + d_first); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto rotate_copy(const ExecutionSpace& ex, + const ::Kokkos::View& source, + std::size_t n_location, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::rotate_copy_exespace_impl( + "Kokkos::rotate_copy_view_api_default", ex, cbegin(source), + cbegin(source) + n_location, cend(source), begin(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto rotate_copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + std::size_t n_location, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::rotate_copy_exespace_impl(label, ex, cbegin(source), + cbegin(source) + n_location, + cend(source), begin(dest)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator rotate_copy(const TeamHandleType& teamHandle, + InputIterator first, + InputIterator n_first, + InputIterator last, + OutputIterator d_first) { + return Impl::rotate_copy_team_impl(teamHandle, first, n_first, last, d_first); +} + +template , int> = 0> +KOKKOS_FUNCTION auto rotate_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + std::size_t n_location, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::rotate_copy_team_impl(teamHandle, cbegin(source), + cbegin(source) + n_location, cend(source), + begin(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Search.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Search.hpp new file mode 100644 index 0000000000..43258a484e --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Search.hpp @@ -0,0 +1,203 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_SEARCH_HPP +#define KOKKOS_STD_ALGORITHMS_SEARCH_HPP + +#include "impl/Kokkos_Search.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1: no binary predicate passed +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 search(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last) { + return Impl::search_exespace_impl("Kokkos::search_iterator_api_default", ex, + first, last, s_first, s_last); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 search(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last) { + return Impl::search_exespace_impl(label, ex, first, last, s_first, s_last); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto search(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_exespace_impl("Kokkos::search_view_api_default", ex, + KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto search(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_exespace_impl(label, ex, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view)); +} + +// overload set 2: binary predicate passed +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 search(const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last, const BinaryPredicateType& pred) { + return Impl::search_exespace_impl("Kokkos::search_iterator_api_default", ex, + first, last, s_first, s_last, pred); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType1 search(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::search_exespace_impl(label, ex, first, last, s_first, s_last, + pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto search(const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_exespace_impl("Kokkos::search_view_api_default", ex, + KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view), pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto search(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_exespace_impl(label, ex, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view), pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1: no binary predicate passed +template , int> = 0> +KOKKOS_FUNCTION IteratorType1 search(const TeamHandleType& teamHandle, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last) { + return Impl::search_team_impl(teamHandle, first, last, s_first, s_last); +} + +template , int> = 0> +KOKKOS_FUNCTION auto search( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_team_impl(teamHandle, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view)); +} + +// overload set 2: binary predicate passed +template , int> = 0> + +KOKKOS_FUNCTION IteratorType1 search(const TeamHandleType& teamHandle, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last, + const BinaryPredicateType& pred) { + return Impl::search_team_impl(teamHandle, first, last, s_first, s_last, pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto search( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + const ::Kokkos::View& s_view, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(s_view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_team_impl(teamHandle, KE::begin(view), KE::end(view), + KE::begin(s_view), KE::end(s_view), pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_SearchN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_SearchN.hpp new file mode 100644 index 0000000000..0f8aa5f1c1 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_SearchN.hpp @@ -0,0 +1,193 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_SEARCH_N_HPP +#define KOKKOS_STD_ALGORITHMS_SEARCH_N_HPP + +#include "impl/Kokkos_SearchN.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1: no binary predicate passed +template < + class ExecutionSpace, class IteratorType, class SizeType, class ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType search_n(const ExecutionSpace& ex, IteratorType first, + IteratorType last, SizeType count, + const ValueType& value) { + return Impl::search_n_exespace_impl("Kokkos::search_n_iterator_api_default", + ex, first, last, count, value); +} + +template < + class ExecutionSpace, class IteratorType, class SizeType, class ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType search_n(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, SizeType count, + const ValueType& value) { + return Impl::search_n_exespace_impl(label, ex, first, last, count, value); +} + +template ::value, + int> = 0> +auto search_n(const ExecutionSpace& ex, + const ::Kokkos::View& view, + SizeType count, const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_n_exespace_impl("Kokkos::search_n_view_api_default", ex, + KE::begin(view), KE::end(view), count, + value); +} + +template ::value, + int> = 0> +auto search_n(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + SizeType count, const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_n_exespace_impl(label, ex, KE::begin(view), KE::end(view), + count, value); +} + +// overload set 2: binary predicate passed +template < + class ExecutionSpace, class IteratorType, class SizeType, class ValueType, + class BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType search_n(const ExecutionSpace& ex, IteratorType first, + IteratorType last, SizeType count, const ValueType& value, + const BinaryPredicateType& pred) { + return Impl::search_n_exespace_impl("Kokkos::search_n_iterator_api_default", + ex, first, last, count, value, pred); +} + +template < + class ExecutionSpace, class IteratorType, class SizeType, class ValueType, + class BinaryPredicateType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType search_n(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, SizeType count, + const ValueType& value, const BinaryPredicateType& pred) { + return Impl::search_n_exespace_impl(label, ex, first, last, count, value, + pred); +} + +template ::value, + int> = 0> +auto search_n(const ExecutionSpace& ex, + const ::Kokkos::View& view, + SizeType count, const ValueType& value, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_n_exespace_impl("Kokkos::search_n_view_api_default", ex, + KE::begin(view), KE::end(view), count, + value, pred); +} + +template ::value, + int> = 0> +auto search_n(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + SizeType count, const ValueType& value, + const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_n_exespace_impl(label, ex, KE::begin(view), KE::end(view), + count, value, pred); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1: no binary predicate passed +template , int> = 0> +KOKKOS_FUNCTION IteratorType search_n(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + SizeType count, const ValueType& value) { + return Impl::search_n_team_impl(teamHandle, first, last, count, value); +} + +template < + class TeamHandleType, class DataType, class... Properties, class SizeType, + class ValueType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto search_n( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, SizeType count, + const ValueType& value) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_n_team_impl(teamHandle, KE::begin(view), KE::end(view), + count, value); +} + +// overload set 2: binary predicate passed +template , int> = 0> +KOKKOS_FUNCTION IteratorType search_n(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + SizeType count, const ValueType& value, + const BinaryPredicateType& pred) { + return Impl::search_n_team_impl(teamHandle, first, last, count, value, pred); +} + +template < + class TeamHandleType, class DataType, class... Properties, class SizeType, + class ValueType, class BinaryPredicateType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION auto search_n( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, SizeType count, + const ValueType& value, const BinaryPredicateType& pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + namespace KE = ::Kokkos::Experimental; + return Impl::search_n_team_impl(teamHandle, KE::begin(view), KE::end(view), + count, value, pred); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ShiftLeft.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ShiftLeft.hpp new file mode 100644 index 0000000000..b3e04a3b97 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ShiftLeft.hpp @@ -0,0 +1,95 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_SHIFT_LEFT_HPP +#define KOKKOS_STD_ALGORITHMS_SHIFT_LEFT_HPP + +#include "impl/Kokkos_ShiftLeft.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType shift_left(const ExecutionSpace& ex, IteratorType first, + IteratorType last, + typename IteratorType::difference_type n) { + return Impl::shift_left_exespace_impl( + "Kokkos::shift_left_iterator_api_default", ex, first, last, n); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType shift_left(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + typename IteratorType::difference_type n) { + return Impl::shift_left_exespace_impl(label, ex, first, last, n); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto shift_left(const ExecutionSpace& ex, + const ::Kokkos::View& view, + typename decltype(begin(view))::difference_type n) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::shift_left_exespace_impl("Kokkos::shift_left_view_api_default", + ex, begin(view), end(view), n); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto shift_left(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + typename decltype(begin(view))::difference_type n) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::shift_left_exespace_impl(label, ex, begin(view), end(view), n); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType +shift_left(const TeamHandleType& teamHandle, IteratorType first, + IteratorType last, typename IteratorType::difference_type n) { + return Impl::shift_left_team_impl(teamHandle, first, last, n); +} + +template , int> = 0> +KOKKOS_FUNCTION auto shift_left( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + typename decltype(begin(view))::difference_type n) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::shift_left_team_impl(teamHandle, begin(view), end(view), n); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ShiftRight.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ShiftRight.hpp new file mode 100644 index 0000000000..0f7ed53948 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_ShiftRight.hpp @@ -0,0 +1,95 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_SHIFT_RIGHT_HPP +#define KOKKOS_STD_ALGORITHMS_SHIFT_RIGHT_HPP + +#include "impl/Kokkos_ShiftRight.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType shift_right(const ExecutionSpace& ex, IteratorType first, + IteratorType last, + typename IteratorType::difference_type n) { + return Impl::shift_right_exespace_impl( + "Kokkos::shift_right_iterator_api_default", ex, first, last, n); +} + +template < + typename ExecutionSpace, typename IteratorType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +IteratorType shift_right(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + typename IteratorType::difference_type n) { + return Impl::shift_right_exespace_impl(label, ex, first, last, n); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto shift_right(const ExecutionSpace& ex, + const ::Kokkos::View& view, + typename decltype(begin(view))::difference_type n) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::shift_right_exespace_impl("Kokkos::shift_right_view_api_default", + ex, begin(view), end(view), n); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto shift_right(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + typename decltype(begin(view))::difference_type n) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::shift_right_exespace_impl(label, ex, begin(view), end(view), n); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType +shift_right(const TeamHandleType& teamHandle, IteratorType first, + IteratorType last, typename IteratorType::difference_type n) { + return Impl::shift_right_team_impl(teamHandle, first, last, n); +} + +template , int> = 0> +KOKKOS_FUNCTION auto shift_right( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + typename decltype(begin(view))::difference_type n) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::shift_right_team_impl(teamHandle, begin(view), end(view), n); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_SwapRanges.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_SwapRanges.hpp new file mode 100644 index 0000000000..d66763d304 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_SwapRanges.hpp @@ -0,0 +1,109 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_SWAP_RANGES_HPP +#define KOKKOS_STD_ALGORITHMS_SWAP_RANGES_HPP + +#include "impl/Kokkos_SwapRanges.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template , int> = 0> +IteratorType2 swap_ranges(const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2) { + return Impl::swap_ranges_exespace_impl( + "Kokkos::swap_ranges_iterator_api_default", ex, first1, last1, first2); +} + +template , int> = 0> +auto swap_ranges(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + assert(source.extent(0) == dest.extent(0)); + return Impl::swap_ranges_exespace_impl("Kokkos::swap_ranges_view_api_default", + ex, begin(source), end(source), + begin(dest)); +} + +template , int> = 0> +IteratorType2 swap_ranges(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2) { + return Impl::swap_ranges_exespace_impl(label, ex, first1, last1, first2); +} + +template , int> = 0> +auto swap_ranges(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + assert(source.extent(0) == dest.extent(0)); + return Impl::swap_ranges_exespace_impl(label, ex, begin(source), end(source), + begin(dest)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION IteratorType2 swap_ranges(const TeamHandleType& teamHandle, + IteratorType1 first1, + IteratorType1 last1, + IteratorType2 first2) { + return Impl::swap_ranges_team_impl(teamHandle, first1, last1, first2); +} + +template , int> = 0> +KOKKOS_FUNCTION auto swap_ranges( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + assert(source.extent(0) == dest.extent(0)); + return Impl::swap_ranges_team_impl(teamHandle, begin(source), end(source), + begin(dest)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Transform.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Transform.hpp new file mode 100644 index 0000000000..84cbed524d --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Transform.hpp @@ -0,0 +1,225 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_HPP +#define KOKKOS_STD_ALGORITHMS_TRANSFORM_HPP + +#include "impl/Kokkos_Transform.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename UnaryOperation, + std::enable_if_t && + is_execution_space_v, + int> = 0> +OutputIterator transform(const ExecutionSpace& ex, InputIterator first1, + InputIterator last1, OutputIterator d_first, + UnaryOperation unary_op) { + return Impl::transform_exespace_impl("Kokkos::transform_iterator_api_default", + ex, first1, last1, d_first, + std::move(unary_op)); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename UnaryOperation, + std::enable_if_t && + is_execution_space_v, + int> = 0> +OutputIterator transform(const std::string& label, const ExecutionSpace& ex, + InputIterator first1, InputIterator last1, + OutputIterator d_first, UnaryOperation unary_op) { + return Impl::transform_exespace_impl(label, ex, first1, last1, d_first, + std::move(unary_op)); +} + +template , int> = 0> +auto transform(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + UnaryOperation unary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::transform_exespace_impl("Kokkos::transform_view_api_default", ex, + begin(source), end(source), begin(dest), + std::move(unary_op)); +} + +template , int> = 0> +auto transform(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + UnaryOperation unary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::transform_exespace_impl(label, ex, begin(source), end(source), + begin(dest), std::move(unary_op)); +} + +template < + typename ExecutionSpace, typename InputIterator1, typename InputIterator2, + typename OutputIterator, typename BinaryOperation, + std::enable_if_t< + Impl::are_iterators_v && + is_execution_space_v, + int> = 0> +OutputIterator transform(const ExecutionSpace& ex, InputIterator1 first1, + InputIterator1 last1, InputIterator2 first2, + OutputIterator d_first, BinaryOperation binary_op) { + return Impl::transform_exespace_impl("Kokkos::transform_iterator_api_default", + ex, first1, last1, first2, d_first, + std::move(binary_op)); +} + +template < + typename ExecutionSpace, typename InputIterator1, typename InputIterator2, + typename OutputIterator, typename BinaryOperation, + std::enable_if_t< + Impl::are_iterators_v && + is_execution_space_v, + int> = 0> +OutputIterator transform(const std::string& label, const ExecutionSpace& ex, + InputIterator1 first1, InputIterator1 last1, + InputIterator2 first2, OutputIterator d_first, + BinaryOperation binary_op) { + return Impl::transform_exespace_impl(label, ex, first1, last1, first2, + d_first, std::move(binary_op)); +} + +template , int> = 0> +auto transform(const ExecutionSpace& ex, + const ::Kokkos::View& source1, + const ::Kokkos::View& source2, + const ::Kokkos::View& dest, + BinaryOperation binary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source2); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::transform_exespace_impl( + "Kokkos::transform_view_api_default", ex, begin(source1), end(source1), + begin(source2), begin(dest), std::move(binary_op)); +} + +template , int> = 0> +auto transform(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source1, + const ::Kokkos::View& source2, + const ::Kokkos::View& dest, + BinaryOperation binary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source2); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::transform_exespace_impl(label, ex, begin(source1), end(source1), + begin(source2), begin(dest), + std::move(binary_op)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template < + typename TeamHandleType, typename InputIterator, typename OutputIterator, + typename UnaryOperation, + std::enable_if_t && + is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIterator transform(const TeamHandleType& teamHandle, + InputIterator first1, + InputIterator last1, + OutputIterator d_first, + UnaryOperation unary_op) { + return Impl::transform_team_impl(teamHandle, first1, last1, d_first, + std::move(unary_op)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto transform( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + UnaryOperation unary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::transform_team_impl(teamHandle, begin(source), end(source), + begin(dest), std::move(unary_op)); +} + +template < + typename TeamHandleType, typename InputIterator1, typename InputIterator2, + typename OutputIterator, typename BinaryOperation, + std::enable_if_t< + Impl::are_iterators_v && + is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIterator transform(const TeamHandleType& teamHandle, + InputIterator1 first1, + InputIterator1 last1, + InputIterator2 first2, + OutputIterator d_first, + BinaryOperation binary_op) { + return Impl::transform_team_impl(teamHandle, first1, last1, first2, d_first, + std::move(binary_op)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto transform( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source1, + const ::Kokkos::View& source2, + const ::Kokkos::View& dest, + BinaryOperation binary_op) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source1); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source2); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::transform_team_impl(teamHandle, begin(source1), end(source1), + begin(source2), begin(dest), + std::move(binary_op)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformExclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformExclusiveScan.hpp new file mode 100644 index 0000000000..37fc0f860e --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformExclusiveScan.hpp @@ -0,0 +1,157 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_TRASFORM_EXCLUSIVE_SCAN_HPP +#define KOKKOS_STD_ALGORITHMS_TRASFORM_EXCLUSIVE_SCAN_HPP + +#include "impl/Kokkos_TransformExclusiveScan.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType transform_exclusive_scan( + const ExecutionSpace& ex, InputIteratorType first, InputIteratorType last, + OutputIteratorType first_dest, ValueType init_value, BinaryOpType binary_op, + UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::transform_exclusive_scan_exespace_impl( + "Kokkos::transform_exclusive_scan_custom_functors_iterator_api", ex, + first, last, first_dest, std::move(init_value), binary_op, unary_op); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType transform_exclusive_scan( + const std::string& label, const ExecutionSpace& ex, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, ValueType init_value, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::transform_exclusive_scan_exespace_impl( + label, ex, first, last, first_dest, std::move(init_value), binary_op, + unary_op); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + typename BinaryOpType, typename UnaryOpType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto transform_exclusive_scan( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value, BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::transform_exclusive_scan_exespace_impl( + "Kokkos::transform_exclusive_scan_custom_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + std::move(init_value), binary_op, unary_op); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + typename BinaryOpType, typename UnaryOpType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto transform_exclusive_scan( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value, BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::transform_exclusive_scan_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), std::move(init_value), binary_op, unary_op); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template && :: + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType transform_exclusive_scan( + const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, ValueType init_value, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(teamHandle); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + return Impl::transform_exclusive_scan_team_impl( + teamHandle, first, last, first_dest, std::move(init_value), binary_op, + unary_op); +} + +template , int> = 0> +KOKKOS_FUNCTION auto transform_exclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + ValueType init_value, BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(teamHandle); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + namespace KE = ::Kokkos::Experimental; + return Impl::transform_exclusive_scan_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), std::move(init_value), binary_op, unary_op); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformInclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformInclusiveScan.hpp new file mode 100644 index 0000000000..5f694dbfd9 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformInclusiveScan.hpp @@ -0,0 +1,282 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_INCLUSIVE_SCAN_HPP +#define KOKKOS_STD_ALGORITHMS_TRANSFORM_INCLUSIVE_SCAN_HPP + +#include "impl/Kokkos_TransformInclusiveScan.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// overload set 1 (no init value) +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType transform_inclusive_scan(const ExecutionSpace& ex, + InputIteratorType first, + InputIteratorType last, + OutputIteratorType first_dest, + BinaryOpType binary_op, + UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::transform_inclusive_scan_exespace_impl( + "Kokkos::transform_inclusive_scan_custom_functors_iterator_api", ex, + first, last, first_dest, binary_op, unary_op); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType transform_inclusive_scan( + const std::string& label, const ExecutionSpace& ex, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + + return Impl::transform_inclusive_scan_exespace_impl( + label, ex, first, last, first_dest, binary_op, unary_op); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOpType, + typename UnaryOpType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto transform_inclusive_scan( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::transform_inclusive_scan_exespace_impl( + "Kokkos::transform_inclusive_scan_custom_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + binary_op, unary_op); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOpType, + typename UnaryOpType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto transform_inclusive_scan( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::transform_inclusive_scan_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op, unary_op); +} + +// overload set 2 (init value) +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType transform_inclusive_scan( + const ExecutionSpace& ex, InputIteratorType first, InputIteratorType last, + OutputIteratorType first_dest, BinaryOpType binary_op, UnaryOpType unary_op, + ValueType init_value) { + Impl::static_assert_is_not_openmptarget(ex); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_inclusive_scan_exespace_impl( + "Kokkos::transform_inclusive_scan_custom_functors_iterator_api", ex, + first, last, first_dest, binary_op, unary_op, std::move(init_value)); +} + +template && :: + Kokkos::is_execution_space_v, + int> = 0> +OutputIteratorType transform_inclusive_scan( + const std::string& label, const ExecutionSpace& ex, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { + Impl::static_assert_is_not_openmptarget(ex); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_inclusive_scan_exespace_impl( + label, ex, first, last, first_dest, binary_op, unary_op, + std::move(init_value)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOpType, + typename UnaryOpType, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto transform_inclusive_scan( + const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + return Impl::transform_inclusive_scan_exespace_impl( + "Kokkos::transform_inclusive_scan_custom_functors_view_api", ex, + KE::cbegin(view_from), KE::cend(view_from), KE::begin(view_dest), + binary_op, unary_op, std::move(init_value)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryOpType, + typename UnaryOpType, typename ValueType, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto transform_inclusive_scan( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { + Impl::static_assert_is_not_openmptarget(ex); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + return Impl::transform_inclusive_scan_exespace_impl( + label, ex, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op, unary_op, std::move(init_value)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// overload set 1 (no init value) +template && + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType transform_inclusive_scan( + const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(teamHandle); + + return Impl::transform_inclusive_scan_team_impl( + teamHandle, first, last, first_dest, binary_op, unary_op); +} + +template , int> = 0> +KOKKOS_FUNCTION auto transform_inclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOpType binary_op, UnaryOpType unary_op) { + Impl::static_assert_is_not_openmptarget(teamHandle); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + namespace KE = ::Kokkos::Experimental; + return Impl::transform_inclusive_scan_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op, unary_op); +} + +// overload set 2 (init value) +template && + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIteratorType transform_inclusive_scan( + const TeamHandleType& teamHandle, InputIteratorType first, + InputIteratorType last, OutputIteratorType first_dest, + BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { + Impl::static_assert_is_not_openmptarget(teamHandle); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_inclusive_scan_team_impl( + teamHandle, first, last, first_dest, binary_op, unary_op, + std::move(init_value)); +} + +template , int> = 0> +KOKKOS_FUNCTION auto transform_inclusive_scan( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view_from, + const ::Kokkos::View& view_dest, + BinaryOpType binary_op, UnaryOpType unary_op, ValueType init_value) { + Impl::static_assert_is_not_openmptarget(teamHandle); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_from); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view_dest); + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + namespace KE = ::Kokkos::Experimental; + return Impl::transform_inclusive_scan_team_impl( + teamHandle, KE::cbegin(view_from), KE::cend(view_from), + KE::begin(view_dest), binary_op, unary_op, std::move(init_value)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp new file mode 100644 index 0000000000..89585ddbea --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp @@ -0,0 +1,429 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_TRANSFORM_REDUCE_HPP +#define KOKKOS_STD_ALGORITHMS_TRANSFORM_REDUCE_HPP + +#include "impl/Kokkos_TransformReduce.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set accepting execution space +// + +// ---------------------------- +// overload set1: +// no custom functors passed, so equivalent to +// transform_reduce(first1, last1, first2, init, plus<>(), multiplies<>()); +// ---------------------------- +template ::value, + int> = 0> +ValueType transform_reduce(const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, + ValueType init_reduction_value) { + return Impl::transform_reduce_default_functors_exespace_impl( + "Kokkos::transform_reduce_default_functors_iterator_api", ex, first1, + last1, first2, std::move(init_reduction_value)); +} + +template ::value, + int> = 0> +ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, + ValueType init_reduction_value) { + return Impl::transform_reduce_default_functors_exespace_impl( + label, ex, first1, last1, first2, std::move(init_reduction_value)); +} + +// overload1 accepting views +template ::value, + int> = 0> +ValueType transform_reduce( + const ExecutionSpace& ex, + const ::Kokkos::View& first_view, + const ::Kokkos::View& second_view, + ValueType init_reduction_value) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); + + return Impl::transform_reduce_default_functors_exespace_impl( + "Kokkos::transform_reduce_default_functors_iterator_api", ex, + KE::cbegin(first_view), KE::cend(first_view), KE::cbegin(second_view), + std::move(init_reduction_value)); +} + +template ::value, + int> = 0> +ValueType transform_reduce( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& first_view, + const ::Kokkos::View& second_view, + ValueType init_reduction_value) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); + + return Impl::transform_reduce_default_functors_exespace_impl( + label, ex, KE::cbegin(first_view), KE::cend(first_view), + KE::cbegin(second_view), std::move(init_reduction_value)); +} + +// +// overload set2: +// accepts a custom transform and joiner functor +// + +// Note the std refers to the arg BinaryReductionOp +// but in the Kokkos naming convention, it corresponds +// to a "joiner" that knows how to join two values +// NOTE: "joiner/transformer" need to be commutative. + +// https://en.cppreference.com/w/cpp/algorithm/transform_reduce + +// api accepting iterators +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename ValueType, typename BinaryJoinerType, typename BinaryTransform, + std::enable_if_t<::Kokkos::is_execution_space::value, int> = + 0> +ValueType transform_reduce(const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, + ValueType init_reduction_value, + BinaryJoinerType joiner, + BinaryTransform transformer) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_reduce_custom_functors_exespace_impl( + "Kokkos::transform_reduce_custom_functors_iterator_api", ex, first1, + last1, first2, std::move(init_reduction_value), std::move(joiner), + std::move(transformer)); +} + +template < + typename ExecutionSpace, typename IteratorType1, typename IteratorType2, + typename ValueType, typename BinaryJoinerType, typename BinaryTransform, + std::enable_if_t<::Kokkos::is_execution_space::value, int> = + 0> +ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, ValueType init_reduction_value, + BinaryJoinerType joiner, + BinaryTransform transformer) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_reduce_custom_functors_exespace_impl( + label, ex, first1, last1, first2, std::move(init_reduction_value), + std::move(joiner), std::move(transformer)); +} + +// accepting views +template ::value, + int> = 0> +ValueType transform_reduce( + const ExecutionSpace& ex, + const ::Kokkos::View& first_view, + const ::Kokkos::View& second_view, + ValueType init_reduction_value, BinaryJoinerType joiner, + BinaryTransform transformer) { + namespace KE = ::Kokkos::Experimental; + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); + + return Impl::transform_reduce_custom_functors_exespace_impl( + "Kokkos::transform_reduce_custom_functors_view_api", ex, + KE::cbegin(first_view), KE::cend(first_view), KE::cbegin(second_view), + std::move(init_reduction_value), std::move(joiner), + std::move(transformer)); +} + +template ::value, + int> = 0> +ValueType transform_reduce( + const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& first_view, + const ::Kokkos::View& second_view, + ValueType init_reduction_value, BinaryJoinerType joiner, + BinaryTransform transformer) { + namespace KE = ::Kokkos::Experimental; + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); + + return Impl::transform_reduce_custom_functors_exespace_impl( + label, ex, KE::cbegin(first_view), KE::cend(first_view), + KE::cbegin(second_view), std::move(init_reduction_value), + std::move(joiner), std::move(transformer)); +} + +// +// overload set3: +// +// accepting iterators +template ::value && + is_execution_space::value, + int> = 0> +ValueType transform_reduce(const ExecutionSpace& ex, IteratorType first1, + IteratorType last1, ValueType init_reduction_value, + BinaryJoinerType joiner, + UnaryTransform transformer) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_reduce_custom_functors_exespace_impl( + "Kokkos::transform_reduce_custom_functors_iterator_api", ex, first1, + last1, std::move(init_reduction_value), std::move(joiner), + std::move(transformer)); +} + +template ::value && + is_execution_space::value, + int> = 0> +ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, + IteratorType first1, IteratorType last1, + ValueType init_reduction_value, + BinaryJoinerType joiner, + UnaryTransform transformer) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_reduce_custom_functors_exespace_impl( + label, ex, first1, last1, std::move(init_reduction_value), + std::move(joiner), std::move(transformer)); +} + +// accepting views +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ValueType, typename BinaryJoinerType, typename UnaryTransform, + std::enable_if_t<::Kokkos::is_execution_space::value, int> = + 0> +ValueType transform_reduce(const ExecutionSpace& ex, + const ::Kokkos::View& view, + ValueType init_reduction_value, + BinaryJoinerType joiner, + UnaryTransform transformer) { + namespace KE = ::Kokkos::Experimental; + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::transform_reduce_custom_functors_exespace_impl( + "Kokkos::transform_reduce_custom_functors_view_api", ex, KE::cbegin(view), + KE::cend(view), std::move(init_reduction_value), std::move(joiner), + std::move(transformer)); +} + +template < + typename ExecutionSpace, typename DataType, typename... Properties, + typename ValueType, typename BinaryJoinerType, typename UnaryTransform, + std::enable_if_t<::Kokkos::is_execution_space::value, int> = + 0> +ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + ValueType init_reduction_value, + BinaryJoinerType joiner, + UnaryTransform transformer) { + namespace KE = ::Kokkos::Experimental; + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::transform_reduce_custom_functors_exespace_impl( + label, ex, KE::cbegin(view), KE::cend(view), + std::move(init_reduction_value), std::move(joiner), + std::move(transformer)); +} + +// +// overload set accepting a team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// + +// ---------------------------- +// overload set1: +// no custom functors passed, so equivalent to +// transform_reduce(first1, last1, first2, init, plus<>(), multiplies<>()); +// ---------------------------- +template < + typename TeamHandleType, typename IteratorType1, typename IteratorType2, + typename ValueType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType transform_reduce(const TeamHandleType& teamHandle, + IteratorType1 first1, + IteratorType1 last1, + IteratorType2 first2, + ValueType init_reduction_value) { + return Impl::transform_reduce_default_functors_team_impl( + teamHandle, first1, last1, first2, std::move(init_reduction_value)); +} + +// overload1 accepting views +template < + typename TeamHandleType, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType +transform_reduce(const TeamHandleType& teamHandle, + const ::Kokkos::View& first_view, + const ::Kokkos::View& second_view, + ValueType init_reduction_value) { + namespace KE = ::Kokkos::Experimental; + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); + + return Impl::transform_reduce_default_functors_team_impl( + teamHandle, KE::cbegin(first_view), KE::cend(first_view), + KE::cbegin(second_view), std::move(init_reduction_value)); +} + +// +// overload set2: +// accepts a custom transform and joiner functor +// + +// Note the std refers to the arg BinaryReductionOp +// but in the Kokkos naming convention, it corresponds +// to a "joiner" that knows how to join two values +// NOTE: "joiner/transformer" need to be commutative. + +// https://en.cppreference.com/w/cpp/algorithm/transform_reduce + +// api accepting iterators +template < + typename TeamHandleType, typename IteratorType1, typename IteratorType2, + typename ValueType, typename BinaryJoinerType, typename BinaryTransform, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType transform_reduce( + const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, ValueType init_reduction_value, + BinaryJoinerType joiner, BinaryTransform transformer) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_reduce_custom_functors_team_impl( + teamHandle, first1, last1, first2, std::move(init_reduction_value), + std::move(joiner), std::move(transformer)); +} + +// accepting views +template < + typename TeamHandleType, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename ValueType, + typename BinaryJoinerType, typename BinaryTransform, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType +transform_reduce(const TeamHandleType& teamHandle, + const ::Kokkos::View& first_view, + const ::Kokkos::View& second_view, + ValueType init_reduction_value, BinaryJoinerType joiner, + BinaryTransform transformer) { + namespace KE = ::Kokkos::Experimental; + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(second_view); + + return Impl::transform_reduce_custom_functors_team_impl( + teamHandle, KE::cbegin(first_view), KE::cend(first_view), + KE::cbegin(second_view), std::move(init_reduction_value), + std::move(joiner), std::move(transformer)); +} + +// +// overload set3: +// +// accepting iterators +template ::value && + is_team_handle::value, + int> = 0> +KOKKOS_FUNCTION ValueType transform_reduce(const TeamHandleType& teamHandle, + IteratorType first1, + IteratorType last1, + ValueType init_reduction_value, + BinaryJoinerType joiner, + UnaryTransform transformer) { + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + return Impl::transform_reduce_custom_functors_team_impl( + teamHandle, first1, last1, std::move(init_reduction_value), + std::move(joiner), std::move(transformer)); +} + +// accepting views +template < + typename TeamHandleType, typename DataType, typename... Properties, + typename ValueType, typename BinaryJoinerType, typename UnaryTransform, + std::enable_if_t<::Kokkos::is_team_handle::value, int> = 0> +KOKKOS_FUNCTION ValueType +transform_reduce(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + ValueType init_reduction_value, BinaryJoinerType joiner, + UnaryTransform transformer) { + namespace KE = ::Kokkos::Experimental; + static_assert(std::is_move_constructible_v, + "ValueType must be move constructible."); + + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + + return Impl::transform_reduce_custom_functors_team_impl( + teamHandle, KE::cbegin(view), KE::cend(view), + std::move(init_reduction_value), std::move(joiner), + std::move(transformer)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Unique.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Unique.hpp new file mode 100644 index 0000000000..2d56315f61 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_Unique.hpp @@ -0,0 +1,157 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_UNIQUE_HPP +#define KOKKOS_STD_ALGORITHMS_UNIQUE_HPP + +#include "impl/Kokkos_Unique.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set1: default predicate, accepting execution space +// +template && + is_execution_space::value, + int> = 0> +IteratorType unique(const ExecutionSpace& ex, IteratorType first, + IteratorType last) { + return Impl::unique_exespace_impl("Kokkos::unique_iterator_api_default", ex, + first, last); +} + +template && + is_execution_space::value, + int> = 0> +IteratorType unique(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + return Impl::unique_exespace_impl(label, ex, first, last); +} + +template ::value, int> = 0> +auto unique(const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::unique_exespace_impl("Kokkos::unique_view_api_default", ex, + begin(view), end(view)); +} + +template ::value, int> = 0> +auto unique(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::unique_exespace_impl(label, ex, begin(view), end(view)); +} + +// +// overload set2: custom predicate, accepting execution space +// +template ::value, int> = 0> +IteratorType unique(const ExecutionSpace& ex, IteratorType first, + IteratorType last, BinaryPredicate pred) { + return Impl::unique_exespace_impl("Kokkos::unique_iterator_api_default", ex, + first, last, pred); +} + +template ::value, int> = 0> +IteratorType unique(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + BinaryPredicate pred) { + return Impl::unique_exespace_impl(label, ex, first, last, pred); +} + +template ::value, int> = 0> +auto unique(const ExecutionSpace& ex, + const ::Kokkos::View& view, + BinaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::unique_exespace_impl("Kokkos::unique_view_api_default", ex, + begin(view), end(view), std::move(pred)); +} + +template ::value, int> = 0> +auto unique(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& view, + BinaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); + return Impl::unique_exespace_impl(label, ex, begin(view), end(view), + std::move(pred)); +} + +// +// overload set3: default predicate, accepting team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template && + is_team_handle::value, + int> = 0> +KOKKOS_FUNCTION IteratorType unique(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last) { + return Impl::unique_team_impl(teamHandle, first, last); +} + +template ::value, int> = 0> +KOKKOS_FUNCTION auto unique( + const TeamHandleType& teamHandle, + const ::Kokkos::View& view) { + return Impl::unique_team_impl(teamHandle, begin(view), end(view)); +} + +// +// overload set4: custom predicate, accepting team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template ::value, int> = 0> +KOKKOS_FUNCTION IteratorType unique(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + BinaryPredicate pred) { + return Impl::unique_team_impl(teamHandle, first, last, std::move(pred)); +} + +template ::value, int> = 0> +KOKKOS_FUNCTION auto unique(const TeamHandleType& teamHandle, + const ::Kokkos::View& view, + BinaryPredicate pred) { + return Impl::unique_team_impl(teamHandle, begin(view), end(view), + std::move(pred)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_UniqueCopy.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_UniqueCopy.hpp new file mode 100644 index 0000000000..4a32d7e095 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/Kokkos_UniqueCopy.hpp @@ -0,0 +1,202 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_UNIQUE_COPY_HPP +#define KOKKOS_STD_ALGORITHMS_UNIQUE_COPY_HPP + +#include "impl/Kokkos_UniqueCopy.hpp" +#include "Kokkos_BeginEnd.hpp" + +namespace Kokkos { +namespace Experimental { + +// +// overload set1: default predicate, accepting execution space +// +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t && + is_execution_space_v, + int> = 0> +OutputIterator unique_copy(const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first) { + return Impl::unique_copy_exespace_impl( + "Kokkos::unique_copy_iterator_api_default", ex, first, last, d_first); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + std::enable_if_t && + is_execution_space_v, + int> = 0> +OutputIterator unique_copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first) { + return Impl::unique_copy_exespace_impl(label, ex, first, last, d_first); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto unique_copy(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::unique_copy_exespace_impl("Kokkos::unique_copy_view_api_default", + ex, cbegin(source), cend(source), + begin(dest)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto unique_copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::unique_copy_exespace_impl(label, ex, cbegin(source), + cend(source), begin(dest)); +} + +// +// overload set2: custom predicate, accepting execution space +// + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename BinaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator unique_copy(const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first, + BinaryPredicate pred) { + return Impl::unique_copy_exespace_impl( + "Kokkos::unique_copy_iterator_api_default", ex, first, last, d_first, + pred); +} + +template < + typename ExecutionSpace, typename InputIterator, typename OutputIterator, + typename BinaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +OutputIterator unique_copy(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first, BinaryPredicate pred) { + return Impl::unique_copy_exespace_impl(label, ex, first, last, d_first, pred); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto unique_copy(const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + BinaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::unique_copy_exespace_impl("Kokkos::unique_copy_view_api_default", + ex, cbegin(source), cend(source), + begin(dest), std::move(pred)); +} + +template < + typename ExecutionSpace, typename DataType1, typename... Properties1, + typename DataType2, typename... Properties2, typename BinaryPredicate, + std::enable_if_t<::Kokkos::is_execution_space_v, int> = 0> +auto unique_copy(const std::string& label, const ExecutionSpace& ex, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + BinaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::unique_copy_exespace_impl( + label, ex, cbegin(source), cend(source), begin(dest), std::move(pred)); +} + +// +// overload set3: default predicate, accepting team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template < + typename TeamHandleType, typename InputIterator, typename OutputIterator, + std::enable_if_t && + Kokkos::is_team_handle_v, + int> = 0> +KOKKOS_FUNCTION OutputIterator unique_copy(const TeamHandleType& teamHandle, + InputIterator first, + InputIterator last, + OutputIterator d_first) { + return Impl::unique_copy_team_impl(teamHandle, first, last, d_first); +} + +template , int> = 0> +KOKKOS_FUNCTION auto unique_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::unique_copy_team_impl(teamHandle, cbegin(source), cend(source), + begin(dest)); +} + +// +// overload set4: custom predicate, accepting team handle +// Note: for now omit the overloads accepting a label +// since they cause issues on device because of the string allocation. +// +template , int> = 0> +KOKKOS_FUNCTION OutputIterator unique_copy(const TeamHandleType& teamHandle, + InputIterator first, + InputIterator last, + OutputIterator d_first, + BinaryPredicate pred) { + return Impl::unique_copy_team_impl(teamHandle, first, last, d_first, pred); +} + +template , int> = 0> +KOKKOS_FUNCTION auto unique_copy( + const TeamHandleType& teamHandle, + const ::Kokkos::View& source, + const ::Kokkos::View& dest, + BinaryPredicate pred) { + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(source); + Impl::static_assert_is_admissible_to_kokkos_std_algorithms(dest); + + return Impl::unique_copy_team_impl(teamHandle, cbegin(source), cend(source), + begin(dest), std::move(pred)); +} + +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AdjacentDifference.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AdjacentDifference.hpp new file mode 100644 index 0000000000..9f7fcf94fe --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AdjacentDifference.hpp @@ -0,0 +1,143 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_ADJACENT_DIFFERENCE_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdAdjacentDifferenceDefaultBinaryOpFunctor { + KOKKOS_FUNCTION + constexpr RetType operator()(const ValueType1& a, const ValueType2& b) const { + return a - b; + } +}; + +template +struct StdAdjacentDiffFunctor { + using index_type = typename InputIteratorType::difference_type; + + const InputIteratorType m_first_from; + const OutputIteratorType m_first_dest; + BinaryOperator m_op; + + KOKKOS_FUNCTION + void operator()(const index_type i) const { + const auto& my_value = m_first_from[i]; + if (i == 0) { + m_first_dest[i] = my_value; + } else { + const auto& left_value = m_first_from[i - 1]; + m_first_dest[i] = m_op(my_value, left_value); + } + } + + KOKKOS_FUNCTION + StdAdjacentDiffFunctor(InputIteratorType first_from, + OutputIteratorType first_dest, BinaryOperator op) + : m_first_from(std::move(first_from)), + m_first_dest(std::move(first_dest)), + m_op(std::move(op)) {} +}; + +// +// exespace impl +// +template +OutputIteratorType adjacent_difference_exespace_impl( + const std::string& label, const ExecutionSpace& ex, + InputIteratorType first_from, InputIteratorType last_from, + OutputIteratorType first_dest, BinaryOp bin_op) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + if (first_from == last_from) { + return first_dest; + } + +#ifdef KOKKOS_ENABLE_DEBUG + // check for overlapping iterators + Impl::expect_no_overlap(first_from, last_from, first_dest); +#endif + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_for( + label, RangePolicy(ex, 0, num_elements), + StdAdjacentDiffFunctor(first_from, first_dest, bin_op)); + ex.fence("Kokkos::adjacent_difference: fence after operation"); + + // return + return first_dest + num_elements; +} + +// +// team impl +// +template +KOKKOS_FUNCTION OutputIteratorType adjacent_difference_team_impl( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest, + BinaryOp bin_op) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + if (first_from == last_from) { + return first_dest; + } + +#ifdef KOKKOS_ENABLE_DEBUG + // check for overlapping iterators + Impl::expect_no_overlap(first_from, last_from, first_dest); +#endif + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_for( + TeamThreadRange(teamHandle, 0, num_elements), + StdAdjacentDiffFunctor(first_from, first_dest, bin_op)); + teamHandle.team_barrier(); + + // return + return first_dest + num_elements; +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AdjacentFind.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AdjacentFind.hpp new file mode 100644 index 0000000000..f30b7be06a --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AdjacentFind.hpp @@ -0,0 +1,166 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_ADJACENT_FIND_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdAdjacentFindFunctor { + using index_type = typename IteratorType::difference_type; + using red_value_type = typename ReducerType::value_type; + + IteratorType m_first; + ReducerType m_reducer; + PredicateType m_p; + + KOKKOS_FUNCTION + void operator()(const index_type i, red_value_type& red_value) const { + const auto& my_value = m_first[i]; + const auto& next_value = m_first[i + 1]; + const bool are_equal = m_p(my_value, next_value); + + // FIXME_NVHPC using a ternary operator causes problems + red_value_type value = {::Kokkos::reduction_identity::min()}; + if (are_equal) { + value.min_loc_true = i; + } + + m_reducer.join(red_value, value); + } + + KOKKOS_FUNCTION + StdAdjacentFindFunctor(IteratorType first, ReducerType reducer, + PredicateType p) + : m_first(std::move(first)), + m_reducer(std::move(reducer)), + m_p(std::move(p)) {} +}; + +// +// exespace impl +// +template +IteratorType adjacent_find_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, IteratorType last, + PredicateType pred) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first); + Impl::expect_valid_range(first, last); + + const auto num_elements = Kokkos::Experimental::distance(first, last); + + if (num_elements <= 1) { + return last; + } + + using index_type = typename IteratorType::difference_type; + using reducer_type = FirstLoc; + using reduction_value_type = typename reducer_type::value_type; + + reduction_value_type red_result; + reducer_type reducer(red_result); + + // note that we use below num_elements-1 because + // each index i in the reduction checks i and (i+1). + ::Kokkos::parallel_reduce( + label, RangePolicy(ex, 0, num_elements - 1), + // use CTAD + StdAdjacentFindFunctor(first, reducer, pred), reducer); + + // fence not needed because reducing into scalar + if (red_result.min_loc_true == + ::Kokkos::reduction_identity::min()) { + return last; + } else { + return first + red_result.min_loc_true; + } +} + +template +IteratorType adjacent_find_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, + IteratorType last) { + using value_type = typename IteratorType::value_type; + using default_pred_t = StdAlgoEqualBinaryPredicate; + return adjacent_find_exespace_impl(label, ex, first, last, default_pred_t()); +} + +// +// team impl +// +template +KOKKOS_FUNCTION IteratorType +adjacent_find_team_impl(const TeamHandleType& teamHandle, IteratorType first, + IteratorType last, PredicateType pred) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + const auto num_elements = Kokkos::Experimental::distance(first, last); + + if (num_elements <= 1) { + return last; + } + + using index_type = typename IteratorType::difference_type; + using reducer_type = FirstLoc; + using reduction_value_type = typename reducer_type::value_type; + + reduction_value_type red_result; + reducer_type reducer(red_result); + + // note that we use below num_elements-1 because + // each index i in the reduction checks i and (i+1). + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, num_elements - 1), + // use CTAD + StdAdjacentFindFunctor(first, reducer, pred), + reducer); + + teamHandle.team_barrier(); + + if (red_result.min_loc_true == + ::Kokkos::reduction_identity::min()) { + return last; + } else { + return first + red_result.min_loc_true; + } +} + +template +KOKKOS_FUNCTION IteratorType adjacent_find_team_impl( + const TeamHandleType& teamHandle, IteratorType first, IteratorType last) { + using value_type = typename IteratorType::value_type; + using default_pred_t = StdAlgoEqualBinaryPredicate; + return adjacent_find_team_impl(teamHandle, first, last, default_pred_t()); +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AllOfAnyOfNoneOf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AllOfAnyOfNoneOf.hpp new file mode 100644 index 0000000000..bdc050f9c1 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_AllOfAnyOfNoneOf.hpp @@ -0,0 +1,84 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_ALL_OF_ANY_OF_NONE_OF_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_ALL_OF_ANY_OF_NONE_OF_IMPL_HPP + +#include "Kokkos_FindIfOrNot.hpp" + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +// +// exespace impl +// +template +bool all_of_exespace_impl(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + Predicate predicate) { + return (find_if_or_not_exespace_impl(label, ex, first, last, + predicate) == last); +} + +template +bool any_of_exespace_impl(const std::string& label, const ExecutionSpace& ex, + InputIterator first, InputIterator last, + Predicate predicate) { + return (find_if_or_not_exespace_impl(label, ex, first, last, + predicate) != last); +} + +template +bool none_of_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + Predicate predicate) { + return (find_if_or_not_exespace_impl(label, ex, first, last, + predicate) == last); +} + +// +// team impl +// +template +KOKKOS_FUNCTION bool all_of_team_impl(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + Predicate predicate) { + return (find_if_or_not_team_impl(teamHandle, first, last, predicate) == + last); +} + +template +KOKKOS_FUNCTION bool any_of_team_impl(const TeamHandleType& teamHandle, + InputIterator first, InputIterator last, + Predicate predicate) { + return (find_if_or_not_team_impl(teamHandle, first, last, predicate) != + last); +} + +template +KOKKOS_FUNCTION bool none_of_team_impl(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + Predicate predicate) { + return (find_if_or_not_team_impl(teamHandle, first, last, predicate) == + last); +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp similarity index 54% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp index ec54cd1900..da16141f5a 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_CONSTRAINTS_HPP_ #define KOKKOS_STD_ALGORITHMS_CONSTRAINTS_HPP_ @@ -52,18 +24,21 @@ namespace Kokkos { namespace Experimental { namespace Impl { +template +class RandomAccessIterator; + template struct is_admissible_to_kokkos_std_algorithms : std::false_type {}; template struct is_admissible_to_kokkos_std_algorithms< - T, std::enable_if_t< ::Kokkos::is_view::value && T::rank == 1 && - (std::is_same::value || - std::is_same::value || - std::is_same::value)> > + T, std::enable_if_t<::Kokkos::is_view::value && T::rank() == 1 && + (std::is_same_v || + std::is_same_v || + std::is_same_v)>> : std::true_type {}; template @@ -83,6 +58,21 @@ using iterator_category_t = typename T::iterator_category; template using is_iterator = Kokkos::is_detected; +template +inline constexpr bool is_iterator_v = is_iterator::value; + +template +struct is_kokkos_iterator : std::false_type {}; + +template +struct is_kokkos_iterator> { + static constexpr bool value = + is_admissible_to_kokkos_std_algorithms::value; +}; + +template +inline constexpr bool is_kokkos_iterator_v = is_kokkos_iterator::value; + // // are_iterators // @@ -91,15 +81,18 @@ struct are_iterators; template struct are_iterators { - static constexpr bool value = is_iterator::value; + static constexpr bool value = is_iterator_v; }; template struct are_iterators { static constexpr bool value = - are_iterators::value && are_iterators::value; + are_iterators::value && (are_iterators::value && ... && true); }; +template +inline constexpr bool are_iterators_v = are_iterators::value; + // // are_random_access_iterators // @@ -109,17 +102,21 @@ struct are_random_access_iterators; template struct are_random_access_iterators { static constexpr bool value = - is_iterator::value && - std::is_base_of::value; + is_iterator_v && std::is_base_of_v; }; template struct are_random_access_iterators { - static constexpr bool value = are_random_access_iterators::value && - are_random_access_iterators::value; + static constexpr bool value = + are_random_access_iterators::value && + (are_random_access_iterators::value && ... && true); }; +template +inline constexpr bool are_random_access_iterators_v = + are_random_access_iterators::value; + // // iterators_are_accessible_from // @@ -141,16 +138,18 @@ struct iterators_are_accessible_from { iterators_are_accessible_from::value; }; -template +template KOKKOS_INLINE_FUNCTION constexpr void -static_assert_random_access_and_accessible(const ExecutionSpace& /* ex */, - IteratorTypes... /* iterators */) { +static_assert_random_access_and_accessible( + const ExecutionSpaceOrTeamHandleType& /* ex_or_th*/, + IteratorTypes... /* iterators */) { static_assert( are_random_access_iterators::value, "Currently, Kokkos standard algorithms require random access iterators."); - static_assert( - iterators_are_accessible_from::value, - "Incompatible view/iterator and execution space"); + static_assert(iterators_are_accessible_from< + typename ExecutionSpaceOrTeamHandleType::execution_space, + IteratorTypes...>::value, + "Incompatible view/iterator and execution space"); } // @@ -166,9 +165,8 @@ struct iterators_have_matching_difference_type { template struct iterators_have_matching_difference_type { - static constexpr bool value = - std::is_same::value; + static constexpr bool value = std::is_same_v; }; template @@ -210,10 +208,10 @@ struct not_openmptarget { #endif }; -template +template KOKKOS_INLINE_FUNCTION constexpr void static_assert_is_not_openmptarget( - const ExecutionSpace&) { - static_assert(not_openmptarget::value, + const ExecutionSpaceOrTeamHandleType& /*ex_or_th*/) { + static_assert(not_openmptarget::value, "Currently, Kokkos standard algorithms do not support custom " "comparators in OpenMPTarget"); } @@ -222,7 +220,8 @@ KOKKOS_INLINE_FUNCTION constexpr void static_assert_is_not_openmptarget( // valid range // template -void expect_valid_range(IteratorType first, IteratorType last) { +KOKKOS_INLINE_FUNCTION void expect_valid_range(IteratorType first, + IteratorType last) { // this is a no-op for release KOKKOS_EXPECTS(last >= first); // avoid compiler complaining when KOKKOS_EXPECTS is no-op @@ -230,6 +229,38 @@ void expect_valid_range(IteratorType first, IteratorType last) { (void)last; } +// +// Check if kokkos iterators are overlapping +// +template +KOKKOS_INLINE_FUNCTION void expect_no_overlap( + [[maybe_unused]] IteratorType1 first, [[maybe_unused]] IteratorType1 last, + [[maybe_unused]] IteratorType2 s_first) { + if constexpr (is_kokkos_iterator_v && + is_kokkos_iterator_v) { + auto const view1 = first.view(); + auto const view2 = s_first.view(); + + std::size_t stride1 = view1.stride(0); + std::size_t stride2 = view2.stride(0); + ptrdiff_t first_diff = view1.data() - view2.data(); + + // FIXME If strides are not identical, checks may not be made + // with the cost of O(1) + // Currently, checks are made only if strides are identical + // If first_diff == 0, there is already an overlap + if (stride1 == stride2 || first_diff == 0) { + [[maybe_unused]] bool is_no_overlap = (first_diff % stride1); + auto* first_pointer1 = view1.data(); + auto* first_pointer2 = view2.data(); + [[maybe_unused]] auto* last_pointer1 = first_pointer1 + (last - first); + [[maybe_unused]] auto* last_pointer2 = first_pointer2 + (last - first); + KOKKOS_EXPECTS(first_pointer1 >= last_pointer2 || + last_pointer1 <= first_pointer2 || is_no_overlap); + } + } +} + } // namespace Impl } // namespace Experimental } // namespace Kokkos diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyBackward.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyBackward.hpp new file mode 100644 index 0000000000..0f68c9e978 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyBackward.hpp @@ -0,0 +1,98 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_COPY_BACKWARD_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdCopyBackwardFunctor { + // we can use difference type from IteratorType1 since + // the calling functions below already static assert that + // the iterators have matching difference type + using index_type = typename IteratorType1::difference_type; + + IteratorType1 m_last; + IteratorType2 m_dest_last; + + KOKKOS_FUNCTION + void operator()(index_type i) const { m_dest_last[-i - 1] = m_last[-i - 1]; } + + KOKKOS_FUNCTION + StdCopyBackwardFunctor(IteratorType1 _last, IteratorType2 _dest_last) + : m_last(std::move(_last)), m_dest_last(std::move(_dest_last)) {} +}; + +template +IteratorType2 copy_backward_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType1 first, + IteratorType1 last, + IteratorType2 d_last) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first, d_last); + Impl::static_assert_iterators_have_matching_difference_type(first, d_last); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(label, + RangePolicy(ex, 0, num_elements), + // use CTAD + StdCopyBackwardFunctor(last, d_last)); + ex.fence("Kokkos::copy_backward: fence after operation"); + + // return + return d_last - num_elements; +} + +// +// team-level impl +// +template +KOKKOS_FUNCTION IteratorType2 +copy_backward_team_impl(const TeamHandleType& teamHandle, IteratorType1 first, + IteratorType1 last, IteratorType2 d_last) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first, d_last); + Impl::static_assert_iterators_have_matching_difference_type(first, d_last); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(TeamThreadRange(teamHandle, 0, num_elements), + // use CTAD + StdCopyBackwardFunctor(last, d_last)); + teamHandle.team_barrier(); + + // return + return d_last - num_elements; +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyCopyN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyCopyN.hpp new file mode 100644 index 0000000000..86e99ecbd0 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyCopyN.hpp @@ -0,0 +1,135 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COPY_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_COPY_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdCopyFunctor { + // we can use difference type from InputIterator since + // the calling functions below already static assert that + // the iterators have matching difference type + using index_type = typename InputIterator::difference_type; + + InputIterator m_first; + OutputIterator m_dest_first; + + KOKKOS_FUNCTION + void operator()(index_type i) const { m_dest_first[i] = m_first[i]; } + + KOKKOS_FUNCTION + StdCopyFunctor(InputIterator _first, OutputIterator _dest_first) + : m_first(std::move(_first)), m_dest_first(std::move(_dest_first)) {} +}; + +template +OutputIterator copy_exespace_impl(const std::string& label, + const ExecutionSpace& ex, InputIterator first, + InputIterator last, OutputIterator d_first) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first, d_first); + Impl::static_assert_iterators_have_matching_difference_type(first, d_first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(label, + RangePolicy(ex, 0, num_elements), + // use CTAD + StdCopyFunctor(first, d_first)); + ex.fence("Kokkos::copy: fence after operation"); + + // return + return d_first + num_elements; +} + +template +OutputIterator copy_n_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + InputIterator first_from, Size count, + OutputIterator first_dest) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + + if (count > 0) { + return copy_exespace_impl(label, ex, first_from, first_from + count, + first_dest); + } else { + return first_dest; + } +} + +// +// team-level impl +// +template +KOKKOS_FUNCTION OutputIterator copy_team_impl(const TeamHandleType& teamHandle, + InputIterator first, + InputIterator last, + OutputIterator d_first) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first, d_first); + Impl::static_assert_iterators_have_matching_difference_type(first, d_first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(TeamThreadRange(teamHandle, 0, num_elements), + // use CTAD + StdCopyFunctor(first, d_first)); + teamHandle.team_barrier(); + + // return + return d_first + num_elements; +} + +template +KOKKOS_FUNCTION OutputIterator +copy_n_team_impl(const TeamHandleType& teamHandle, InputIterator first_from, + Size count, OutputIterator first_dest) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + + if (count > 0) { + return copy_team_impl(teamHandle, first_from, first_from + count, + first_dest); + } else { + return first_dest; + } +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyIf.hpp similarity index 50% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyIf.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyIf.hpp index aebb5a9a46..ad7b8bb8ca 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_CopyIf.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CopyIf.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_COPY_IF_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_COPY_IF_IMPL_HPP @@ -48,6 +20,7 @@ #include #include "Kokkos_Constraints.hpp" #include "Kokkos_HelperPredicates.hpp" +#include "Kokkos_MustUseKokkosSingleInTeam.hpp" #include #include @@ -55,8 +28,10 @@ namespace Kokkos { namespace Experimental { namespace Impl { -template +template struct StdCopyIfFunctor { + using index_type = typename FirstFrom::difference_type; + FirstFrom m_first_from; FirstDest m_first_dest; PredType m_pred; @@ -68,7 +43,7 @@ struct StdCopyIfFunctor { m_pred(std::move(pred)) {} KOKKOS_FUNCTION - void operator()(const IndexType i, IndexType& update, + void operator()(const index_type i, index_type& update, const bool final_pass) const { const auto& myval = m_first_from[i]; if (final_pass) { @@ -85,9 +60,11 @@ struct StdCopyIfFunctor { template -OutputIterator copy_if_impl(const std::string& label, const ExecutionSpace& ex, - InputIterator first, InputIterator last, - OutputIterator d_first, PredicateType pred) { +OutputIterator copy_if_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + InputIterator first, InputIterator last, + OutputIterator d_first, + PredicateType pred) { /* To explain the impl, suppose that our data is: @@ -118,23 +95,68 @@ OutputIterator copy_if_impl(const std::string& label, const ExecutionSpace& ex, if (first == last) { return d_first; } else { - // aliases - using index_type = typename InputIterator::difference_type; - using func_type = StdCopyIfFunctor; - // run const auto num_elements = Kokkos::Experimental::distance(first, last); - index_type count = 0; + + typename InputIterator::difference_type count = 0; ::Kokkos::parallel_scan(label, RangePolicy(ex, 0, num_elements), - func_type(first, d_first, pred), count); + // use CTAD + StdCopyIfFunctor(first, d_first, pred), count); // fence not needed because of the scan accumulating into count return d_first + count; } } +template +KOKKOS_FUNCTION OutputIterator copy_if_team_impl( + const TeamHandleType& teamHandle, InputIterator first, InputIterator last, + OutputIterator d_first, PredicateType pred) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first, d_first); + Impl::static_assert_iterators_have_matching_difference_type(first, d_first); + Impl::expect_valid_range(first, last); + + if (first == last) { + return d_first; + } + + const std::size_t num_elements = Kokkos::Experimental::distance(first, last); + if constexpr (stdalgo_must_use_kokkos_single_for_team_scan_v< + typename TeamHandleType::execution_space>) { + std::size_t count = 0; + Kokkos::single( + Kokkos::PerTeam(teamHandle), + [=](std::size_t& lcount) { + lcount = 0; + for (std::size_t i = 0; i < num_elements; ++i) { + const auto& myval = first[i]; + if (pred(myval)) { + d_first[lcount++] = myval; + } + } + }, + count); + // no barrier needed since single above broadcasts to all members + return d_first + count; + + } else { + typename InputIterator::difference_type count = 0; + ::Kokkos::parallel_scan(TeamThreadRange(teamHandle, 0, num_elements), + StdCopyIfFunctor(first, d_first, pred), count); + // no barrier needed because of the scan accumulating into count + return d_first + count; + } + +#if defined KOKKOS_COMPILER_INTEL || \ + (defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \ + !defined(KOKKOS_COMPILER_MSVC)) + __builtin_unreachable(); +#endif +} + } // namespace Impl } // namespace Experimental } // namespace Kokkos diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CountCountIf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CountCountIf.hpp new file mode 100644 index 0000000000..9b6b403aa4 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_CountCountIf.hpp @@ -0,0 +1,112 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_COUNT_IF_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_COUNT_IF_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdCountIfFunctor { + using index_type = typename IteratorType::difference_type; + IteratorType m_first; + Predicate m_predicate; + + KOKKOS_FUNCTION + void operator()(index_type i, index_type& lsum) const { + if (m_predicate(m_first[i])) { + lsum++; + } + } + + KOKKOS_FUNCTION + StdCountIfFunctor(IteratorType _first, Predicate _predicate) + : m_first(std::move(_first)), m_predicate(std::move(_predicate)) {} +}; + +template +typename IteratorType::difference_type count_if_exespace_impl( + const std::string& label, const ExecutionSpace& ex, IteratorType first, + IteratorType last, Predicate predicate) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + typename IteratorType::difference_type count = 0; + ::Kokkos::parallel_reduce(label, + RangePolicy(ex, 0, num_elements), + // use CTAD + StdCountIfFunctor(first, predicate), count); + ex.fence("Kokkos::count_if: fence after operation"); + + return count; +} + +template +auto count_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + const T& value) { + return count_if_exespace_impl( + label, ex, first, last, + ::Kokkos::Experimental::Impl::StdAlgoEqualsValUnaryPredicate(value)); +} + +// +// team-level impl +// +template +KOKKOS_FUNCTION typename IteratorType::difference_type count_if_team_impl( + const TeamHandleType& teamHandle, IteratorType first, IteratorType last, + Predicate predicate) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + typename IteratorType::difference_type count = 0; + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, num_elements), + // use CTAD + StdCountIfFunctor(first, predicate), count); + teamHandle.team_barrier(); + + return count; +} + +template +KOKKOS_FUNCTION auto count_team_impl(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + const T& value) { + return count_if_team_impl( + teamHandle, first, last, + ::Kokkos::Experimental::Impl::StdAlgoEqualsValUnaryPredicate(value)); +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_Equal.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_Equal.hpp new file mode 100644 index 0000000000..62b7d226f6 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_Equal.hpp @@ -0,0 +1,183 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_EQUAL_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_EQUAL_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdEqualFunctor { + using index_type = typename IteratorType1::difference_type; + + IteratorType1 m_first1; + IteratorType2 m_first2; + BinaryPredicateType m_predicate; + + KOKKOS_FUNCTION + void operator()(index_type i, std::size_t& lsum) const { + if (!m_predicate(m_first1[i], m_first2[i])) { + lsum = 1; + } + } + + KOKKOS_FUNCTION + StdEqualFunctor(IteratorType1 _first1, IteratorType2 _first2, + BinaryPredicateType _predicate) + : m_first1(std::move(_first1)), + m_first2(std::move(_first2)), + m_predicate(std::move(_predicate)) {} +}; + +// +// exespace impl +// +template +bool equal_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, BinaryPredicateType predicate) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first1, first2); + Impl::static_assert_iterators_have_matching_difference_type(first1, first2); + Impl::expect_valid_range(first1, last1); + + // run + const auto num_elements = Kokkos::Experimental::distance(first1, last1); + std::size_t different = 0; + ::Kokkos::parallel_reduce( + label, RangePolicy(ex, 0, num_elements), + StdEqualFunctor(first1, first2, predicate), different); + ex.fence("Kokkos::equal: fence after operation"); + + return !different; +} + +template +bool equal_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2) { + using value_type1 = typename IteratorType1::value_type; + using value_type2 = typename IteratorType2::value_type; + using pred_t = StdAlgoEqualBinaryPredicate; + return equal_exespace_impl(label, ex, first1, last1, first2, pred_t()); +} + +template +bool equal_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType predicate) { + const auto d1 = ::Kokkos::Experimental::distance(first1, last1); + const auto d2 = ::Kokkos::Experimental::distance(first2, last2); + if (d1 != d2) { + return false; + } + + return equal_exespace_impl(label, ex, first1, last1, first2, predicate); +} + +template +bool equal_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2) { + Impl::expect_valid_range(first1, last1); + Impl::expect_valid_range(first2, last2); + + using value_type1 = typename IteratorType1::value_type; + using value_type2 = typename IteratorType2::value_type; + using pred_t = StdAlgoEqualBinaryPredicate; + return equal_exespace_impl(label, ex, first1, last1, first2, last2, pred_t()); +} + +// +// team impl +// +template +KOKKOS_FUNCTION bool equal_team_impl(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, + BinaryPredicateType predicate) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first1, first2); + Impl::static_assert_iterators_have_matching_difference_type(first1, first2); + Impl::expect_valid_range(first1, last1); + + // run + const auto num_elements = Kokkos::Experimental::distance(first1, last1); + std::size_t different = 0; + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, num_elements), + StdEqualFunctor(first1, first2, predicate), + different); + teamHandle.team_barrier(); + + return !different; +} + +template +KOKKOS_FUNCTION bool equal_team_impl(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2) { + using value_type1 = typename IteratorType1::value_type; + using value_type2 = typename IteratorType2::value_type; + using pred_t = StdAlgoEqualBinaryPredicate; + return equal_team_impl(teamHandle, first1, last1, first2, pred_t()); +} + +template +KOKKOS_FUNCTION bool equal_team_impl(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, + BinaryPredicateType predicate) { + const auto d1 = ::Kokkos::Experimental::distance(first1, last1); + const auto d2 = ::Kokkos::Experimental::distance(first2, last2); + if (d1 != d2) { + return false; + } + + return equal_team_impl(teamHandle, first1, last1, first2, predicate); +} + +template +KOKKOS_FUNCTION bool equal_team_impl(const TeamHandleType& teamHandle, + IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, + IteratorType2 last2) { + Impl::expect_valid_range(first1, last1); + Impl::expect_valid_range(first2, last2); + + using value_type1 = typename IteratorType1::value_type; + using value_type2 = typename IteratorType2::value_type; + using pred_t = StdAlgoEqualBinaryPredicate; + return equal_team_impl(teamHandle, first1, last1, first2, last2, pred_t()); +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_ExclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_ExclusiveScan.hpp new file mode 100644 index 0000000000..6da992b4bb --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_ExclusiveScan.hpp @@ -0,0 +1,200 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_EXCLUSIVE_SCAN_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include "Kokkos_ValueWrapperForNoNeutralElement.hpp" +#include "Kokkos_IdentityReferenceUnaryFunctor.hpp" +#include "Kokkos_FunctorsForExclusiveScan.hpp" +#include +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +// +// exespace impl +// +template +OutputIteratorType exclusive_scan_default_op_exespace_impl( + const std::string& label, const ExecutionSpace& ex, + InputIteratorType first_from, InputIteratorType last_from, + OutputIteratorType first_dest, ValueType init_value) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + // does it make sense to do this static_assert too? + // using input_iterator_value_type = typename InputIteratorType::value_type; + // static_assert + // (std::is_convertible, + // ValueType>::value, + // "exclusive_scan: InputIteratorType::value_type not convertible to + // ValueType"); + + // we are unnecessarily duplicating code, but this is on purpose + // so that we can use the default_op for OpenMPTarget. + // Originally, I had this implemented as: + // ''' + // using bop_type = StdExclusiveScanDefaultJoinFunctor; + // call exclusive_scan_custom_op_impl(..., bop_type()); + // ''' + // which avoids duplicating the functors, but for OpenMPTarget + // I cannot use a custom binary op. + // This is the same problem that occurs for reductions. + + // aliases + using index_type = typename InputIteratorType::difference_type; + using func_type = std::conditional_t< + ::Kokkos::is_detected::value, + ExclusiveScanDefaultFunctorForKnownNeutralElement< + ExecutionSpace, index_type, ValueType, InputIteratorType, + OutputIteratorType>, + ExclusiveScanDefaultFunctorWithValueWrapper>; + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_scan( + label, RangePolicy(ex, 0, num_elements), + func_type(std::move(init_value), first_from, first_dest)); + + ex.fence("Kokkos::exclusive_scan_default_op: fence after operation"); + + return first_dest + num_elements; +} + +template +OutputIteratorType exclusive_scan_custom_op_exespace_impl( + const std::string& label, const ExecutionSpace& ex, + InputIteratorType first_from, InputIteratorType last_from, + OutputIteratorType first_dest, ValueType init_value, BinaryOpType bop) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first_from, first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + // aliases + using index_type = typename InputIteratorType::difference_type; + using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; + using func_type = TransformExclusiveScanFunctorWithValueWrapper< + ExecutionSpace, index_type, ValueType, InputIteratorType, + OutputIteratorType, BinaryOpType, unary_op_type>; + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_scan(label, + RangePolicy(ex, 0, num_elements), + func_type(std::move(init_value), first_from, + first_dest, bop, unary_op_type())); + ex.fence("Kokkos::exclusive_scan_custom_op: fence after operation"); + + // return + return first_dest + num_elements; +} + +// +// team impl +// +template +KOKKOS_FUNCTION OutputIteratorType exclusive_scan_default_op_team_impl( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest, + ValueType init_value) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + static_assert( + ::Kokkos::is_detected_v, + "The team-level impl of Kokkos::Experimental::exclusive_scan currently " + "does not support types without reduction identity"); + + // aliases + using exe_space = typename TeamHandleType::execution_space; + using index_type = typename InputIteratorType::difference_type; + using func_type = ExclusiveScanDefaultFunctorForKnownNeutralElement< + exe_space, index_type, ValueType, InputIteratorType, OutputIteratorType>; + + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_scan( + TeamThreadRange(teamHandle, 0, num_elements), + func_type(std::move(init_value), first_from, first_dest)); + teamHandle.team_barrier(); + return first_dest + num_elements; +} + +template +KOKKOS_FUNCTION OutputIteratorType exclusive_scan_custom_op_team_impl( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest, + ValueType init_value, BinaryOpType bop) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + static_assert( + ::Kokkos::is_detected_v, + "The team-level impl of Kokkos::Experimental::exclusive_scan currently " + "does not support types without reduction identity"); + + // aliases + using exe_space = typename TeamHandleType::execution_space; + using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; + using index_type = typename InputIteratorType::difference_type; + using func_type = TransformExclusiveScanFunctorWithoutValueWrapper< + exe_space, index_type, ValueType, InputIteratorType, OutputIteratorType, + BinaryOpType, unary_op_type>; + + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_scan(TeamThreadRange(teamHandle, 0, num_elements), + func_type(std::move(init_value), first_from, + first_dest, bop, unary_op_type())); + teamHandle.team_barrier(); + + return first_dest + num_elements; +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FillFillN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FillFillN.hpp new file mode 100644 index 0000000000..972e57f2cc --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FillFillN.hpp @@ -0,0 +1,115 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FILL_AND_FILL_N_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_FILL_AND_FILL_N_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdFillFunctor { + using index_type = typename InputIterator::difference_type; + InputIterator m_first; + T m_value; + + KOKKOS_FUNCTION + void operator()(index_type i) const { m_first[i] = m_value; } + + KOKKOS_FUNCTION + StdFillFunctor(InputIterator _first, T _value) + : m_first(std::move(_first)), m_value(std::move(_value)) {} +}; + +// +// exespace impl +// +template +void fill_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, const T& value) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(label, + RangePolicy(ex, 0, num_elements), + StdFillFunctor(first, value)); + ex.fence("Kokkos::fill: fence after operation"); +} + +template +IteratorType fill_n_exespace_impl(const std::string& label, + const ExecutionSpace& ex, IteratorType first, + SizeType n, const T& value) { + auto last = first + n; + Impl::static_assert_random_access_and_accessible(ex, first); + Impl::expect_valid_range(first, last); + + if (n <= 0) { + return first; + } + + fill_exespace_impl(label, ex, first, last, value); + return last; +} + +// +// team-level impl +// +template +KOKKOS_FUNCTION void fill_team_impl(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + const T& value) { + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(TeamThreadRange(teamHandle, 0, num_elements), + StdFillFunctor(first, value)); + + teamHandle.team_barrier(); +} + +template +KOKKOS_FUNCTION IteratorType fill_n_team_impl(const TeamHandleType& teamHandle, + IteratorType first, SizeType n, + const T& value) { + auto last = first + n; + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + if (n <= 0) { + return first; + } + + fill_team_impl(teamHandle, first, last, value); + return last; +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindEnd.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindEnd.hpp similarity index 50% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindEnd.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindEnd.hpp index 35a6c4b4ac..1f1ec5e54f 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindEnd.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindEnd.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_FIND_END_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_FIND_END_IMPL_HPP @@ -87,9 +59,11 @@ struct StdFindEndFunctor { } } - const auto rv = - found ? red_value_type{i} - : red_value_type{::Kokkos::reduction_identity::max()}; + // FIXME_NVHPC using a ternary operator causes problems + red_value_type rv = {::Kokkos::reduction_identity::max()}; + if (found) { + rv.max_loc_true = i; + } m_reducer.join(red_value, rv); } @@ -106,12 +80,17 @@ struct StdFindEndFunctor { m_p(std::move(p)) {} }; +// +// exespace impl +// template -IteratorType1 find_end_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last, - const BinaryPredicateType& pred) { +IteratorType1 find_end_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last, + const BinaryPredicateType& pred) { // checks Impl::static_assert_random_access_and_accessible(ex, first, s_first); Impl::static_assert_iterators_have_matching_difference_type(first, s_first); @@ -123,7 +102,6 @@ IteratorType1 find_end_impl(const std::string& label, const ExecutionSpace& ex, const auto num_elements = KE::distance(first, last); const auto s_count = KE::distance(s_first, s_last); KOKKOS_EXPECTS(num_elements >= s_count); - (void)s_count; // needed when macro above is a no-op if (s_first == s_last) { return last; @@ -135,7 +113,8 @@ IteratorType1 find_end_impl(const std::string& label, const ExecutionSpace& ex, // special case where the two ranges have equal size if (num_elements == s_count) { - const auto equal_result = equal_impl(label, ex, first, last, s_first, pred); + const auto equal_result = + equal_exespace_impl(label, ex, first, last, s_first, pred); return (equal_result) ? first : last; } else { using index_type = typename IteratorType1::difference_type; @@ -174,14 +153,97 @@ IteratorType1 find_end_impl(const std::string& label, const ExecutionSpace& ex, } template -IteratorType1 find_end_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType1 first, IteratorType1 last, - IteratorType2 s_first, IteratorType2 s_last) { +IteratorType1 find_end_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, + IteratorType2 s_last) { + using value_type1 = typename IteratorType1::value_type; + using value_type2 = typename IteratorType2::value_type; + using predicate_type = StdAlgoEqualBinaryPredicate; + return find_end_exespace_impl(label, ex, first, last, s_first, s_last, + predicate_type()); +} + +// +// team impl +// +template +KOKKOS_FUNCTION IteratorType1 +find_end_team_impl(const TeamHandleType& teamHandle, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last, const BinaryPredicateType& pred) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first, s_first); + Impl::static_assert_iterators_have_matching_difference_type(first, s_first); + Impl::expect_valid_range(first, last); + Impl::expect_valid_range(s_first, s_last); + + // the target sequence should not be larger than the range [first, last) + namespace KE = ::Kokkos::Experimental; + const auto num_elements = KE::distance(first, last); + const auto s_count = KE::distance(s_first, s_last); + KOKKOS_EXPECTS(num_elements >= s_count); + + if (s_first == s_last) { + return last; + } + + if (first == last) { + return last; + } + + // special case where the two ranges have equal size + if (num_elements == s_count) { + const auto equal_result = + equal_team_impl(teamHandle, first, last, s_first, pred); + return (equal_result) ? first : last; + } else { + using index_type = typename IteratorType1::difference_type; + using reducer_type = LastLoc; + using reduction_value_type = typename reducer_type::value_type; + using func_t = StdFindEndFunctor; + + // run + reduction_value_type red_result; + reducer_type reducer(red_result); + + // decide the size of the range policy of the par_red: + // note that the last feasible index to start looking is the index + // whose distance from the "last" is equal to the sequence count. + // the +1 is because we need to include that location too. + const auto range_size = num_elements - s_count + 1; + + // run par reduce + ::Kokkos::parallel_reduce( + TeamThreadRange(teamHandle, 0, range_size), + func_t(first, last, s_first, s_last, reducer, pred), reducer); + + teamHandle.team_barrier(); + + // decide and return + if (red_result.max_loc_true == + ::Kokkos::reduction_identity::max()) { + // if here, a subrange has not been found + return last; + } else { + // a location has been found + return first + red_result.max_loc_true; + } + } +} + +template +KOKKOS_FUNCTION IteratorType1 find_end_team_impl( + const TeamHandleType& teamHandle, IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last) { using value_type1 = typename IteratorType1::value_type; using value_type2 = typename IteratorType2::value_type; using predicate_type = StdAlgoEqualBinaryPredicate; - return find_end_impl(label, ex, first, last, s_first, s_last, - predicate_type()); + return find_end_team_impl(teamHandle, first, last, s_first, s_last, + predicate_type()); } } // namespace Impl diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindFirstOf.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindFirstOf.hpp similarity index 51% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindFirstOf.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindFirstOf.hpp index 6907bbdbc3..145e235b9d 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindFirstOf.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindFirstOf.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_FIND_FIRST_OF_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_FIND_FIRST_OF_IMPL_HPP @@ -80,10 +52,11 @@ struct StdFindFirstOfFunctor { } } - const auto rv = - found ? red_value_type{i} - : red_value_type{::Kokkos::reduction_identity::min()}; - + // FIXME_NVHPC using a ternary operator causes problems + red_value_type rv = {::Kokkos::reduction_identity::min()}; + if (found) { + rv.min_loc_true = i; + } m_reducer.join(red_value, rv); } @@ -98,13 +71,15 @@ struct StdFindFirstOfFunctor { m_p(std::move(p)) {} }; +// +// exespace impl +// template -IteratorType1 find_first_of_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last, - const BinaryPredicateType& pred) { +IteratorType1 find_first_of_exespace_impl( + const std::string& label, const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, IteratorType2 s_last, + const BinaryPredicateType& pred) { // checks Impl::static_assert_random_access_and_accessible(ex, first, s_first); Impl::static_assert_iterators_have_matching_difference_type(first, s_first); @@ -143,15 +118,71 @@ IteratorType1 find_first_of_impl(const std::string& label, } template -IteratorType1 find_first_of_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType1 first, - IteratorType1 last, IteratorType2 s_first, - IteratorType2 s_last) { +IteratorType1 find_first_of_exespace_impl( + const std::string& label, const ExecutionSpace& ex, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, IteratorType2 s_last) { + using value_type1 = typename IteratorType1::value_type; + using value_type2 = typename IteratorType2::value_type; + using predicate_type = StdAlgoEqualBinaryPredicate; + return find_first_of_exespace_impl(label, ex, first, last, s_first, s_last, + predicate_type()); +} + +// +// team impl +// +template +KOKKOS_FUNCTION IteratorType1 +find_first_of_team_impl(const TeamHandleType& teamHandle, IteratorType1 first, + IteratorType1 last, IteratorType2 s_first, + IteratorType2 s_last, const BinaryPredicateType& pred) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first, s_first); + Impl::static_assert_iterators_have_matching_difference_type(first, s_first); + Impl::expect_valid_range(first, last); + Impl::expect_valid_range(s_first, s_last); + + if ((s_first == s_last) || (first == last)) { + return last; + } + + using index_type = typename IteratorType1::difference_type; + using reducer_type = FirstLoc; + using reduction_value_type = typename reducer_type::value_type; + using func_t = StdFindFirstOfFunctor; + + // run + reduction_value_type red_result; + reducer_type reducer(red_result); + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, num_elements), + func_t(first, s_first, s_last, reducer, pred), + reducer); + + teamHandle.team_barrier(); + + // decide and return + if (red_result.min_loc_true == + ::Kokkos::reduction_identity::min()) { + // if here, nothing found + return last; + } else { + // a location has been found + return first + red_result.min_loc_true; + } +} + +template +KOKKOS_FUNCTION IteratorType1 find_first_of_team_impl( + const TeamHandleType& teamHandle, IteratorType1 first, IteratorType1 last, + IteratorType2 s_first, IteratorType2 s_last) { using value_type1 = typename IteratorType1::value_type; using value_type2 = typename IteratorType2::value_type; using predicate_type = StdAlgoEqualBinaryPredicate; - return find_first_of_impl(label, ex, first, last, s_first, s_last, - predicate_type()); + return find_first_of_team_impl(teamHandle, first, last, s_first, s_last, + predicate_type()); } } // namespace Impl diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindIfOrNot.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindIfOrNot.hpp similarity index 51% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindIfOrNot.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindIfOrNot.hpp index c79c4b5216..8fffb59094 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_FindIfOrNot.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FindIfOrNot.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_FIND_IF_AND_FIND_IF_NOT_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_FIND_IF_AND_FIND_IF_NOT_IMPL_HPP @@ -72,10 +44,11 @@ struct StdFindIfOrNotFunctor { // if doing find_if_not, look for when predicate is false const bool found_condition = is_find_if ? m_p(my_value) : !m_p(my_value); - auto rv = - found_condition - ? red_value_type{i} - : red_value_type{::Kokkos::reduction_identity::min()}; + // FIXME_NVHPC using a ternary operator causes problems + red_value_type rv = {::Kokkos::reduction_identity::min()}; + if (found_condition) { + rv.min_loc_true = i; + } m_reducer.join(red_value, rv); } @@ -88,11 +61,15 @@ struct StdFindIfOrNotFunctor { m_p(std::move(p)) {} }; +// +// exespace impl +// template -IteratorType find_if_or_not_impl(const std::string& label, - const ExecutionSpace& ex, IteratorType first, - IteratorType last, PredicateType pred) { +IteratorType find_if_or_not_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, IteratorType last, + PredicateType pred) { // checks Impl::static_assert_random_access_and_accessible( ex, first); // only need one It per type @@ -131,14 +108,68 @@ IteratorType find_if_or_not_impl(const std::string& label, } template -InputIterator find_impl(const std::string& label, ExecutionSpace ex, - InputIterator first, InputIterator last, - const T& value) { - return find_if_or_not_impl( +InputIterator find_exespace_impl(const std::string& label, ExecutionSpace ex, + InputIterator first, InputIterator last, + const T& value) { + return find_if_or_not_exespace_impl( label, ex, first, last, ::Kokkos::Experimental::Impl::StdAlgoEqualsValUnaryPredicate(value)); } +// +// team impl +// +template +KOKKOS_FUNCTION IteratorType +find_if_or_not_team_impl(const TeamHandleType& teamHandle, IteratorType first, + IteratorType last, PredicateType pred) { + // checks + Impl::static_assert_random_access_and_accessible( + teamHandle, first); // only need one It per type + Impl::expect_valid_range(first, last); + + if (first == last) { + return last; + } + + // aliases + using index_type = typename IteratorType::difference_type; + using reducer_type = FirstLoc; + using reduction_value_type = typename reducer_type::value_type; + using func_t = StdFindIfOrNotFunctor; + + // run + reduction_value_type red_result; + reducer_type reducer(red_result); + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, num_elements), + func_t(first, reducer, pred), reducer); + + teamHandle.team_barrier(); + + // decide and return + if (red_result.min_loc_true == + ::Kokkos::reduction_identity::min()) { + // here, it means a valid loc has not been found, + return last; + } else { + // a location has been found + return first + red_result.min_loc_true; + } +} + +template +KOKKOS_FUNCTION InputIterator find_team_impl(const TeamHandleType& teamHandle, + InputIterator first, + InputIterator last, + const T& value) { + return find_if_or_not_team_impl( + teamHandle, first, last, + ::Kokkos::Experimental::Impl::StdAlgoEqualsValUnaryPredicate(value)); +} + } // namespace Impl } // namespace Experimental } // namespace Kokkos diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_ForEachForEachN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_ForEachForEachN.hpp new file mode 100644 index 0000000000..99cc4a1cf3 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_ForEachForEachN.hpp @@ -0,0 +1,121 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FOR_EACH_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_FOR_EACH_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdForEachFunctor { + using index_type = typename IteratorType::difference_type; + IteratorType m_first; + UnaryFunctorType m_functor; + + KOKKOS_FUNCTION + void operator()(index_type i) const { m_functor(m_first[i]); } + + KOKKOS_FUNCTION + StdForEachFunctor(IteratorType _first, UnaryFunctorType _functor) + : m_first(std::move(_first)), m_functor(std::move(_functor)) {} +}; + +template +void for_each_exespace_impl(const std::string& label, const HandleType& handle, + IteratorType first, IteratorType last, + UnaryFunctorType functor) { + // checks + Impl::static_assert_random_access_and_accessible(handle, first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for( + label, RangePolicy(handle, 0, num_elements), + StdForEachFunctor(first, functor)); + handle.fence("Kokkos::for_each: fence after operation"); +} + +template +IteratorType for_each_n_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, SizeType n, + UnaryFunctorType functor) { + auto last = first + n; + Impl::static_assert_random_access_and_accessible(ex, first, last); + Impl::expect_valid_range(first, last); + + if (n == 0) { + return first; + } + + for_each_exespace_impl(label, ex, first, last, std::move(functor)); + // no need to fence since for_each_exespace_impl fences already + + return last; +} + +// +// team impl +// +template +KOKKOS_FUNCTION void for_each_team_impl(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + UnaryFunctorType functor) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for( + TeamThreadRange(teamHandle, 0, num_elements), + StdForEachFunctor(first, functor)); + teamHandle.team_barrier(); +} + +template +KOKKOS_FUNCTION IteratorType +for_each_n_team_impl(const TeamHandleType& teamHandle, IteratorType first, + SizeType n, UnaryFunctorType functor) { + auto last = first + n; + Impl::static_assert_random_access_and_accessible(teamHandle, first, last); + Impl::expect_valid_range(first, last); + + if (n == 0) { + return first; + } + + for_each_team_impl(teamHandle, first, last, std::move(functor)); + // no need to fence since for_each_team_impl fences already + + return last; +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FunctorsForExclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FunctorsForExclusiveScan.hpp new file mode 100644 index 0000000000..5a7fe16984 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_FunctorsForExclusiveScan.hpp @@ -0,0 +1,221 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_FUNCTORS_FOR_EXCLUSIVE_SCAN_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_FUNCTORS_FOR_EXCLUSIVE_SCAN_IMPL_HPP + +#include +#include "Kokkos_ValueWrapperForNoNeutralElement.hpp" + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +using ex_scan_has_reduction_identity_sum_t = + decltype(Kokkos::reduction_identity::sum()); + +template +struct ExclusiveScanDefaultFunctorForKnownNeutralElement { + using execution_space = ExeSpace; + ValueType m_init_value; + FirstFrom m_first_from; + FirstDest m_first_dest; + + KOKKOS_FUNCTION + ExclusiveScanDefaultFunctorForKnownNeutralElement(ValueType init, + FirstFrom first_from, + FirstDest first_dest) + : m_init_value(std::move(init)), + m_first_from(std::move(first_from)), + m_first_dest(std::move(first_dest)) {} + + KOKKOS_FUNCTION + void operator()(const IndexType i, ValueType& update, + const bool final_pass) const { + const auto tmp = m_first_from[i]; + if (final_pass) m_first_dest[i] = update + m_init_value; + update += tmp; + } +}; + +template +struct ExclusiveScanDefaultFunctorWithValueWrapper { + using execution_space = ExeSpace; + using value_type = + ::Kokkos::Experimental::Impl::ValueWrapperForNoNeutralElement; + ValueType m_init_value; + FirstFrom m_first_from; + FirstDest m_first_dest; + + KOKKOS_FUNCTION + ExclusiveScanDefaultFunctorWithValueWrapper(ValueType init, + FirstFrom first_from, + FirstDest first_dest) + : m_init_value(std::move(init)), + m_first_from(std::move(first_from)), + m_first_dest(std::move(first_dest)) {} + + KOKKOS_FUNCTION + void operator()(const IndexType i, value_type& update, + const bool final_pass) const { + const auto tmp = value_type{m_first_from[i], false}; + if (final_pass) { + if (i == 0) { + m_first_dest[i] = m_init_value; + } else { + m_first_dest[i] = update.val + m_init_value; + } + } + + this->join(update, tmp); + } + + KOKKOS_FUNCTION + void init(value_type& update) const { + update.val = {}; + update.is_initial = true; + } + + KOKKOS_FUNCTION + void join(value_type& update, const value_type& input) const { + if (input.is_initial) return; + + if (update.is_initial) { + update.val = input.val; + update.is_initial = false; + } else { + update.val = update.val + input.val; + } + } +}; + +template +struct TransformExclusiveScanFunctorWithValueWrapper { + using execution_space = ExeSpace; + using value_type = + ::Kokkos::Experimental::Impl::ValueWrapperForNoNeutralElement; + + ValueType m_init_value; + FirstFrom m_first_from; + FirstDest m_first_dest; + BinaryOpType m_binary_op; + UnaryOpType m_unary_op; + + KOKKOS_FUNCTION + TransformExclusiveScanFunctorWithValueWrapper(ValueType init, + FirstFrom first_from, + FirstDest first_dest, + BinaryOpType bop, + UnaryOpType uop) + : m_init_value(std::move(init)), + m_first_from(std::move(first_from)), + m_first_dest(std::move(first_dest)), + m_binary_op(std::move(bop)), + m_unary_op(std::move(uop)) {} + + KOKKOS_FUNCTION + void operator()(const IndexType i, value_type& update, + const bool final_pass) const { + const auto tmp = value_type{m_unary_op(m_first_from[i]), false}; + if (final_pass) { + if (i == 0) { + // for both ExclusiveScan and TransformExclusiveScan, + // init is unmodified + m_first_dest[i] = m_init_value; + } else { + m_first_dest[i] = m_binary_op(update.val, m_init_value); + } + } + + this->join(update, tmp); + } + + KOKKOS_FUNCTION void init(value_type& value) const { + value.val = {}; + value.is_initial = true; + } + + KOKKOS_FUNCTION + void join(value_type& update, const value_type& input) const { + if (input.is_initial) return; + + if (update.is_initial) { + update.val = input.val; + } else { + update.val = m_binary_op(update.val, input.val); + } + update.is_initial = false; + } +}; + +template +struct TransformExclusiveScanFunctorWithoutValueWrapper { + using execution_space = ExeSpace; + + ValueType m_init_value; + FirstFrom m_first_from; + FirstDest m_first_dest; + BinaryOpType m_binary_op; + UnaryOpType m_unary_op; + + KOKKOS_FUNCTION + TransformExclusiveScanFunctorWithoutValueWrapper(ValueType init, + FirstFrom first_from, + FirstDest first_dest, + BinaryOpType bop, + UnaryOpType uop) + : m_init_value(std::move(init)), + m_first_from(std::move(first_from)), + m_first_dest(std::move(first_dest)), + m_binary_op(std::move(bop)), + m_unary_op(std::move(uop)) {} + + KOKKOS_FUNCTION + void operator()(const IndexType i, ValueType& update, + const bool final_pass) const { + const auto tmp = ValueType{m_unary_op(m_first_from[i])}; + if (final_pass) { + if (i == 0) { + // for both ExclusiveScan and TransformExclusiveScan, + // init is unmodified + m_first_dest[i] = m_init_value; + } else { + m_first_dest[i] = m_binary_op(update, m_init_value); + } + } + + this->join(update, tmp); + } + + KOKKOS_FUNCTION + void init(ValueType& update) const { update = {}; } + + KOKKOS_FUNCTION + void join(ValueType& update, const ValueType& input) const { + update = m_binary_op(update, input); + } +}; + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_GenerateGenerateN.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_GenerateGenerateN.hpp new file mode 100644 index 0000000000..157de1125e --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_GenerateGenerateN.hpp @@ -0,0 +1,110 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_GENERATE_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_GENERATE_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdGenerateFunctor { + using index_type = typename IteratorType::difference_type; + IteratorType m_first; + Generator m_generator; + + KOKKOS_FUNCTION + void operator()(index_type i) const { m_first[i] = m_generator(); } + + KOKKOS_FUNCTION + StdGenerateFunctor(IteratorType _first, Generator _g) + : m_first(std::move(_first)), m_generator(std::move(_g)) {} +}; + +// +// generate impl +// +template +void generate_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + Generator g) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(label, + RangePolicy(ex, 0, num_elements), + StdGenerateFunctor(first, g)); + ex.fence("Kokkos::generate: fence after operation"); +} + +template +KOKKOS_FUNCTION void generate_team_impl(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + Generator g) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + // run + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_for(TeamThreadRange(teamHandle, 0, num_elements), + StdGenerateFunctor(first, g)); + teamHandle.team_barrier(); +} + +// +// generate_n impl +// +template +IteratorType generate_n_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, Size count, + Generator g) { + if (count <= 0) { + return first; + } + + generate_exespace_impl(label, ex, first, first + count, g); + return first + count; +} + +template +KOKKOS_FUNCTION IteratorType +generate_n_team_impl(const TeamHandleType& teamHandle, IteratorType first, + Size count, Generator g) { + if (count <= 0) { + return first; + } + + generate_team_impl(teamHandle, first, first + count, g); + return first + count; +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_HelperPredicates.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_HelperPredicates.hpp similarity index 56% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_HelperPredicates.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_HelperPredicates.hpp index 244bce48e4..db0def6f22 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_HelperPredicates.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_HelperPredicates.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_HELPER_PREDICATES_HPP #define KOKKOS_STD_ALGORITHMS_HELPER_PREDICATES_HPP diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IdentityReferenceUnaryFunctor.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IdentityReferenceUnaryFunctor.hpp new file mode 100644 index 0000000000..252511c5d0 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IdentityReferenceUnaryFunctor.hpp @@ -0,0 +1,36 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_NUMERIC_IDENTITY_REFERENCE_UNARY_FUNCTOR_HPP +#define KOKKOS_STD_ALGORITHMS_NUMERIC_IDENTITY_REFERENCE_UNARY_FUNCTOR_HPP + +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdNumericScanIdentityReferenceUnaryFunctor { + KOKKOS_FUNCTION + constexpr const ValueType& operator()(const ValueType& a) const { return a; } +}; + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_InclusiveScan.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_InclusiveScan.hpp similarity index 54% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_InclusiveScan.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_InclusiveScan.hpp index 2088ebd439..0b4acec0fe 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_InclusiveScan.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_InclusiveScan.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_INCLUSIVE_SCAN_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_INCLUSIVE_SCAN_IMPL_HPP @@ -118,6 +90,8 @@ struct InclusiveScanDefaultFunctor { KOKKOS_FUNCTION void join(value_type& update, const value_type& input) const { + if (input.is_initial) return; + if (update.is_initial) { update.val = input.val; } else { @@ -127,9 +101,12 @@ struct InclusiveScanDefaultFunctor { } }; +// +// exespace impl +// template -OutputIteratorType inclusive_scan_default_op_impl( +OutputIteratorType inclusive_scan_default_op_exespace_impl( const std::string& label, const ExecutionSpace& ex, InputIteratorType first_from, InputIteratorType last_from, OutputIteratorType first_dest) { @@ -169,7 +146,7 @@ OutputIteratorType inclusive_scan_default_op_impl( // ------------------------------------------------------------- template -OutputIteratorType inclusive_scan_custom_binary_op_impl( +OutputIteratorType inclusive_scan_custom_binary_op_exespace_impl( const std::string& label, const ExecutionSpace& ex, InputIteratorType first_from, InputIteratorType last_from, OutputIteratorType first_dest, BinaryOpType binary_op) { @@ -184,7 +161,7 @@ OutputIteratorType inclusive_scan_custom_binary_op_impl( using value_type = std::remove_const_t; using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; - using func_type = TransformInclusiveScanNoInitValueFunctor< + using func_type = ExeSpaceTransformInclusiveScanNoInitValueFunctor< ExecutionSpace, index_type, value_type, InputIteratorType, OutputIteratorType, BinaryOpType, unary_op_type>; @@ -205,7 +182,7 @@ OutputIteratorType inclusive_scan_custom_binary_op_impl( // ------------------------------------------------------------- template -OutputIteratorType inclusive_scan_custom_binary_op_impl( +OutputIteratorType inclusive_scan_custom_binary_op_exespace_impl( const std::string& label, const ExecutionSpace& ex, InputIteratorType first_from, InputIteratorType last_from, OutputIteratorType first_dest, BinaryOpType binary_op, @@ -219,7 +196,7 @@ OutputIteratorType inclusive_scan_custom_binary_op_impl( // aliases using index_type = typename InputIteratorType::difference_type; using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; - using func_type = TransformInclusiveScanWithInitValueFunctor< + using func_type = ExeSpaceTransformInclusiveScanWithInitValueFunctor< ExecutionSpace, index_type, ValueType, InputIteratorType, OutputIteratorType, BinaryOpType, unary_op_type>; @@ -229,13 +206,142 @@ OutputIteratorType inclusive_scan_custom_binary_op_impl( ::Kokkos::parallel_scan(label, RangePolicy(ex, 0, num_elements), func_type(first_from, first_dest, binary_op, - unary_op_type(), init_value)); + unary_op_type(), std::move(init_value))); ex.fence("Kokkos::inclusive_scan_custom_binary_op: fence after operation"); // return return first_dest + num_elements; } +// +// team impl +// +template +KOKKOS_FUNCTION OutputIteratorType inclusive_scan_default_op_team_impl( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + using value_type = + std::remove_const_t; + + // #if defined(KOKKOS_ENABLE_CUDA) + + using exe_space = typename TeamHandleType::execution_space; + using index_type = typename InputIteratorType::difference_type; + using func_type = std::conditional_t< + ::Kokkos::is_detected::value, + InclusiveScanDefaultFunctorForKnownIdentityElement< + exe_space, index_type, value_type, InputIteratorType, + OutputIteratorType>, + InclusiveScanDefaultFunctor>; + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_scan(TeamThreadRange(teamHandle, 0, num_elements), + func_type(first_from, first_dest)); + teamHandle.team_barrier(); + + // return + return first_dest + num_elements; +} + +// ------------------------------------------------------------- +// inclusive_scan_custom_binary_op_impl +// ------------------------------------------------------------- +template +KOKKOS_FUNCTION OutputIteratorType inclusive_scan_custom_binary_op_team_impl( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest, + BinaryOpType binary_op) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + using value_type = + std::remove_const_t; + + static_assert( + ::Kokkos::is_detected_v, + "At the moment inclusive_scan doesn't support types without reduction " + "identity"); + + // #if defined(KOKKOS_ENABLE_CUDA) + + // aliases + using exe_space = typename TeamHandleType::execution_space; + using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; + using func_type = TeamTransformInclusiveScanNoInitValueFunctor< + exe_space, value_type, InputIteratorType, OutputIteratorType, + BinaryOpType, unary_op_type>; + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + + ::Kokkos::parallel_scan( + TeamThreadRange(teamHandle, 0, num_elements), + func_type(first_from, first_dest, binary_op, unary_op_type())); + teamHandle.team_barrier(); + + return first_dest + num_elements; +} + +// ------------------------------------------------------------- +// inclusive_scan_custom_binary_op_impl with init_value +// ------------------------------------------------------------- +template +KOKKOS_FUNCTION OutputIteratorType inclusive_scan_custom_binary_op_team_impl( + const TeamHandleType& teamHandle, InputIteratorType first_from, + InputIteratorType last_from, OutputIteratorType first_dest, + BinaryOpType binary_op, ValueType init_value) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first_from, + first_dest); + Impl::static_assert_iterators_have_matching_difference_type(first_from, + first_dest); + Impl::expect_valid_range(first_from, last_from); + + static_assert( + ::Kokkos::is_detected_v, + "At the moment inclusive_scan doesn't support types without reduction " + "identity"); + + // #if defined(KOKKOS_ENABLE_CUDA) + + // aliases + using exe_space = typename TeamHandleType::execution_space; + using unary_op_type = StdNumericScanIdentityReferenceUnaryFunctor; + using func_type = TeamTransformInclusiveScanWithInitValueFunctor< + exe_space, ValueType, InputIteratorType, OutputIteratorType, BinaryOpType, + unary_op_type>; + + // run + const auto num_elements = + Kokkos::Experimental::distance(first_from, last_from); + ::Kokkos::parallel_scan(TeamThreadRange(teamHandle, 0, num_elements), + func_type(first_from, first_dest, binary_op, + unary_op_type(), std::move(init_value))); + teamHandle.team_barrier(); + + // return + return first_dest + num_elements; +} + } // namespace Impl } // namespace Experimental } // namespace Kokkos diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsPartitioned.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsPartitioned.hpp similarity index 52% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsPartitioned.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsPartitioned.hpp index 0f00bebb6d..281efca36b 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsPartitioned.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsPartitioned.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_IS_PARTITIONED_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_IS_PARTITIONED_IMPL_HPP @@ -71,8 +43,12 @@ struct StdIsPartitionedFunctor { ::Kokkos::reduction_identity::min(); constexpr index_type m_red_id_max = ::Kokkos::reduction_identity::max(); - auto rv = predicate_value ? red_value_type{i, m_red_id_min} - : red_value_type{m_red_id_max, i}; + + // FIXME_NVHPC using a ternary operator causes problems + red_value_type rv = {m_red_id_max, i}; + if (predicate_value) { + rv = {i, m_red_id_min}; + } m_reducer.join(redValue, rv); } @@ -86,9 +62,9 @@ struct StdIsPartitionedFunctor { }; template -bool is_partitioned_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - PredicateType pred) { +bool is_partitioned_exespace_impl(const std::string& label, + const ExecutionSpace& ex, IteratorType first, + IteratorType last, PredicateType pred) { // true if all elements in the range [first, last) that satisfy // the predicate "pred" appear before all elements that don't. // Also returns true if [first, last) is empty. @@ -121,6 +97,63 @@ bool is_partitioned_impl(const std::string& label, const ExecutionSpace& ex, const auto num_elements = Kokkos::Experimental::distance(first, last); ::Kokkos::parallel_reduce(label, RangePolicy(ex, 0, num_elements), + + func_t(first, reducer, pred), reducer); + + // fence not needed because reducing into scalar + + // decide and return + constexpr index_type red_id_min = + ::Kokkos::reduction_identity::min(); + constexpr index_type red_id_max = + ::Kokkos::reduction_identity::max(); + + if (red_result.max_loc_true != red_id_max && + red_result.min_loc_false != red_id_min) { + // this occurs when the reduction yields nontrivial values + return red_result.max_loc_true < red_result.min_loc_false; + } else if (red_result.max_loc_true == red_id_max && + red_result.min_loc_false == 0) { + // this occurs when all values do NOT satisfy + // the predicate, and this corner case should also be true + return true; + } else if (first + red_result.max_loc_true == --last) { + // this occurs when all values satisfy the predicate, + // this corner case should also be true + return true; + } else { + return false; + } +} + +template +KOKKOS_FUNCTION bool is_partitioned_team_impl(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last, + PredicateType pred) { + /* see exespace impl for the description of the impl */ + + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + // trivial case + if (first == last) { + return true; + } + + // aliases + using index_type = typename IteratorType::difference_type; + using reducer_type = StdIsPartitioned; + using reduction_value_type = typename reducer_type::value_type; + using func_t = + StdIsPartitionedFunctor; + + // run + reduction_value_type red_result; + reducer_type reducer(red_result); + const auto num_elements = Kokkos::Experimental::distance(first, last); + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, num_elements), func_t(first, reducer, pred), reducer); // fence not needed because reducing into scalar @@ -133,8 +166,16 @@ bool is_partitioned_impl(const std::string& label, const ExecutionSpace& ex, if (red_result.max_loc_true != red_id_max && red_result.min_loc_false != red_id_min) { + // this occurs when the reduction yields nontrivial values return red_result.max_loc_true < red_result.min_loc_false; + } else if (red_result.max_loc_true == red_id_max && + red_result.min_loc_false == 0) { + // this occurs when all values do NOT satisfy + // the predicate, and this corner case should also be true + return true; } else if (first + red_result.max_loc_true == --last) { + // this occurs when all values satisfy the predicate, + // this corner case should also be true return true; } else { return false; diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsSorted.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsSorted.hpp similarity index 51% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsSorted.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsSorted.hpp index 4e36ae3890..b2c912848a 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_IsSorted.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsSorted.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_IS_SORTED_IMPL_HPP @@ -76,10 +48,13 @@ struct StdIsSortedFunctor { : m_first(std::move(_first1)), m_comparator(std::move(comparator)) {} }; +// +// exespace impl +// template -bool is_sorted_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last, - ComparatorType comp) { +bool is_sorted_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last, + ComparatorType comp) { // checks Impl::static_assert_random_access_and_accessible(ex, first); Impl::expect_valid_range(first, last); @@ -103,11 +78,49 @@ bool is_sorted_impl(const std::string& label, const ExecutionSpace& ex, } template -bool is_sorted_impl(const std::string& label, const ExecutionSpace& ex, - IteratorType first, IteratorType last) { +bool is_sorted_exespace_impl(const std::string& label, const ExecutionSpace& ex, + IteratorType first, IteratorType last) { + using value_type = typename IteratorType::value_type; + using pred_t = Impl::StdAlgoLessThanBinaryPredicate; + return is_sorted_exespace_impl(label, ex, first, last, pred_t()); +} + +// +// team impl +// +template +KOKKOS_FUNCTION bool is_sorted_team_impl(const TeamHandleType& teamHandle, + IteratorType first, IteratorType last, + ComparatorType comp) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + const auto num_elements = Kokkos::Experimental::distance(first, last); + if (num_elements <= 1) { + return true; + } + + // use num_elements-1 because each index handles i and i+1 + const auto num_elements_minus_one = num_elements - 1; + + // result is incremented by one if sorting breaks at index i + std::size_t result = 0; + ::Kokkos::parallel_reduce( + TeamThreadRange(teamHandle, 0, num_elements_minus_one), + // use CTAD here + StdIsSortedFunctor(first, std::move(comp)), result); + + return result == 0; +} + +template +KOKKOS_FUNCTION bool is_sorted_team_impl(const TeamHandleType& teamHandle, + IteratorType first, + IteratorType last) { using value_type = typename IteratorType::value_type; using pred_t = Impl::StdAlgoLessThanBinaryPredicate; - return is_sorted_impl(label, ex, first, last, pred_t()); + return is_sorted_team_impl(teamHandle, first, last, pred_t()); } } // namespace Impl diff --git a/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsSortedUntil.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsSortedUntil.hpp new file mode 100644 index 0000000000..d33580ca53 --- /dev/null +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_IsSortedUntil.hpp @@ -0,0 +1,171 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_IMPL_HPP +#define KOKKOS_STD_ALGORITHMS_IS_SORTED_UNTIL_IMPL_HPP + +#include +#include "Kokkos_Constraints.hpp" +#include "Kokkos_HelperPredicates.hpp" +#include +#include +#include + +namespace Kokkos { +namespace Experimental { +namespace Impl { + +template +struct StdIsSortedUntilFunctor { + using index_type = typename IteratorType::difference_type; + using value_type = typename ReducerType::value_type; + + IteratorType m_first; + ComparatorType m_comparator; + ReducerType m_reducer; + + KOKKOS_FUNCTION + void operator()(const index_type i, value_type& reduction_result) const { + const auto& val_i = m_first[i]; + const auto& val_ip1 = m_first[i + 1]; + if (m_comparator(val_ip1, val_i)) { + m_reducer.join(reduction_result, i); + } + } + + KOKKOS_FUNCTION + StdIsSortedUntilFunctor(IteratorType first, ComparatorType comparator, + ReducerType reducer) + : m_first(std::move(first)), + m_comparator(std::move(comparator)), + m_reducer(std::move(reducer)) {} +}; + +// +// overloads accepting exespace +// +template +IteratorType is_sorted_until_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, + IteratorType last, + ComparatorType comp) { + // checks + Impl::static_assert_random_access_and_accessible(ex, first); + Impl::expect_valid_range(first, last); + + const auto num_elements = Kokkos::Experimental::distance(first, last); + + // trivial case + if (num_elements <= 1) { + return last; + } + + /* + Do a par_reduce computing the *min* index that breaks the sorting. + If such an index is found, then the range is sorted until that element. + If no such index is found, then the range is sorted until the end. + */ + using index_type = typename IteratorType::difference_type; + index_type reduction_result; + ::Kokkos::Min reducer(reduction_result); + ::Kokkos::parallel_reduce( + label, + // use num_elements-1 because each index handles i and i+1 + RangePolicy(ex, 0, num_elements - 1), + StdIsSortedUntilFunctor(first, comp, reducer), reducer); + + /* If the reduction result is equal to the initial value, + it means the range is sorted until the end */ + index_type reduction_result_init; + reducer.init(reduction_result_init); + if (reduction_result == reduction_result_init) { + return last; + } else { + /* If such an index is found, then the range is sorted until there and + we need to return an iterator past the element found so do +1 */ + return first + (reduction_result + 1); + } +} + +template +IteratorType is_sorted_until_exespace_impl(const std::string& label, + const ExecutionSpace& ex, + IteratorType first, + IteratorType last) { + using value_type = typename IteratorType::value_type; + using pred_t = Impl::StdAlgoLessThanBinaryPredicate; + return is_sorted_until_exespace_impl(label, ex, first, last, pred_t()); +} + +// +// overloads accepting team handle +// +template +KOKKOS_FUNCTION IteratorType +is_sorted_until_team_impl(const ExecutionSpace& teamHandle, IteratorType first, + IteratorType last, ComparatorType comp) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first); + Impl::expect_valid_range(first, last); + + const auto num_elements = Kokkos::Experimental::distance(first, last); + + // trivial case + if (num_elements <= 1) { + return last; + } + + /* + Do a par_reduce computing the *min* index that breaks the sorting. + If one such index is found, then the range is sorted until that element, + if no such index is found, then it means the range is sorted until the end. + */ + using index_type = typename IteratorType::difference_type; + index_type red_result; + index_type red_result_init; + ::Kokkos::Min reducer(red_result); + reducer.init(red_result_init); + ::Kokkos::parallel_reduce( // use num_elements-1 because each index handles i + // and i+1 + TeamThreadRange(teamHandle, 0, num_elements - 1), + StdIsSortedUntilFunctor(first, comp, reducer), reducer); + teamHandle.team_barrier(); + + /* If the reduction result is equal to the initial value, + and it means the range is sorted until the end */ + if (red_result == red_result_init) { + return last; + } else { + /* If such index is found, then the range is sorted until there and + we need to return an iterator past the element found so do +1 */ + return first + (red_result + 1); + } +} + +template +KOKKOS_FUNCTION IteratorType is_sorted_until_team_impl( + const ExecutionSpace& teamHandle, IteratorType first, IteratorType last) { + using value_type = typename IteratorType::value_type; + using pred_t = Impl::StdAlgoLessThanBinaryPredicate; + return is_sorted_until_team_impl(teamHandle, first, last, pred_t()); +} + +} // namespace Impl +} // namespace Experimental +} // namespace Kokkos + +#endif diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_LexicographicalCompare.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_LexicographicalCompare.hpp similarity index 53% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_LexicographicalCompare.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_LexicographicalCompare.hpp index c3dd13e6b5..b95a66c3bd 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_LexicographicalCompare.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_LexicographicalCompare.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_LEXICOGRAPHICAL_COMPARE_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_LEXICOGRAPHICAL_COMPARE_IMPL_HPP @@ -91,12 +63,14 @@ struct StdLexicographicalCompareFunctor { const auto& my_value1 = m_first1[i]; const auto& my_value2 = m_first2[i]; - bool different = m_comparator(my_value1, my_value2) || - m_comparator(my_value2, my_value1); - auto rv = - different - ? red_value_type{i} - : red_value_type{::Kokkos::reduction_identity::min()}; + const bool different = m_comparator(my_value1, my_value2) || + m_comparator(my_value2, my_value1); + + // FIXME_NVHPC using a ternary operator causes problems + red_value_type rv = {::Kokkos::reduction_identity::min()}; + if (different) { + rv.min_loc_true = i; + } m_reducer.join(red_value, rv); } @@ -110,13 +84,15 @@ struct StdLexicographicalCompareFunctor { m_comparator(std::move(_comp)) {} }; +// +// exespace impl +// template -bool lexicographical_compare_impl(const std::string& label, - const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2, - ComparatorType comp) { +bool lexicographical_compare_exespace_impl( + const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, IteratorType2 last2, + ComparatorType comp) { // checks Impl::static_assert_random_access_and_accessible(ex, first1, first2); Impl::static_assert_iterators_have_matching_difference_type(first1, first2); @@ -165,16 +141,84 @@ bool lexicographical_compare_impl(const std::string& label, } template -bool lexicographical_compare_impl(const std::string& label, - const ExecutionSpace& ex, - IteratorType1 first1, IteratorType1 last1, - IteratorType2 first2, IteratorType2 last2) { +bool lexicographical_compare_exespace_impl( + const std::string& label, const ExecutionSpace& ex, IteratorType1 first1, + IteratorType1 last1, IteratorType2 first2, IteratorType2 last2) { + using value_type_1 = typename IteratorType1::value_type; + using value_type_2 = typename IteratorType2::value_type; + using predicate_t = + Impl::StdAlgoLessThanBinaryPredicate; + return lexicographical_compare_exespace_impl(label, ex, first1, last1, first2, + last2, predicate_t()); +} + +// +// team impl +// +template +KOKKOS_FUNCTION bool lexicographical_compare_team_impl( + const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2, ComparatorType comp) { + // checks + Impl::static_assert_random_access_and_accessible(teamHandle, first1, first2); + Impl::static_assert_iterators_have_matching_difference_type(first1, first2); + Impl::expect_valid_range(first1, last1); + Impl::expect_valid_range(first2, last2); + + // aliases + using index_type = typename IteratorType1::difference_type; + using reducer_type = FirstLoc; + using reduction_value_type = typename reducer_type::value_type; + + // run + const auto d1 = Kokkos::Experimental::distance(first1, last1); + const auto d2 = Kokkos::Experimental::distance(first2, last2); + const auto range = Kokkos::min(d1, d2); + reduction_value_type red_result; + reducer_type reducer(red_result); + using func1_t = + StdLexicographicalCompareFunctor; + + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, range), + func1_t(first1, first2, reducer, comp), reducer); + + teamHandle.team_barrier(); + + // no mismatch + if (red_result.min_loc_true == + ::Kokkos::reduction_identity::min()) { + auto new_last1 = first1 + range; + auto new_last2 = first2 + range; + bool is_prefix = (new_last1 == last1) && (new_last2 != last2); + return is_prefix; + } + + // check mismatched + int less = 0; + auto it1 = first1 + red_result.min_loc_true; + auto it2 = first2 + red_result.min_loc_true; + using func2_t = StdCompareFunctor; + ::Kokkos::parallel_reduce(TeamThreadRange(teamHandle, 0, 1), + func2_t(it1, it2, comp), less); + + teamHandle.team_barrier(); + + return static_cast(less); +} + +template +KOKKOS_FUNCTION bool lexicographical_compare_team_impl( + const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, + IteratorType2 first2, IteratorType2 last2) { using value_type_1 = typename IteratorType1::value_type; using value_type_2 = typename IteratorType2::value_type; using predicate_t = Impl::StdAlgoLessThanBinaryPredicate; - return lexicographical_compare_impl(label, ex, first1, last1, first2, last2, - predicate_t()); + return lexicographical_compare_team_impl(teamHandle, first1, last1, first2, + last2, predicate_t()); } } // namespace Impl diff --git a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_MinMaxMinmaxElement.hpp b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_MinMaxMinmaxElement.hpp similarity index 55% rename from bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_MinMaxMinmaxElement.hpp rename to bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_MinMaxMinmaxElement.hpp index 0a9d41b9bb..2f51db03b4 100644 --- a/bundled/kokkos-3.7.00/algorithms/src/std_algorithms/impl/Kokkos_MinMaxMinmaxElement.hpp +++ b/bundled/kokkos-4.5.01/algorithms/src/std_algorithms/impl/Kokkos_MinMaxMinmaxElement.hpp @@ -1,46 +1,18 @@ -/* //@HEADER // ************************************************************************ // -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ //@HEADER -*/ #ifndef KOKKOS_STD_ALGORITHMS_MIN_MAX_MINMAX_ELEMENT_IMPL_HPP #define KOKKOS_STD_ALGORITHMS_MIN_MAX_MINMAX_ELEMENT_IMPL_HPP @@ -91,12 +63,16 @@ struct StdMinMaxElemFunctor { : m_first(std::move(first)), m_reducer(std::move(reducer)) {} }; +// +// exespace impl +// template