45 #ifndef KOKKOS_CORE_EXP_MD_RANGE_POLICY_HPP 46 #define KOKKOS_CORE_EXP_MD_RANGE_POLICY_HPP 48 #include <initializer_list> 51 #include <Kokkos_Array.hpp> 52 #include <impl/KokkosExp_Host_IterateTile.hpp> 53 #include <Kokkos_ExecPolicy.hpp> 54 #include <type_traits> 69 template <
typename ExecSpace>
70 struct default_outer_direction {
72 static constexpr Iterate value = Iterate::Right;
75 template <
typename ExecSpace>
76 struct default_inner_direction {
78 static constexpr Iterate value = Iterate::Right;
82 template <
unsigned N, Iterate OuterDir = Iterate::Default,
83 Iterate InnerDir = Iterate::Default>
85 static_assert(N != 0u,
"Kokkos Error: rank 0 undefined");
86 static_assert(N != 1u,
87 "Kokkos Error: rank 1 is not a multi-dimensional range");
88 static_assert(N < 7u,
"Kokkos Error: Unsupported rank...");
90 using iteration_pattern = Rank<N, OuterDir, InnerDir>;
92 static constexpr
int rank = N;
93 static constexpr Iterate outer_direction = OuterDir;
94 static constexpr Iterate inner_direction = InnerDir;
101 constexpr std::enable_if_t<!std::is_signed<T>::value,
bool>
102 is_less_than_value_initialized_variable(T) {
107 constexpr std::enable_if_t<std::is_signed<T>::value,
bool>
108 is_less_than_value_initialized_variable(T arg) {
113 template <
class To,
class From>
114 constexpr To checked_narrow_cast(From arg) {
115 constexpr
const bool is_different_signedness =
116 (std::is_signed<To>::value != std::is_signed<From>::value);
117 auto const ret =
static_cast<To
>(arg);
118 if (static_cast<From>(ret) != arg ||
119 (is_different_signedness &&
120 is_less_than_value_initialized_variable(arg) !=
121 is_less_than_value_initialized_variable(ret))) {
122 Kokkos::abort(
"unsafe narrowing conversion");
132 template <
class IndexType,
class Array,
class U, std::
size_t M>
133 constexpr Array to_array_potentially_narrowing(
const U (&init)[M]) {
134 using T =
typename Array::value_type;
136 constexpr std::size_t N = a.size();
137 static_assert(M <= N,
"");
138 auto* ptr = a.data();
143 for (
auto x : init) {
144 *ptr++ = checked_narrow_cast<T>(x);
145 (void)checked_narrow_cast<IndexType>(x);
154 template <
class IndexType,
class NVCC_WONT_LET_ME_CALL_YOU_Array,
class U,
156 constexpr NVCC_WONT_LET_ME_CALL_YOU_Array to_array_potentially_narrowing(
158 using T =
typename NVCC_WONT_LET_ME_CALL_YOU_Array::value_type;
159 NVCC_WONT_LET_ME_CALL_YOU_Array a{};
160 constexpr std::size_t N = a.size();
161 static_assert(M <= N,
"");
162 for (std::size_t i = 0; i < M; ++i) {
163 a[i] = checked_narrow_cast<T>(other[i]);
164 (void)checked_narrow_cast<IndexType>(other[i]);
169 struct TileSizeProperties {
171 int default_largest_tile_size;
172 int default_tile_size;
173 int max_total_tile_size;
176 template <
typename ExecutionSpace>
177 TileSizeProperties get_tile_size_properties(
const ExecutionSpace&) {
179 TileSizeProperties properties;
180 properties.max_threads = std::numeric_limits<int>::max();
181 properties.default_largest_tile_size = 0;
182 properties.default_tile_size = 2;
183 properties.max_total_tile_size = std::numeric_limits<int>::max();
190 template <
typename... Properties>
191 struct MDRangePolicy :
public Kokkos::Impl::PolicyTraits<Properties...> {
192 using traits = Kokkos::Impl::PolicyTraits<Properties...>;
193 using range_policy = RangePolicy<Properties...>;
195 typename traits::execution_space m_space;
197 using impl_range_policy =
198 RangePolicy<
typename traits::execution_space,
199 typename traits::schedule_type,
typename traits::index_type>;
201 using execution_policy =
202 MDRangePolicy<Properties...>;
205 template <
class... OtherProperties>
206 friend struct MDRangePolicy;
208 static_assert(!std::is_same<typename traits::iteration_pattern, void>::value,
209 "Kokkos Error: MD iteration pattern not defined");
211 using iteration_pattern =
typename traits::iteration_pattern;
212 using work_tag =
typename traits::work_tag;
213 using launch_bounds =
typename traits::launch_bounds;
214 using member_type =
typename range_policy::member_type;
216 static constexpr
int rank = iteration_pattern::rank;
218 using index_type =
typename traits::index_type;
219 using array_index_type = std::int64_t;
233 point_type m_lower = {};
234 point_type m_upper = {};
235 tile_type m_tile = {};
236 point_type m_tile_end = {};
237 index_type m_num_tiles = 1;
238 index_type m_prod_tile_dims = 1;
239 bool m_tune_tile_size =
false;
241 static constexpr
auto outer_direction =
242 (iteration_pattern::outer_direction != Iterate::Default)
243 ? iteration_pattern::outer_direction
244 : default_outer_direction<typename traits::execution_space>::value;
246 static constexpr
auto inner_direction =
247 iteration_pattern::inner_direction != Iterate::Default
248 ? iteration_pattern::inner_direction
249 : default_inner_direction<typename traits::execution_space>::value;
251 static constexpr
auto Right = Iterate::Right;
252 static constexpr
auto Left = Iterate::Left;
254 KOKKOS_INLINE_FUNCTION
const typename traits::execution_space& space()
const {
258 MDRangePolicy() =
default;
260 template <
typename LT, std::size_t LN,
typename UT, std::size_t UN,
261 typename TT = array_index_type, std::size_t TN = rank,
262 typename = std::enable_if_t<std::is_integral<LT>::value &&
263 std::is_integral<UT>::value &&
264 std::is_integral<TT>::value>>
265 MDRangePolicy(
const LT (&lower)[LN],
const UT (&upper)[UN],
266 const TT (&tile)[TN] = {})
268 Impl::to_array_potentially_narrowing<index_type, decltype(m_lower)>(
270 Impl::to_array_potentially_narrowing<index_type, decltype(m_upper)>(
272 Impl::to_array_potentially_narrowing<index_type, decltype(m_tile)>(
275 LN == rank && UN == rank && TN <= rank,
276 "MDRangePolicy: Constructor initializer lists have wrong size");
279 template <
typename LT, std::size_t LN,
typename UT, std::size_t UN,
280 typename TT = array_index_type, std::size_t TN = rank,
281 typename = std::enable_if_t<std::is_integral<LT>::value &&
282 std::is_integral<UT>::value &&
283 std::is_integral<TT>::value>>
284 MDRangePolicy(
const typename traits::execution_space& work_space,
285 const LT (&lower)[LN],
const UT (&upper)[UN],
286 const TT (&tile)[TN] = {})
289 Impl::to_array_potentially_narrowing<index_type, decltype(m_lower)>(
291 Impl::to_array_potentially_narrowing<index_type, decltype(m_upper)>(
293 Impl::to_array_potentially_narrowing<index_type, decltype(m_tile)>(
296 LN == rank && UN == rank && TN <= rank,
297 "MDRangePolicy: Constructor initializer lists have wrong size");
303 MDRangePolicy(point_type
const& lower, point_type
const& upper,
304 tile_type
const& tile = tile_type{})
305 : MDRangePolicy(
typename traits::execution_space(), lower, upper, tile) {}
307 MDRangePolicy(
const typename traits::execution_space& work_space,
308 point_type
const& lower, point_type
const& upper,
309 tile_type
const& tile = tile_type{})
310 : m_space(work_space), m_lower(lower), m_upper(upper), m_tile(tile) {
311 init_helper(Impl::get_tile_size_properties(work_space));
314 template <
typename T, std::size_t NT = rank,
315 typename = std::enable_if_t<std::is_integral<T>::value>>
319 : MDRangePolicy(
typename traits::execution_space(), lower, upper, tile) {}
321 template <
typename T, std::size_t NT = rank,
322 typename = std::enable_if_t<std::is_integral<T>::value>>
323 MDRangePolicy(
const typename traits::execution_space& work_space,
329 Impl::to_array_potentially_narrowing<index_type, decltype(m_lower)>(
331 Impl::to_array_potentially_narrowing<index_type, decltype(m_upper)>(
333 Impl::to_array_potentially_narrowing<index_type, decltype(m_tile)>(
336 template <
class... OtherProperties>
337 MDRangePolicy(
const MDRangePolicy<OtherProperties...> p)
343 m_tile_end(p.m_tile_end),
344 m_num_tiles(p.m_num_tiles),
345 m_prod_tile_dims(p.m_prod_tile_dims),
346 m_tune_tile_size(p.m_tune_tile_size) {}
348 void impl_change_tile_size(
const point_type& tile) {
350 init_helper(Impl::get_tile_size_properties(m_space));
352 bool impl_tune_tile_size()
const {
return m_tune_tile_size; }
355 void init_helper(Impl::TileSizeProperties properties) {
356 m_prod_tile_dims = 1;
360 if (inner_direction == Iterate::Right) {
362 rank_start = rank - 1;
365 for (
int i = rank_start; i != rank_end; i += increment) {
366 const index_type length = m_upper[i] - m_lower[i];
367 if (m_tile[i] <= 0) {
368 m_tune_tile_size =
true;
369 if ((inner_direction == Iterate::Right && (i < rank - 1)) ||
370 (inner_direction == Iterate::Left && (i > 0))) {
371 if (m_prod_tile_dims * properties.default_tile_size <
372 static_cast<index_type>(properties.max_total_tile_size)) {
373 m_tile[i] = properties.default_tile_size;
378 m_tile[i] = properties.default_largest_tile_size == 0
379 ? std::max<int>(length, 1)
380 : properties.default_largest_tile_size;
384 static_cast<index_type
>((length + m_tile[i] - 1) / m_tile[i]);
385 m_num_tiles *= m_tile_end[i];
386 m_prod_tile_dims *= m_tile[i];
388 if (m_prod_tile_dims > static_cast<index_type>(properties.max_threads)) {
389 printf(
" Product of tile dimensions exceed maximum limit: %d\n",
390 static_cast<int>(properties.max_threads));
392 "ExecSpace Error: MDRange tile dims exceed maximum number " 393 "of threads per block - choose smaller tile dims");
403 using Kokkos::Iterate;
404 using Kokkos::MDRangePolicy;
409 #endif // KOKKOS_CORE_EXP_MD_RANGE_POLICY_HPP
Derived from the C++17 'std::array'. Dropping the iterator interface.
Declaration of various MemoryLayout options.