Kokkos Core Kernels Package
Version of the Day
core
src
Kokkos_Atomic.hpp
Go to the documentation of this file.
1
/*
2
//@HEADER
3
// ************************************************************************
4
//
5
// Kokkos v. 3.0
6
// Copyright (2020) National Technology & Engineering
7
// Solutions of Sandia, LLC (NTESS).
8
//
9
// Under the terms of Contract DE-NA0003525 with NTESS,
10
// the U.S. Government retains certain rights in this software.
11
//
12
// Redistribution and use in source and binary forms, with or without
13
// modification, are permitted provided that the following conditions are
14
// met:
15
//
16
// 1. Redistributions of source code must retain the above copyright
17
// notice, this list of conditions and the following disclaimer.
18
//
19
// 2. Redistributions in binary form must reproduce the above copyright
20
// notice, this list of conditions and the following disclaimer in the
21
// documentation and/or other materials provided with the distribution.
22
//
23
// 3. Neither the name of the Corporation nor the names of the
24
// contributors may be used to endorse or promote products derived from
25
// this software without specific prior written permission.
26
//
27
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
//
39
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40
//
41
// ************************************************************************
42
//@HEADER
43
*/
44
67
68
#ifndef KOKKOS_ATOMIC_HPP
69
#define KOKKOS_ATOMIC_HPP
70
71
#include <Kokkos_Macros.hpp>
72
#include <Kokkos_HostSpace.hpp>
73
#include <impl/Kokkos_Traits.hpp>
74
75
//----------------------------------------------------------------------------
76
77
// Need to fix this for pure clang on windows
78
#if defined(_WIN32)
79
#define KOKKOS_ENABLE_WINDOWS_ATOMICS
80
81
#if defined(KOKKOS_ENABLE_CUDA)
82
#define KOKKOS_ENABLE_CUDA_ATOMICS
83
#if defined(KOKKOS_COMPILER_CLANG)
84
#define KOKKOS_ENABLE_GNU_ATOMICS
85
#endif
86
#endif
87
88
#else // _WIN32
89
#if defined(KOKKOS_ENABLE_CUDA)
90
91
// Compiling NVIDIA device code, must use Cuda atomics:
92
93
#define KOKKOS_ENABLE_CUDA_ATOMICS
94
95
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HIP_GPU) || \
96
defined(KOKKOS_IMPL_ENABLE_OVERLOAD_HOST_DEVICE)
97
98
#define KOKKOS_ENABLE_HIP_ATOMICS
99
100
#endif
101
102
#if !defined(KOKKOS_ENABLE_GNU_ATOMICS) && \
103
!defined(KOKKOS_ENABLE_INTEL_ATOMICS) && \
104
!defined(KOKKOS_ENABLE_OPENMP_ATOMICS) && \
105
!defined(KOKKOS_ENABLE_STD_ATOMICS) && \
106
!defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
107
108
// Compiling for non-Cuda atomic implementation has not been pre-selected.
109
// Choose the best implementation for the detected compiler.
110
// Preference: GCC, INTEL, OMP31
111
112
#if defined(KOKKOS_INTERNAL_NOT_PARALLEL)
113
114
#define KOKKOS_ENABLE_SERIAL_ATOMICS
115
116
#elif defined(KOKKOS_COMPILER_GNU) || defined(KOKKOS_COMPILER_CLANG) || \
117
(defined(KOKKOS_COMPILER_NVCC) || defined(KOKKOS_COMPILER_IBM))
118
119
#define KOKKOS_ENABLE_GNU_ATOMICS
120
121
#elif defined(KOKKOS_COMPILER_INTEL) || defined(KOKKOS_COMPILER_CRAYC)
122
123
#define KOKKOS_ENABLE_INTEL_ATOMICS
124
125
#elif defined(_OPENMP) && (201107 <= _OPENMP)
126
127
#define KOKKOS_ENABLE_OPENMP_ATOMICS
128
129
#else
130
131
#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
132
133
#endif
134
135
#endif
/* Not pre-selected atomic implementation */
136
#endif
137
138
#ifdef KOKKOS_ENABLE_CUDA
139
#include <Cuda/Kokkos_Cuda_Locks.hpp>
140
#endif
141
142
namespace
Kokkos
{
143
template
<
typename
T>
144
KOKKOS_INLINE_FUNCTION
void
atomic_add(
volatile
T*
const
dest,
const
T src);
145
146
// Atomic increment
147
template
<
typename
T>
148
KOKKOS_INLINE_FUNCTION
void
atomic_increment(
volatile
T* a);
149
150
template
<
typename
T>
151
KOKKOS_INLINE_FUNCTION
void
atomic_decrement(
volatile
T* a);
152
}
// namespace Kokkos
153
154
namespace
Kokkos
{
155
156
inline
const
char
* atomic_query_version() {
157
#if defined(KOKKOS_ENABLE_CUDA_ATOMICS)
158
return
"KOKKOS_ENABLE_CUDA_ATOMICS"
;
159
#elif defined(KOKKOS_ENABLE_GNU_ATOMICS)
160
return
"KOKKOS_ENABLE_GNU_ATOMICS"
;
161
#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS)
162
return
"KOKKOS_ENABLE_INTEL_ATOMICS"
;
163
#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS)
164
return
"KOKKOS_ENABLE_OPENMP_ATOMICS"
;
165
#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
166
return
"KOKKOS_ENABLE_WINDOWS_ATOMICS"
;
167
#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
168
return
"KOKKOS_ENABLE_SERIAL_ATOMICS"
;
169
#else
170
#error "No valid response for atomic_query_version!"
171
#endif
172
}
173
174
}
// namespace Kokkos
175
176
//----------------------------------------------------------------------------
177
// Atomic Memory Orders
178
//
179
// Implements Strongly-typed analogs of C++ standard memory orders
180
#include "impl/Kokkos_Atomic_Memory_Order.hpp"
181
182
#if defined(KOKKOS_ENABLE_HIP)
183
#include <HIP/Kokkos_HIP_Atomic.hpp>
184
#endif
185
186
#if defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
187
#include "impl/Kokkos_Atomic_Windows.hpp"
188
#endif
189
//----------------------------------------------------------------------------
190
// Atomic Assembly
191
//
192
// Implements CAS128-bit in assembly
193
194
#include "impl/Kokkos_Atomic_Assembly.hpp"
195
196
//----------------------------------------------------------------------------
197
// Memory fence
198
//
199
// All loads and stores from this thread will be globally consistent before
200
// continuing
201
//
202
// void memory_fence() {...};
203
#include "impl/Kokkos_Memory_Fence.hpp"
204
205
//----------------------------------------------------------------------------
206
// Atomic exchange
207
//
208
// template< typename T >
209
// T atomic_exchange( volatile T* const dest , const T val )
210
// { T tmp = *dest ; *dest = val ; return tmp ; }
211
212
#include "impl/Kokkos_Atomic_Exchange.hpp"
213
214
//----------------------------------------------------------------------------
215
// Atomic compare-and-exchange
216
//
217
// template<class T>
218
// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare,
219
// const T val) { bool equal = compare == *dest ; if ( equal ) { *dest = val ; }
220
// return equal ; }
221
222
#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
223
224
#include "impl/Kokkos_Atomic_Generic.hpp"
225
226
//----------------------------------------------------------------------------
227
// Atomic fetch and add
228
//
229
// template<class T>
230
// T atomic_fetch_add(volatile T* const dest, const T val)
231
// { T tmp = *dest ; *dest += val ; return tmp ; }
232
233
#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
234
235
//----------------------------------------------------------------------------
236
// Atomic increment
237
//
238
// template<class T>
239
// T atomic_increment(volatile T* const dest)
240
// { dest++; }
241
242
#include "impl/Kokkos_Atomic_Increment.hpp"
243
244
//----------------------------------------------------------------------------
245
// Atomic Decrement
246
//
247
// template<class T>
248
// T atomic_decrement(volatile T* const dest)
249
// { dest--; }
250
251
#include "impl/Kokkos_Atomic_Decrement.hpp"
252
253
//----------------------------------------------------------------------------
254
// Atomic fetch and sub
255
//
256
// template<class T>
257
// T atomic_fetch_sub(volatile T* const dest, const T val)
258
// { T tmp = *dest ; *dest -= val ; return tmp ; }
259
260
#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
261
262
//----------------------------------------------------------------------------
263
// Atomic fetch and or
264
//
265
// template<class T>
266
// T atomic_fetch_or(volatile T* const dest, const T val)
267
// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
268
269
#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
270
271
//----------------------------------------------------------------------------
272
// Atomic fetch and and
273
//
274
// template<class T>
275
// T atomic_fetch_and(volatile T* const dest, const T val)
276
// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
277
278
#include "impl/Kokkos_Atomic_Fetch_And.hpp"
279
280
//----------------------------------------------------------------------------
281
// Atomic MinMax
282
//
283
// template<class T>
284
// T atomic_min(volatile T* const dest, const T val)
285
// { T tmp = *dest ; *dest = min(*dest, val); return tmp ; }
286
// template<class T>
287
// T atomic_max(volatile T* const dest, const T val)
288
// { T tmp = *dest ; *dest = max(*dest, val); return tmp ; }
289
290
#include "impl/Kokkos_Atomic_MinMax.hpp"
291
292
//----------------------------------------------------------------------------
293
// Provide volatile_load and safe_load
294
//
295
// T volatile_load(T const volatile * const ptr);
296
//
297
// T const& safe_load(T const * const ptr);
298
// XEON PHI
299
// T safe_load(T const * const ptr
300
301
#include "impl/Kokkos_Volatile_Load.hpp"
302
303
//----------------------------------------------------------------------------
304
// Provide atomic loads and stores with memory order semantics
305
306
#include "impl/Kokkos_Atomic_Load.hpp"
307
#include "impl/Kokkos_Atomic_Store.hpp"
308
309
// Generic functions using the above defined functions
310
#include "impl/Kokkos_Atomic_Generic_Secondary.hpp"
311
//----------------------------------------------------------------------------
312
// This atomic-style macro should be an inlined function, not a macro
313
314
#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
315
!defined(__CUDA_ARCH__)
316
317
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
318
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
319
320
#else
321
322
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
323
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
324
325
#endif
326
327
//----------------------------------------------------------------------------
328
329
#endif
/* KOKKOS_ATOMIC_HPP */
Kokkos
Definition:
dummy.cpp:3
Generated by
1.8.14