]> https://gitweb.dealii.org/ - dealii.git/blob
4039448c6bb63038b65b58bd819c9adf0a75a8a8
[dealii.git] /
1
2 // Inline PTX: h u16 , r u32,  l u64, f f32, d f64
3 // Ops:
4
5 // binary operations
6
7 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND() \
8 template<class ctype> \
9 inline __device__ typename std::enable_if<sizeof(ctype)==4, ctype>::type atomic_fetch_and(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
10   uint32_t asm_value = reinterpret_cast<uint32_t&>(value); \
11   uint32_t asm_result = 0u; \
12   asm volatile( \
13           "{\n\t" \
14           ".reg .pred p;\n\t" \
15           "isspacep.global p, %1;\n\t" \
16           "@p  atom.and.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;\n\t" \
17           "@!p atom.and"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;\n\t" \
18           "}\n\t" \
19     : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \
20   return reinterpret_cast<ctype&>(asm_result); \
21 } \
22 template<class ctype> \
23 inline __device__ typename std::enable_if<sizeof(ctype)==8, ctype>::type atomic_fetch_and(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
24   uint64_t asm_value = reinterpret_cast<uint64_t&>(value); \
25   uint64_t asm_result = 0u; \
26   asm volatile( \
27           "{\n\t" \
28           ".reg .pred p;\n\t" \
29           "isspacep.global p, %1;\n\t" \
30           "@p  atom.and.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;\n\t" \
31           "@!p atom.and"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;\n\t" \
32           "}\n\t" \
33     : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \
34   return reinterpret_cast<ctype&>(asm_result); \
35 }
36
37 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_OR() \
38 template<class ctype> \
39 inline __device__ typename std::enable_if<sizeof(ctype)==4, ctype>::type atomic_fetch_or(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
40   uint32_t asm_value = reinterpret_cast<uint32_t&>(value); \
41   uint32_t asm_result = 0u; \
42   asm volatile( \
43           "{\n\t" \
44           ".reg .pred p;\n\t" \
45           "isspacep.global p, %1;\n\t" \
46           "@p  atom.or.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;\n\t" \
47           "@!p atom.or"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;\n\t" \
48           "}\n\t" \
49     : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \
50   return reinterpret_cast<ctype&>(asm_result); \
51 } \
52 template<class ctype> \
53 inline __device__ typename std::enable_if<sizeof(ctype)==8, ctype>::type atomic_fetch_or(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
54   uint64_t asm_value = reinterpret_cast<uint64_t&>(value); \
55   uint64_t asm_result = 0u; \
56   asm volatile( \
57           "{\n\t" \
58           ".reg .pred p;\n\t" \
59           "isspacep.global p, %1;\n\t" \
60           "@p  atom.or.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;\n\t" \
61           "@!p atom.or"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;\n\t" \
62           "}\n\t" \
63     : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \
64   return reinterpret_cast<ctype&>(asm_result); \
65 }
66
67 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_XOR() \
68 template<class ctype> \
69 inline __device__ typename std::enable_if<sizeof(ctype)==4, ctype>::type atomic_fetch_xor(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
70   uint32_t asm_value = reinterpret_cast<uint32_t&>(value); \
71   uint32_t asm_result = 0u; \
72   asm volatile( \
73           "{\n\t" \
74           ".reg .pred p;\n\t" \
75           "isspacep.global p, %1;\n\t" \
76           "@p  atom.xor.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;\n\t" \
77           "@!p atom.xor"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b32" " %0,[%1],%2;\n\t" \
78           "}\n\t" \
79     : "=r"(asm_result) : "l"(dest),"r"(asm_value) : "memory"); \
80   return reinterpret_cast<ctype&>(asm_result); \
81 } \
82 template<class ctype> \
83 inline __device__ typename std::enable_if<sizeof(ctype)==8, ctype>::type atomic_fetch_xor(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
84   uint64_t asm_value = reinterpret_cast<uint64_t&>(value); \
85   uint64_t asm_result = 0u; \
86   asm volatile( \
87           "{\n\t" \
88           ".reg .pred p;\n\t" \
89           "isspacep.global p, %1;\n\t" \
90           "@p  atom.xor.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;\n\t" \
91           "@!p atom.xor"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM ".b64" " %0,[%1],%2;\n\t" \
92           "}\n\t" \
93     : "=l"(asm_result) : "l"(dest),"l"(asm_value) : "memory"); \
94   return reinterpret_cast<ctype&>(asm_result); \
95 }
96
97 // Fetch atomics
98 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
99 inline __device__ ctype atomic_fetch_add(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
100   ctype result=0; \
101   asm volatile( \
102           "{\n\t" \
103           ".reg .pred p;\n\t" \
104           "isspacep.global p, %1;\n\t" \
105           "@p  atom.add.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
106           "@!p atom.add"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
107           "}\n\t" \
108     : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \
109   return result; \
110 }
111
112 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
113 inline __device__ ctype atomic_fetch_sub(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
114   ctype result=0; \
115   ctype neg_value = -value; \
116   asm volatile( \
117           "{\n\t" \
118           ".reg .pred p;\n\t" \
119           "isspacep.global p, %1;\n\t" \
120           "@p  atom.add.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
121           "@!p atom.add"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
122           "}\n\t" \
123     : reg_ret_ctype(result) : "l"(dest),reg_ctype(neg_value) : "memory"); \
124   return result; \
125 }
126
127 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
128 inline __device__ ctype atomic_fetch_min(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
129   ctype result=0; \
130   asm volatile( \
131           "{\n\t" \
132           ".reg .pred p;\n\t" \
133           "isspacep.global p, %1;\n\t" \
134           "@p  atom.min.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
135           "@!p atom.min"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
136           "}\n\t" \
137     : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \
138   return result; \
139 }
140
141 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
142 inline __device__ ctype atomic_fetch_max(ctype* dest, ctype value, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
143   ctype result=0; \
144   asm volatile( \
145           "{\n\t" \
146           ".reg .pred p;\n\t" \
147           "isspacep.global p, %1;\n\t" \
148           "@p  atom.max.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
149           "@!p atom.max"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
150           "}\n\t" \
151     : reg_ret_ctype(result) : "l"(dest),reg_ctype(value) : "memory"); \
152   return result; \
153 }
154
155 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
156 inline __device__ ctype atomic_fetch_inc(ctype* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
157   ctype result = 0; \
158   ctype limit = desul::Impl::numeric_limits_max<ctype>::value; \
159   asm volatile( \
160           "{\n\t" \
161           ".reg .pred p;\n\t" \
162           "isspacep.global p, %1;\n\t" \
163           "@p  atom.inc.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
164           "@!p atom.inc"       __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
165           "}\n\t" \
166     : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \
167   return result; \
168 } \
169 inline __device__ ctype atomic_fetch_inc_mod(ctype* dest, ctype limit, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
170   ctype result = 0; \
171   asm volatile( \
172           "{\n\t" \
173           ".reg .pred p;\n\t" \
174           "isspacep.global p, %1;\n\t" \
175           "@p  atom.inc.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
176           "@!p atom.inc"       __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
177           "}\n\t" \
178     : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \
179   return result; \
180 }
181
182 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
183 inline __device__ ctype atomic_fetch_dec(ctype* dest, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
184   ctype result = 0; \
185   ctype limit = desul::Impl::numeric_limits_max<ctype>::value; \
186   asm volatile( \
187           "{\n\t" \
188           ".reg .pred p;\n\t" \
189           "isspacep.global p, %1;\n\t" \
190           "@p  atom.dec.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
191           "@!p atom.dec"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
192           "}\n\t" \
193     : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \
194   return result; \
195 } \
196 inline __device__ ctype atomic_fetch_dec_mod(ctype* dest, ctype limit, __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER, __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE) { \
197   ctype result = 0; \
198   asm volatile( \
199           "{\n\t" \
200           ".reg .pred p;\n\t" \
201           "isspacep.global p, %1;\n\t" \
202           "@p  atom.dec.global" __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
203           "@!p atom.dec"        __DESUL_IMPL_CUDA_ASM_MEMORY_ORDER_ASM __DESUL_IMPL_CUDA_ASM_MEMORY_SCOPE_ASM asm_ctype " %0,[%1],%2;\n\t" \
204           "}\n\t" \
205     : reg_ret_ctype(result) : "l"(dest),reg_ctype(limit) : "memory"); \
206   return result; \
207 }
208
209 // Group ops for integer ctypes
210 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
211 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
212 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
213 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
214 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype)
215
216 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
217 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
218 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
219 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN(ctype,asm_ctype,reg_ctype,reg_ret_ctype) \
220 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX(ctype,asm_ctype,reg_ctype,reg_ret_ctype)
221
222 #define __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_BIN_OP() \
223 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND() \
224 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_OR() \
225 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_XOR()
226
227
228 // Instantiate Functions
229 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(float,".f32","f","=f")
230 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(float,".f32","f","=f")
231 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD(double,".f64","d","=d")
232 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_SUB(double,".f64","d","=d")
233
234 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(uint32_t,".u32","r","=r")
235 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_UNSIGNED_OP(uint64_t,".u64","l","=l")
236 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(int32_t,".s32","r","=r")
237 //__DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INTEGER_OP(int64_t,".s64","l","=l")
238
239 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC(uint32_t,".u32","r","=r")
240 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC(uint32_t,".u32","r","=r")
241
242 __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_BIN_OP()
243
244 #undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_ADD
245 #undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MIN
246 #undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_MAX
247 #undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_INC
248 #undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_DEC
249 #undef __DESUL_IMPL_CUDA_ASM_ATOMIC_FETCH_AND
250

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.