Free Electron
atomic.h
1 #ifndef AL_ATOMIC_H
2 #define AL_ATOMIC_H
3 
4 #include "static_assert.h"
5 #include "bool.h"
6 
7 #ifdef __GNUC__
8 /* This helps cast away the const-ness of a pointer without accidentally
9  * changing the pointer type. This is necessary due to Clang's inability to use
10  * atomic_load on a const _Atomic variable.
11  */
12 #define CONST_CAST(T, V) __extension__({ \
13  const T _tmp = (V); \
14  (T)_tmp; \
15 })
16 #else
17 #define CONST_CAST(T, V) ((T)(V))
18 #endif
19 
20 #ifdef __cplusplus
21 extern "C" {
22 #endif
23 
24 /* Atomics using C11 */
25 #ifdef HAVE_C11_ATOMIC
26 
27 #include <stdatomic.h>
28 
29 #define almemory_order memory_order
30 #define almemory_order_relaxed memory_order_relaxed
31 #define almemory_order_consume memory_order_consume
32 #define almemory_order_acquire memory_order_acquire
33 #define almemory_order_release memory_order_release
34 #define almemory_order_acq_rel memory_order_acq_rel
35 #define almemory_order_seq_cst memory_order_seq_cst
36 
37 #define ATOMIC(T) T _Atomic
38 #define ATOMIC_FLAG atomic_flag
39 
40 #define ATOMIC_INIT atomic_init
41 #define ATOMIC_INIT_STATIC ATOMIC_VAR_INIT
42 /*#define ATOMIC_FLAG_INIT ATOMIC_FLAG_INIT*/
43 
44 #define ATOMIC_LOAD atomic_load_explicit
45 #define ATOMIC_STORE atomic_store_explicit
46 
47 #define ATOMIC_ADD atomic_fetch_add_explicit
48 #define ATOMIC_SUB atomic_fetch_sub_explicit
49 
50 #define ATOMIC_EXCHANGE atomic_exchange_explicit
51 #define ATOMIC_COMPARE_EXCHANGE_STRONG atomic_compare_exchange_strong_explicit
52 #define ATOMIC_COMPARE_EXCHANGE_WEAK atomic_compare_exchange_weak_explicit
53 
54 #define ATOMIC_FLAG_TEST_AND_SET atomic_flag_test_and_set_explicit
55 #define ATOMIC_FLAG_CLEAR atomic_flag_clear_explicit
56 
57 #define ATOMIC_THREAD_FENCE atomic_thread_fence
58 
59 /* Atomics using GCC intrinsics */
60 #elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__QNXNTO__)
61 
62 enum almemory_order {
63  almemory_order_relaxed,
64  almemory_order_consume,
65  almemory_order_acquire,
66  almemory_order_release,
67  almemory_order_acq_rel,
68  almemory_order_seq_cst
69 };
70 
71 #define ATOMIC(T) struct { T volatile value; }
72 #define ATOMIC_FLAG ATOMIC(int)
73 
74 #define ATOMIC_INIT(_val, _newval) do { (_val)->value = (_newval); } while(0)
75 #define ATOMIC_INIT_STATIC(_newval) {(_newval)}
76 #define ATOMIC_FLAG_INIT ATOMIC_INIT_STATIC(0)
77 
78 #define ATOMIC_LOAD(_val, _MO) __extension__({ \
79  __typeof((_val)->value) _r = (_val)->value; \
80  __asm__ __volatile__("" ::: "memory"); \
81  _r; \
82 })
83 #define ATOMIC_STORE(_val, _newval, _MO) do { \
84  __asm__ __volatile__("" ::: "memory"); \
85  (_val)->value = (_newval); \
86 } while(0)
87 
88 #define ATOMIC_ADD(_val, _incr, _MO) __sync_fetch_and_add(&(_val)->value, (_incr))
89 #define ATOMIC_SUB(_val, _decr, _MO) __sync_fetch_and_sub(&(_val)->value, (_decr))
90 
91 #define ATOMIC_EXCHANGE(_val, _newval, _MO) __extension__({ \
92  __asm__ __volatile__("" ::: "memory"); \
93  __sync_lock_test_and_set(&(_val)->value, (_newval)); \
94 })
95 #define ATOMIC_COMPARE_EXCHANGE_STRONG(_val, _oldval, _newval, _MO1, _MO2) __extension__({ \
96  __typeof(*(_oldval)) _o = *(_oldval); \
97  *(_oldval) = __sync_val_compare_and_swap(&(_val)->value, _o, (_newval)); \
98  *(_oldval) == _o; \
99 })
100 
101 #define ATOMIC_FLAG_TEST_AND_SET(_val, _MO) __extension__({ \
102  __asm__ __volatile__("" ::: "memory"); \
103  __sync_lock_test_and_set(&(_val)->value, 1); \
104 })
105 #define ATOMIC_FLAG_CLEAR(_val, _MO) __extension__({ \
106  __sync_lock_release(&(_val)->value); \
107  __asm__ __volatile__("" ::: "memory"); \
108 })
109 
110 
111 #define ATOMIC_THREAD_FENCE(order) do { \
112  enum { must_be_constant = (order) }; \
113  const int _o = must_be_constant; \
114  if(_o > almemory_order_relaxed) \
115  __asm__ __volatile__("" ::: "memory"); \
116 } while(0)
117 
118 /* Atomics using x86/x86-64 GCC inline assembly */
119 #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
120 
121 #define WRAP_ADD(S, ret, dest, incr) __asm__ __volatile__( \
122  "lock; xadd"S" %0,(%1)" \
123  : "=r" (ret) \
124  : "r" (dest), "0" (incr) \
125  : "memory" \
126 )
127 #define WRAP_SUB(S, ret, dest, decr) __asm__ __volatile__( \
128  "lock; xadd"S" %0,(%1)" \
129  : "=r" (ret) \
130  : "r" (dest), "0" (-(decr)) \
131  : "memory" \
132 )
133 
134 #define WRAP_XCHG(S, ret, dest, newval) __asm__ __volatile__( \
135  "lock; xchg"S" %0,(%1)" \
136  : "=r" (ret) \
137  : "r" (dest), "0" (newval) \
138  : "memory" \
139 )
140 #define WRAP_CMPXCHG(S, ret, dest, oldval, newval) __asm__ __volatile__( \
141  "lock; cmpxchg"S" %2,(%1)" \
142  : "=a" (ret) \
143  : "r" (dest), "r" (newval), "0" (oldval) \
144  : "memory" \
145 )
146 
147 
148 enum almemory_order {
149  almemory_order_relaxed,
150  almemory_order_consume,
151  almemory_order_acquire,
152  almemory_order_release,
153  almemory_order_acq_rel,
154  almemory_order_seq_cst
155 };
156 
157 #define ATOMIC(T) struct { T volatile value; }
158 
159 #define ATOMIC_INIT(_val, _newval) do { (_val)->value = (_newval); } while(0)
160 #define ATOMIC_INIT_STATIC(_newval) {(_newval)}
161 
162 #define ATOMIC_LOAD(_val, _MO) __extension__({ \
163  __typeof((_val)->value) _r = (_val)->value; \
164  __asm__ __volatile__("" ::: "memory"); \
165  _r; \
166 })
167 #define ATOMIC_STORE(_val, _newval, _MO) do { \
168  __asm__ __volatile__("" ::: "memory"); \
169  (_val)->value = (_newval); \
170 } while(0)
171 
172 #define ATOMIC_ADD(_val, _incr, _MO) __extension__({ \
173  static_assert(sizeof((_val)->value)==4 || sizeof((_val)->value)==8, "Unsupported size!"); \
174  __typeof((_val)->value) _r; \
175  if(sizeof((_val)->value) == 4) WRAP_ADD("l", _r, &(_val)->value, _incr); \
176  else if(sizeof((_val)->value) == 8) WRAP_ADD("q", _r, &(_val)->value, _incr); \
177  _r; \
178 })
179 #define ATOMIC_SUB(_val, _decr, _MO) __extension__({ \
180  static_assert(sizeof((_val)->value)==4 || sizeof((_val)->value)==8, "Unsupported size!"); \
181  __typeof((_val)->value) _r; \
182  if(sizeof((_val)->value) == 4) WRAP_SUB("l", _r, &(_val)->value, _decr); \
183  else if(sizeof((_val)->value) == 8) WRAP_SUB("q", _r, &(_val)->value, _decr); \
184  _r; \
185 })
186 
187 #define ATOMIC_EXCHANGE(_val, _newval, _MO) __extension__({ \
188  __typeof((_val)->value) _r; \
189  if(sizeof((_val)->value) == 4) WRAP_XCHG("l", _r, &(_val)->value, (_newval)); \
190  else if(sizeof((_val)->value) == 8) WRAP_XCHG("q", _r, &(_val)->value, (_newval)); \
191  _r; \
192 })
193 #define ATOMIC_COMPARE_EXCHANGE_STRONG(_val, _oldval, _newval, _MO1, _MO2) __extension__({ \
194  __typeof(*(_oldval)) _old = *(_oldval); \
195  if(sizeof((_val)->value) == 4) WRAP_CMPXCHG("l", *(_oldval), &(_val)->value, _old, (_newval)); \
196  else if(sizeof((_val)->value) == 8) WRAP_CMPXCHG("q", *(_oldval), &(_val)->value, _old, (_newval)); \
197  *(_oldval) == _old; \
198 })
199 
200 #define ATOMIC_EXCHANGE_PTR(_val, _newval, _MO) __extension__({ \
201  void *_r; \
202  if(sizeof(void*) == 4) WRAP_XCHG("l", _r, &(_val)->value, (_newval)); \
203  else if(sizeof(void*) == 8) WRAP_XCHG("q", _r, &(_val)->value, (_newval));\
204  _r; \
205 })
206 #define ATOMIC_COMPARE_EXCHANGE_PTR_STRONG(_val, _oldval, _newval, _MO1, _MO2) __extension__({ \
207  void *_old = *(_oldval); \
208  if(sizeof(void*) == 4) WRAP_CMPXCHG("l", *(_oldval), &(_val)->value, _old, (_newval)); \
209  else if(sizeof(void*) == 8) WRAP_CMPXCHG("q", *(_oldval), &(_val)->value, _old, (_newval)); \
210  *(_oldval) == _old; \
211 })
212 
213 #define ATOMIC_THREAD_FENCE(order) do { \
214  enum { must_be_constant = (order) }; \
215  const int _o = must_be_constant; \
216  if(_o > almemory_order_relaxed) \
217  __asm__ __volatile__("" ::: "memory"); \
218 } while(0)
219 
220 /* Atomics using Windows methods */
221 #elif defined(_WIN32)
222 
223 #define WIN32_LEAN_AND_MEAN
224 #include <windows.h>
225 
226 /* NOTE: This mess is *extremely* touchy. It lacks quite a bit of safety
227  * checking due to the lack of multi-statement expressions, typeof(), and C99
228  * compound literals. It is incapable of properly exchanging floats, which get
229  * casted to LONG/int, and could cast away potential warnings.
230  *
231  * Unfortunately, it's the only semi-safe way that doesn't rely on C99 (because
232  * MSVC).
233  */
234 
235 inline LONG AtomicAdd32(volatile LONG *dest, LONG incr)
236 {
237  return InterlockedExchangeAdd(dest, incr);
238 }
239 inline LONGLONG AtomicAdd64(volatile LONGLONG *dest, LONGLONG incr)
240 {
241  return InterlockedExchangeAdd64(dest, incr);
242 }
243 inline LONG AtomicSub32(volatile LONG *dest, LONG decr)
244 {
245  return InterlockedExchangeAdd(dest, -decr);
246 }
247 inline LONGLONG AtomicSub64(volatile LONGLONG *dest, LONGLONG decr)
248 {
249  return InterlockedExchangeAdd64(dest, -decr);
250 }
251 
252 inline LONG AtomicSwap32(volatile LONG *dest, LONG newval)
253 {
254  return InterlockedExchange(dest, newval);
255 }
256 inline LONGLONG AtomicSwap64(volatile LONGLONG *dest, LONGLONG newval)
257 {
258  return InterlockedExchange64(dest, newval);
259 }
260 inline void *AtomicSwapPtr(void *volatile *dest, void *newval)
261 {
262  return InterlockedExchangePointer(dest, newval);
263 }
264 
265 inline bool CompareAndSwap32(volatile LONG *dest, LONG newval, LONG *oldval)
266 {
267  LONG old = *oldval;
268  *oldval = InterlockedCompareExchange(dest, newval, *oldval);
269  return old == *oldval;
270 }
271 inline bool CompareAndSwap64(volatile LONGLONG *dest, LONGLONG newval, LONGLONG *oldval)
272 {
273  LONGLONG old = *oldval;
274  *oldval = InterlockedCompareExchange64(dest, newval, *oldval);
275  return old == *oldval;
276 }
277 inline bool CompareAndSwapPtr(void *volatile *dest, void *newval, void **oldval)
278 {
279  void *old = *oldval;
280  *oldval = InterlockedCompareExchangePointer(dest, newval, *oldval);
281  return old == *oldval;
282 }
283 
284 #define WRAP_ADDSUB(T, _func, _ptr, _amnt) _func((T volatile*)(_ptr), (_amnt))
285 #define WRAP_XCHG(T, _func, _ptr, _newval) _func((T volatile*)(_ptr), (_newval))
286 #define WRAP_CMPXCHG(T, _func, _ptr, _newval, _oldval) _func((T volatile*)(_ptr), (_newval), (T*)(_oldval))
287 
288 
289 enum almemory_order {
290  almemory_order_relaxed,
291  almemory_order_consume,
292  almemory_order_acquire,
293  almemory_order_release,
294  almemory_order_acq_rel,
295  almemory_order_seq_cst
296 };
297 
298 #define ATOMIC(T) struct { T volatile value; }
299 
300 #define ATOMIC_INIT(_val, _newval) do { (_val)->value = (_newval); } while(0)
301 #define ATOMIC_INIT_STATIC(_newval) {(_newval)}
302 
303 #define ATOMIC_LOAD(_val, _MO) ((_val)->value)
304 #define ATOMIC_STORE(_val, _newval, _MO) do { \
305  (_val)->value = (_newval); \
306 } while(0)
307 
308 int _al_invalid_atomic_size(); /* not defined */
309 void *_al_invalid_atomic_ptr_size(); /* not defined */
310 
311 #define ATOMIC_ADD(_val, _incr, _MO) \
312  ((sizeof((_val)->value)==4) ? WRAP_ADDSUB(LONG, AtomicAdd32, &(_val)->value, (_incr)) : \
313  (sizeof((_val)->value)==8) ? WRAP_ADDSUB(LONGLONG, AtomicAdd64, &(_val)->value, (_incr)) : \
314  _al_invalid_atomic_size())
315 #define ATOMIC_SUB(_val, _decr, _MO) \
316  ((sizeof((_val)->value)==4) ? WRAP_ADDSUB(LONG, AtomicSub32, &(_val)->value, (_decr)) : \
317  (sizeof((_val)->value)==8) ? WRAP_ADDSUB(LONGLONG, AtomicSub64, &(_val)->value, (_decr)) : \
318  _al_invalid_atomic_size())
319 
320 #define ATOMIC_EXCHANGE(_val, _newval, _MO) \
321  ((sizeof((_val)->value)==4) ? WRAP_XCHG(LONG, AtomicSwap32, &(_val)->value, (_newval)) : \
322  (sizeof((_val)->value)==8) ? WRAP_XCHG(LONGLONG, AtomicSwap64, &(_val)->value, (_newval)) : \
323  (LONG)_al_invalid_atomic_size())
324 #define ATOMIC_COMPARE_EXCHANGE_STRONG(_val, _oldval, _newval, _MO1, _MO2) \
325  ((sizeof((_val)->value)==4) ? WRAP_CMPXCHG(LONG, CompareAndSwap32, &(_val)->value, (_newval), (_oldval)) : \
326  (sizeof((_val)->value)==8) ? WRAP_CMPXCHG(LONGLONG, CompareAndSwap64, &(_val)->value, (_newval), (_oldval)) : \
327  (bool)_al_invalid_atomic_size())
328 
329 #define ATOMIC_EXCHANGE_PTR(_val, _newval, _MO) \
330  ((sizeof((_val)->value)==sizeof(void*)) ? AtomicSwapPtr((void*volatile*)&(_val)->value, (_newval)) : \
331  _al_invalid_atomic_ptr_size())
332 #define ATOMIC_COMPARE_EXCHANGE_PTR_STRONG(_val, _oldval, _newval, _MO1, _MO2)\
333  ((sizeof((_val)->value)==sizeof(void*)) ? CompareAndSwapPtr((void*volatile*)&(_val)->value, (_newval), (void**)(_oldval)) : \
334  (bool)_al_invalid_atomic_size())
335 
336 #define ATOMIC_THREAD_FENCE(order) do { \
337  enum { must_be_constant = (order) }; \
338  const int _o = must_be_constant; \
339  if(_o > almemory_order_relaxed) \
340  _ReadWriteBarrier(); \
341 } while(0)
342 
343 #else
344 
345 #error "No atomic functions available on this platform!"
346 
347 #define ATOMIC(T) T
348 
349 #define ATOMIC_INIT(_val, _newval) ((void)0)
350 #define ATOMIC_INIT_STATIC(_newval) (0)
351 
352 #define ATOMIC_LOAD(...) (0)
353 #define ATOMIC_STORE(...) ((void)0)
354 
355 #define ATOMIC_ADD(...) (0)
356 #define ATOMIC_SUB(...) (0)
357 
358 #define ATOMIC_EXCHANGE(...) (0)
359 #define ATOMIC_COMPARE_EXCHANGE_STRONG(...) (0)
360 
361 #define ATOMIC_THREAD_FENCE(...) ((void)0)
362 #endif
363 
364 /* If no PTR xchg variants are provided, the normal ones can handle it. */
365 #ifndef ATOMIC_EXCHANGE_PTR
366 #define ATOMIC_EXCHANGE_PTR ATOMIC_EXCHANGE
367 #define ATOMIC_COMPARE_EXCHANGE_PTR_STRONG ATOMIC_COMPARE_EXCHANGE_STRONG
368 #define ATOMIC_COMPARE_EXCHANGE_PTR_WEAK ATOMIC_COMPARE_EXCHANGE_WEAK
369 #endif
370 
371 /* If no weak cmpxchg is provided (not all systems will have one), substitute a
372  * strong cmpxchg. */
373 #ifndef ATOMIC_COMPARE_EXCHANGE_WEAK
374 #define ATOMIC_COMPARE_EXCHANGE_WEAK ATOMIC_COMPARE_EXCHANGE_STRONG
375 #endif
376 #ifndef ATOMIC_COMPARE_EXCHANGE_PTR_WEAK
377 #define ATOMIC_COMPARE_EXCHANGE_PTR_WEAK ATOMIC_COMPARE_EXCHANGE_PTR_STRONG
378 #endif
379 
380 /* If no ATOMIC_FLAG is defined, simulate one with an atomic int using exchange
381  * and store ops.
382  */
383 #ifndef ATOMIC_FLAG
384 #define ATOMIC_FLAG ATOMIC(int)
385 #define ATOMIC_FLAG_INIT ATOMIC_INIT_STATIC(0)
386 #define ATOMIC_FLAG_TEST_AND_SET(_val, _MO) ATOMIC_EXCHANGE(_val, 1, _MO)
387 #define ATOMIC_FLAG_CLEAR(_val, _MO) ATOMIC_STORE(_val, 0, _MO)
388 #endif
389 
390 
391 #define ATOMIC_LOAD_SEQ(_val) ATOMIC_LOAD(_val, almemory_order_seq_cst)
392 #define ATOMIC_STORE_SEQ(_val, _newval) ATOMIC_STORE(_val, _newval, almemory_order_seq_cst)
393 
394 #define ATOMIC_ADD_SEQ(_val, _incr) ATOMIC_ADD(_val, _incr, almemory_order_seq_cst)
395 #define ATOMIC_SUB_SEQ(_val, _decr) ATOMIC_SUB(_val, _decr, almemory_order_seq_cst)
396 
397 #define ATOMIC_EXCHANGE_SEQ(_val, _newval) ATOMIC_EXCHANGE(_val, _newval, almemory_order_seq_cst)
398 #define ATOMIC_COMPARE_EXCHANGE_STRONG_SEQ(_val, _oldval, _newval) \
399  ATOMIC_COMPARE_EXCHANGE_STRONG(_val, _oldval, _newval, almemory_order_seq_cst, almemory_order_seq_cst)
400 #define ATOMIC_COMPARE_EXCHANGE_WEAK_SEQ(_val, _oldval, _newval) \
401  ATOMIC_COMPARE_EXCHANGE_WEAK(_val, _oldval, _newval, almemory_order_seq_cst, almemory_order_seq_cst)
402 
403 #define ATOMIC_EXCHANGE_PTR_SEQ(_val, _newval) ATOMIC_EXCHANGE_PTR(_val, _newval, almemory_order_seq_cst)
404 #define ATOMIC_COMPARE_EXCHANGE_PTR_STRONG_SEQ(_val, _oldval, _newval) \
405  ATOMIC_COMPARE_EXCHANGE_PTR_STRONG(_val, _oldval, _newval, almemory_order_seq_cst, almemory_order_seq_cst)
406 #define ATOMIC_COMPARE_EXCHANGE_PTR_WEAK_SEQ(_val, _oldval, _newval) \
407  ATOMIC_COMPARE_EXCHANGE_PTR_WEAK(_val, _oldval, _newval, almemory_order_seq_cst, almemory_order_seq_cst)
408 
409 
410 typedef unsigned int uint;
411 typedef ATOMIC(uint) RefCount;
412 
413 inline void InitRef(RefCount *ptr, uint value)
414 { ATOMIC_INIT(ptr, value); }
415 inline uint ReadRef(RefCount *ptr)
416 { return ATOMIC_LOAD(ptr, almemory_order_acquire); }
417 inline uint IncrementRef(RefCount *ptr)
418 { return ATOMIC_ADD(ptr, 1, almemory_order_acq_rel)+1; }
419 inline uint DecrementRef(RefCount *ptr)
420 { return ATOMIC_SUB(ptr, 1, almemory_order_acq_rel)-1; }
421 
422 
423 /* WARNING: A livelock is theoretically possible if another thread keeps
424  * changing the head without giving this a chance to actually swap in the new
425  * one (practically impossible with this little code, but...).
426  */
427 #define ATOMIC_REPLACE_HEAD(T, _head, _entry) do { \
428  T _first = ATOMIC_LOAD(_head, almemory_order_acquire); \
429  do { \
430  ATOMIC_STORE(&(_entry)->next, _first, almemory_order_relaxed); \
431  } while(ATOMIC_COMPARE_EXCHANGE_PTR_WEAK(_head, &_first, _entry, \
432  almemory_order_acq_rel, almemory_order_acquire) == 0); \
433 } while(0)
434 
435 #ifdef __cplusplus
436 }
437 #endif
438 
439 #endif /* AL_ATOMIC_H */
Generated by   doxygen 1.8.13