net/mlx5: Better return types for CQE API
[sfrench/cifs-2.6.git] / include / linux / atomic.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Atomic operations usable in machine independent code */
3 #ifndef _LINUX_ATOMIC_H
4 #define _LINUX_ATOMIC_H
5 #include <asm/atomic.h>
6 #include <asm/barrier.h>
7
8 /*
9  * Relaxed variants of xchg, cmpxchg and some atomic operations.
10  *
11  * We support four variants:
12  *
13  * - Fully ordered: The default implementation, no suffix required.
14  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
15  * - Release: Provides RELEASE semantics, _release suffix.
16  * - Relaxed: No ordering guarantees, _relaxed suffix.
17  *
18  * For compound atomics performing both a load and a store, ACQUIRE
19  * semantics apply only to the load and RELEASE semantics only to the
20  * store portion of the operation. Note that a failed cmpxchg_acquire
21  * does -not- imply any memory ordering constraints.
22  *
23  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
24  */
25
26 #ifndef atomic_read_acquire
27 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
28 #endif
29
30 #ifndef atomic_set_release
31 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
32 #endif
33
34 /*
35  * The idea here is to build acquire/release variants by adding explicit
36  * barriers on top of the relaxed variant. In the case where the relaxed
37  * variant is already fully ordered, no additional barriers are needed.
38  *
39  * Besides, if an arch has a special barrier for acquire/release, it could
40  * implement its own __atomic_op_* and use the same framework for building
41  * variants
42  *
43  * If an architecture overrides __atomic_op_acquire() it will probably want
44  * to define smp_mb__after_spinlock().
45  */
46 #ifndef __atomic_op_acquire
47 #define __atomic_op_acquire(op, args...)                                \
48 ({                                                                      \
49         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
50         smp_mb__after_atomic();                                         \
51         __ret;                                                          \
52 })
53 #endif
54
55 #ifndef __atomic_op_release
56 #define __atomic_op_release(op, args...)                                \
57 ({                                                                      \
58         smp_mb__before_atomic();                                        \
59         op##_relaxed(args);                                             \
60 })
61 #endif
62
63 #ifndef __atomic_op_fence
64 #define __atomic_op_fence(op, args...)                                  \
65 ({                                                                      \
66         typeof(op##_relaxed(args)) __ret;                               \
67         smp_mb__before_atomic();                                        \
68         __ret = op##_relaxed(args);                                     \
69         smp_mb__after_atomic();                                         \
70         __ret;                                                          \
71 })
72 #endif
73
74 /* atomic_add_return_relaxed */
75 #ifndef atomic_add_return_relaxed
76 #define  atomic_add_return_relaxed      atomic_add_return
77 #define  atomic_add_return_acquire      atomic_add_return
78 #define  atomic_add_return_release      atomic_add_return
79
80 #else /* atomic_add_return_relaxed */
81
82 #ifndef atomic_add_return_acquire
83 #define  atomic_add_return_acquire(...)                                 \
84         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
85 #endif
86
87 #ifndef atomic_add_return_release
88 #define  atomic_add_return_release(...)                                 \
89         __atomic_op_release(atomic_add_return, __VA_ARGS__)
90 #endif
91
92 #ifndef atomic_add_return
93 #define  atomic_add_return(...)                                         \
94         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
95 #endif
96 #endif /* atomic_add_return_relaxed */
97
98 /* atomic_inc_return_relaxed */
99 #ifndef atomic_inc_return_relaxed
100 #define  atomic_inc_return_relaxed      atomic_inc_return
101 #define  atomic_inc_return_acquire      atomic_inc_return
102 #define  atomic_inc_return_release      atomic_inc_return
103
104 #else /* atomic_inc_return_relaxed */
105
106 #ifndef atomic_inc_return_acquire
107 #define  atomic_inc_return_acquire(...)                                 \
108         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
109 #endif
110
111 #ifndef atomic_inc_return_release
112 #define  atomic_inc_return_release(...)                                 \
113         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
114 #endif
115
116 #ifndef atomic_inc_return
117 #define  atomic_inc_return(...)                                         \
118         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
119 #endif
120 #endif /* atomic_inc_return_relaxed */
121
122 /* atomic_sub_return_relaxed */
123 #ifndef atomic_sub_return_relaxed
124 #define  atomic_sub_return_relaxed      atomic_sub_return
125 #define  atomic_sub_return_acquire      atomic_sub_return
126 #define  atomic_sub_return_release      atomic_sub_return
127
128 #else /* atomic_sub_return_relaxed */
129
130 #ifndef atomic_sub_return_acquire
131 #define  atomic_sub_return_acquire(...)                                 \
132         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
133 #endif
134
135 #ifndef atomic_sub_return_release
136 #define  atomic_sub_return_release(...)                                 \
137         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
138 #endif
139
140 #ifndef atomic_sub_return
141 #define  atomic_sub_return(...)                                         \
142         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
143 #endif
144 #endif /* atomic_sub_return_relaxed */
145
146 /* atomic_dec_return_relaxed */
147 #ifndef atomic_dec_return_relaxed
148 #define  atomic_dec_return_relaxed      atomic_dec_return
149 #define  atomic_dec_return_acquire      atomic_dec_return
150 #define  atomic_dec_return_release      atomic_dec_return
151
152 #else /* atomic_dec_return_relaxed */
153
154 #ifndef atomic_dec_return_acquire
155 #define  atomic_dec_return_acquire(...)                                 \
156         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
157 #endif
158
159 #ifndef atomic_dec_return_release
160 #define  atomic_dec_return_release(...)                                 \
161         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
162 #endif
163
164 #ifndef atomic_dec_return
165 #define  atomic_dec_return(...)                                         \
166         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
167 #endif
168 #endif /* atomic_dec_return_relaxed */
169
170
171 /* atomic_fetch_add_relaxed */
172 #ifndef atomic_fetch_add_relaxed
173 #define atomic_fetch_add_relaxed        atomic_fetch_add
174 #define atomic_fetch_add_acquire        atomic_fetch_add
175 #define atomic_fetch_add_release        atomic_fetch_add
176
177 #else /* atomic_fetch_add_relaxed */
178
179 #ifndef atomic_fetch_add_acquire
180 #define atomic_fetch_add_acquire(...)                                   \
181         __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
182 #endif
183
184 #ifndef atomic_fetch_add_release
185 #define atomic_fetch_add_release(...)                                   \
186         __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
187 #endif
188
189 #ifndef atomic_fetch_add
190 #define atomic_fetch_add(...)                                           \
191         __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
192 #endif
193 #endif /* atomic_fetch_add_relaxed */
194
195 /* atomic_fetch_inc_relaxed */
196 #ifndef atomic_fetch_inc_relaxed
197
198 #ifndef atomic_fetch_inc
199 #define atomic_fetch_inc(v)             atomic_fetch_add(1, (v))
200 #define atomic_fetch_inc_relaxed(v)     atomic_fetch_add_relaxed(1, (v))
201 #define atomic_fetch_inc_acquire(v)     atomic_fetch_add_acquire(1, (v))
202 #define atomic_fetch_inc_release(v)     atomic_fetch_add_release(1, (v))
203 #else /* atomic_fetch_inc */
204 #define atomic_fetch_inc_relaxed        atomic_fetch_inc
205 #define atomic_fetch_inc_acquire        atomic_fetch_inc
206 #define atomic_fetch_inc_release        atomic_fetch_inc
207 #endif /* atomic_fetch_inc */
208
209 #else /* atomic_fetch_inc_relaxed */
210
211 #ifndef atomic_fetch_inc_acquire
212 #define atomic_fetch_inc_acquire(...)                                   \
213         __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
214 #endif
215
216 #ifndef atomic_fetch_inc_release
217 #define atomic_fetch_inc_release(...)                                   \
218         __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
219 #endif
220
221 #ifndef atomic_fetch_inc
222 #define atomic_fetch_inc(...)                                           \
223         __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
224 #endif
225 #endif /* atomic_fetch_inc_relaxed */
226
227 /* atomic_fetch_sub_relaxed */
228 #ifndef atomic_fetch_sub_relaxed
229 #define atomic_fetch_sub_relaxed        atomic_fetch_sub
230 #define atomic_fetch_sub_acquire        atomic_fetch_sub
231 #define atomic_fetch_sub_release        atomic_fetch_sub
232
233 #else /* atomic_fetch_sub_relaxed */
234
235 #ifndef atomic_fetch_sub_acquire
236 #define atomic_fetch_sub_acquire(...)                                   \
237         __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
238 #endif
239
240 #ifndef atomic_fetch_sub_release
241 #define atomic_fetch_sub_release(...)                                   \
242         __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
243 #endif
244
245 #ifndef atomic_fetch_sub
246 #define atomic_fetch_sub(...)                                           \
247         __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
248 #endif
249 #endif /* atomic_fetch_sub_relaxed */
250
251 /* atomic_fetch_dec_relaxed */
252 #ifndef atomic_fetch_dec_relaxed
253
254 #ifndef atomic_fetch_dec
255 #define atomic_fetch_dec(v)             atomic_fetch_sub(1, (v))
256 #define atomic_fetch_dec_relaxed(v)     atomic_fetch_sub_relaxed(1, (v))
257 #define atomic_fetch_dec_acquire(v)     atomic_fetch_sub_acquire(1, (v))
258 #define atomic_fetch_dec_release(v)     atomic_fetch_sub_release(1, (v))
259 #else /* atomic_fetch_dec */
260 #define atomic_fetch_dec_relaxed        atomic_fetch_dec
261 #define atomic_fetch_dec_acquire        atomic_fetch_dec
262 #define atomic_fetch_dec_release        atomic_fetch_dec
263 #endif /* atomic_fetch_dec */
264
265 #else /* atomic_fetch_dec_relaxed */
266
267 #ifndef atomic_fetch_dec_acquire
268 #define atomic_fetch_dec_acquire(...)                                   \
269         __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
270 #endif
271
272 #ifndef atomic_fetch_dec_release
273 #define atomic_fetch_dec_release(...)                                   \
274         __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
275 #endif
276
277 #ifndef atomic_fetch_dec
278 #define atomic_fetch_dec(...)                                           \
279         __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
280 #endif
281 #endif /* atomic_fetch_dec_relaxed */
282
283 /* atomic_fetch_or_relaxed */
284 #ifndef atomic_fetch_or_relaxed
285 #define atomic_fetch_or_relaxed atomic_fetch_or
286 #define atomic_fetch_or_acquire atomic_fetch_or
287 #define atomic_fetch_or_release atomic_fetch_or
288
289 #else /* atomic_fetch_or_relaxed */
290
291 #ifndef atomic_fetch_or_acquire
292 #define atomic_fetch_or_acquire(...)                                    \
293         __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
294 #endif
295
296 #ifndef atomic_fetch_or_release
297 #define atomic_fetch_or_release(...)                                    \
298         __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
299 #endif
300
301 #ifndef atomic_fetch_or
302 #define atomic_fetch_or(...)                                            \
303         __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
304 #endif
305 #endif /* atomic_fetch_or_relaxed */
306
307 /* atomic_fetch_and_relaxed */
308 #ifndef atomic_fetch_and_relaxed
309 #define atomic_fetch_and_relaxed        atomic_fetch_and
310 #define atomic_fetch_and_acquire        atomic_fetch_and
311 #define atomic_fetch_and_release        atomic_fetch_and
312
313 #else /* atomic_fetch_and_relaxed */
314
315 #ifndef atomic_fetch_and_acquire
316 #define atomic_fetch_and_acquire(...)                                   \
317         __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
318 #endif
319
320 #ifndef atomic_fetch_and_release
321 #define atomic_fetch_and_release(...)                                   \
322         __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
323 #endif
324
325 #ifndef atomic_fetch_and
326 #define atomic_fetch_and(...)                                           \
327         __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
328 #endif
329 #endif /* atomic_fetch_and_relaxed */
330
331 #ifdef atomic_andnot
332 /* atomic_fetch_andnot_relaxed */
333 #ifndef atomic_fetch_andnot_relaxed
334 #define atomic_fetch_andnot_relaxed     atomic_fetch_andnot
335 #define atomic_fetch_andnot_acquire     atomic_fetch_andnot
336 #define atomic_fetch_andnot_release     atomic_fetch_andnot
337
338 #else /* atomic_fetch_andnot_relaxed */
339
340 #ifndef atomic_fetch_andnot_acquire
341 #define atomic_fetch_andnot_acquire(...)                                        \
342         __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
343 #endif
344
345 #ifndef atomic_fetch_andnot_release
346 #define atomic_fetch_andnot_release(...)                                        \
347         __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
348 #endif
349
350 #ifndef atomic_fetch_andnot
351 #define atomic_fetch_andnot(...)                                                \
352         __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
353 #endif
354 #endif /* atomic_fetch_andnot_relaxed */
355 #endif /* atomic_andnot */
356
357 /* atomic_fetch_xor_relaxed */
358 #ifndef atomic_fetch_xor_relaxed
359 #define atomic_fetch_xor_relaxed        atomic_fetch_xor
360 #define atomic_fetch_xor_acquire        atomic_fetch_xor
361 #define atomic_fetch_xor_release        atomic_fetch_xor
362
363 #else /* atomic_fetch_xor_relaxed */
364
365 #ifndef atomic_fetch_xor_acquire
366 #define atomic_fetch_xor_acquire(...)                                   \
367         __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
368 #endif
369
370 #ifndef atomic_fetch_xor_release
371 #define atomic_fetch_xor_release(...)                                   \
372         __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
373 #endif
374
375 #ifndef atomic_fetch_xor
376 #define atomic_fetch_xor(...)                                           \
377         __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
378 #endif
379 #endif /* atomic_fetch_xor_relaxed */
380
381
382 /* atomic_xchg_relaxed */
383 #ifndef atomic_xchg_relaxed
384 #define  atomic_xchg_relaxed            atomic_xchg
385 #define  atomic_xchg_acquire            atomic_xchg
386 #define  atomic_xchg_release            atomic_xchg
387
388 #else /* atomic_xchg_relaxed */
389
390 #ifndef atomic_xchg_acquire
391 #define  atomic_xchg_acquire(...)                                       \
392         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
393 #endif
394
395 #ifndef atomic_xchg_release
396 #define  atomic_xchg_release(...)                                       \
397         __atomic_op_release(atomic_xchg, __VA_ARGS__)
398 #endif
399
400 #ifndef atomic_xchg
401 #define  atomic_xchg(...)                                               \
402         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
403 #endif
404 #endif /* atomic_xchg_relaxed */
405
406 /* atomic_cmpxchg_relaxed */
407 #ifndef atomic_cmpxchg_relaxed
408 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
409 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
410 #define  atomic_cmpxchg_release         atomic_cmpxchg
411
412 #else /* atomic_cmpxchg_relaxed */
413
414 #ifndef atomic_cmpxchg_acquire
415 #define  atomic_cmpxchg_acquire(...)                                    \
416         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
417 #endif
418
419 #ifndef atomic_cmpxchg_release
420 #define  atomic_cmpxchg_release(...)                                    \
421         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
422 #endif
423
424 #ifndef atomic_cmpxchg
425 #define  atomic_cmpxchg(...)                                            \
426         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
427 #endif
428 #endif /* atomic_cmpxchg_relaxed */
429
430 #ifndef atomic_try_cmpxchg
431
432 #define __atomic_try_cmpxchg(type, _p, _po, _n)                         \
433 ({                                                                      \
434         typeof(_po) __po = (_po);                                       \
435         typeof(*(_po)) __r, __o = *__po;                                \
436         __r = atomic_cmpxchg##type((_p), __o, (_n));                    \
437         if (unlikely(__r != __o))                                       \
438                 *__po = __r;                                            \
439         likely(__r == __o);                                             \
440 })
441
442 #define atomic_try_cmpxchg(_p, _po, _n)         __atomic_try_cmpxchg(, _p, _po, _n)
443 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
444 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
445 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
446
447 #else /* atomic_try_cmpxchg */
448 #define atomic_try_cmpxchg_relaxed      atomic_try_cmpxchg
449 #define atomic_try_cmpxchg_acquire      atomic_try_cmpxchg
450 #define atomic_try_cmpxchg_release      atomic_try_cmpxchg
451 #endif /* atomic_try_cmpxchg */
452
453 /* cmpxchg_relaxed */
454 #ifndef cmpxchg_relaxed
455 #define  cmpxchg_relaxed                cmpxchg
456 #define  cmpxchg_acquire                cmpxchg
457 #define  cmpxchg_release                cmpxchg
458
459 #else /* cmpxchg_relaxed */
460
461 #ifndef cmpxchg_acquire
462 #define  cmpxchg_acquire(...)                                           \
463         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
464 #endif
465
466 #ifndef cmpxchg_release
467 #define  cmpxchg_release(...)                                           \
468         __atomic_op_release(cmpxchg, __VA_ARGS__)
469 #endif
470
471 #ifndef cmpxchg
472 #define  cmpxchg(...)                                                   \
473         __atomic_op_fence(cmpxchg, __VA_ARGS__)
474 #endif
475 #endif /* cmpxchg_relaxed */
476
477 /* cmpxchg64_relaxed */
478 #ifndef cmpxchg64_relaxed
479 #define  cmpxchg64_relaxed              cmpxchg64
480 #define  cmpxchg64_acquire              cmpxchg64
481 #define  cmpxchg64_release              cmpxchg64
482
483 #else /* cmpxchg64_relaxed */
484
485 #ifndef cmpxchg64_acquire
486 #define  cmpxchg64_acquire(...)                                         \
487         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
488 #endif
489
490 #ifndef cmpxchg64_release
491 #define  cmpxchg64_release(...)                                         \
492         __atomic_op_release(cmpxchg64, __VA_ARGS__)
493 #endif
494
495 #ifndef cmpxchg64
496 #define  cmpxchg64(...)                                                 \
497         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
498 #endif
499 #endif /* cmpxchg64_relaxed */
500
501 /* xchg_relaxed */
502 #ifndef xchg_relaxed
503 #define  xchg_relaxed                   xchg
504 #define  xchg_acquire                   xchg
505 #define  xchg_release                   xchg
506
507 #else /* xchg_relaxed */
508
509 #ifndef xchg_acquire
510 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
511 #endif
512
513 #ifndef xchg_release
514 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
515 #endif
516
517 #ifndef xchg
518 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
519 #endif
520 #endif /* xchg_relaxed */
521
522 /**
523  * atomic_add_unless - add unless the number is already a given value
524  * @v: pointer of type atomic_t
525  * @a: the amount to add to v...
526  * @u: ...unless v is equal to u.
527  *
528  * Atomically adds @a to @v, so long as @v was not already @u.
529  * Returns non-zero if @v was not @u, and zero otherwise.
530  */
531 static inline int atomic_add_unless(atomic_t *v, int a, int u)
532 {
533         return __atomic_add_unless(v, a, u) != u;
534 }
535
536 /**
537  * atomic_inc_not_zero - increment unless the number is zero
538  * @v: pointer of type atomic_t
539  *
540  * Atomically increments @v by 1, so long as @v is non-zero.
541  * Returns non-zero if @v was non-zero, and zero otherwise.
542  */
543 #ifndef atomic_inc_not_zero
544 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
545 #endif
546
547 #ifndef atomic_andnot
548 static inline void atomic_andnot(int i, atomic_t *v)
549 {
550         atomic_and(~i, v);
551 }
552
553 static inline int atomic_fetch_andnot(int i, atomic_t *v)
554 {
555         return atomic_fetch_and(~i, v);
556 }
557
558 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
559 {
560         return atomic_fetch_and_relaxed(~i, v);
561 }
562
563 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
564 {
565         return atomic_fetch_and_acquire(~i, v);
566 }
567
568 static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
569 {
570         return atomic_fetch_and_release(~i, v);
571 }
572 #endif
573
574 /**
575  * atomic_inc_not_zero_hint - increment if not null
576  * @v: pointer of type atomic_t
577  * @hint: probable value of the atomic before the increment
578  *
579  * This version of atomic_inc_not_zero() gives a hint of probable
580  * value of the atomic. This helps processor to not read the memory
581  * before doing the atomic read/modify/write cycle, lowering
582  * number of bus transactions on some arches.
583  *
584  * Returns: 0 if increment was not done, 1 otherwise.
585  */
586 #ifndef atomic_inc_not_zero_hint
587 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
588 {
589         int val, c = hint;
590
591         /* sanity test, should be removed by compiler if hint is a constant */
592         if (!hint)
593                 return atomic_inc_not_zero(v);
594
595         do {
596                 val = atomic_cmpxchg(v, c, c + 1);
597                 if (val == c)
598                         return 1;
599                 c = val;
600         } while (c);
601
602         return 0;
603 }
604 #endif
605
606 #ifndef atomic_inc_unless_negative
607 static inline int atomic_inc_unless_negative(atomic_t *p)
608 {
609         int v, v1;
610         for (v = 0; v >= 0; v = v1) {
611                 v1 = atomic_cmpxchg(p, v, v + 1);
612                 if (likely(v1 == v))
613                         return 1;
614         }
615         return 0;
616 }
617 #endif
618
619 #ifndef atomic_dec_unless_positive
620 static inline int atomic_dec_unless_positive(atomic_t *p)
621 {
622         int v, v1;
623         for (v = 0; v <= 0; v = v1) {
624                 v1 = atomic_cmpxchg(p, v, v - 1);
625                 if (likely(v1 == v))
626                         return 1;
627         }
628         return 0;
629 }
630 #endif
631
632 /*
633  * atomic_dec_if_positive - decrement by 1 if old value positive
634  * @v: pointer of type atomic_t
635  *
636  * The function returns the old value of *v minus 1, even if
637  * the atomic variable, v, was not decremented.
638  */
639 #ifndef atomic_dec_if_positive
640 static inline int atomic_dec_if_positive(atomic_t *v)
641 {
642         int c, old, dec;
643         c = atomic_read(v);
644         for (;;) {
645                 dec = c - 1;
646                 if (unlikely(dec < 0))
647                         break;
648                 old = atomic_cmpxchg((v), c, dec);
649                 if (likely(old == c))
650                         break;
651                 c = old;
652         }
653         return dec;
654 }
655 #endif
656
657 #define atomic_cond_read_relaxed(v, c)  smp_cond_load_relaxed(&(v)->counter, (c))
658 #define atomic_cond_read_acquire(v, c)  smp_cond_load_acquire(&(v)->counter, (c))
659
660 #ifdef CONFIG_GENERIC_ATOMIC64
661 #include <asm-generic/atomic64.h>
662 #endif
663
664 #ifndef atomic64_read_acquire
665 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
666 #endif
667
668 #ifndef atomic64_set_release
669 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
670 #endif
671
672 /* atomic64_add_return_relaxed */
673 #ifndef atomic64_add_return_relaxed
674 #define  atomic64_add_return_relaxed    atomic64_add_return
675 #define  atomic64_add_return_acquire    atomic64_add_return
676 #define  atomic64_add_return_release    atomic64_add_return
677
678 #else /* atomic64_add_return_relaxed */
679
680 #ifndef atomic64_add_return_acquire
681 #define  atomic64_add_return_acquire(...)                               \
682         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
683 #endif
684
685 #ifndef atomic64_add_return_release
686 #define  atomic64_add_return_release(...)                               \
687         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
688 #endif
689
690 #ifndef atomic64_add_return
691 #define  atomic64_add_return(...)                                       \
692         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
693 #endif
694 #endif /* atomic64_add_return_relaxed */
695
696 /* atomic64_inc_return_relaxed */
697 #ifndef atomic64_inc_return_relaxed
698 #define  atomic64_inc_return_relaxed    atomic64_inc_return
699 #define  atomic64_inc_return_acquire    atomic64_inc_return
700 #define  atomic64_inc_return_release    atomic64_inc_return
701
702 #else /* atomic64_inc_return_relaxed */
703
704 #ifndef atomic64_inc_return_acquire
705 #define  atomic64_inc_return_acquire(...)                               \
706         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
707 #endif
708
709 #ifndef atomic64_inc_return_release
710 #define  atomic64_inc_return_release(...)                               \
711         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
712 #endif
713
714 #ifndef atomic64_inc_return
715 #define  atomic64_inc_return(...)                                       \
716         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
717 #endif
718 #endif /* atomic64_inc_return_relaxed */
719
720
721 /* atomic64_sub_return_relaxed */
722 #ifndef atomic64_sub_return_relaxed
723 #define  atomic64_sub_return_relaxed    atomic64_sub_return
724 #define  atomic64_sub_return_acquire    atomic64_sub_return
725 #define  atomic64_sub_return_release    atomic64_sub_return
726
727 #else /* atomic64_sub_return_relaxed */
728
729 #ifndef atomic64_sub_return_acquire
730 #define  atomic64_sub_return_acquire(...)                               \
731         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
732 #endif
733
734 #ifndef atomic64_sub_return_release
735 #define  atomic64_sub_return_release(...)                               \
736         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
737 #endif
738
739 #ifndef atomic64_sub_return
740 #define  atomic64_sub_return(...)                                       \
741         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
742 #endif
743 #endif /* atomic64_sub_return_relaxed */
744
745 /* atomic64_dec_return_relaxed */
746 #ifndef atomic64_dec_return_relaxed
747 #define  atomic64_dec_return_relaxed    atomic64_dec_return
748 #define  atomic64_dec_return_acquire    atomic64_dec_return
749 #define  atomic64_dec_return_release    atomic64_dec_return
750
751 #else /* atomic64_dec_return_relaxed */
752
753 #ifndef atomic64_dec_return_acquire
754 #define  atomic64_dec_return_acquire(...)                               \
755         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
756 #endif
757
758 #ifndef atomic64_dec_return_release
759 #define  atomic64_dec_return_release(...)                               \
760         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
761 #endif
762
763 #ifndef atomic64_dec_return
764 #define  atomic64_dec_return(...)                                       \
765         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
766 #endif
767 #endif /* atomic64_dec_return_relaxed */
768
769
770 /* atomic64_fetch_add_relaxed */
771 #ifndef atomic64_fetch_add_relaxed
772 #define atomic64_fetch_add_relaxed      atomic64_fetch_add
773 #define atomic64_fetch_add_acquire      atomic64_fetch_add
774 #define atomic64_fetch_add_release      atomic64_fetch_add
775
776 #else /* atomic64_fetch_add_relaxed */
777
778 #ifndef atomic64_fetch_add_acquire
779 #define atomic64_fetch_add_acquire(...)                                 \
780         __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
781 #endif
782
783 #ifndef atomic64_fetch_add_release
784 #define atomic64_fetch_add_release(...)                                 \
785         __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
786 #endif
787
788 #ifndef atomic64_fetch_add
789 #define atomic64_fetch_add(...)                                         \
790         __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
791 #endif
792 #endif /* atomic64_fetch_add_relaxed */
793
794 /* atomic64_fetch_inc_relaxed */
795 #ifndef atomic64_fetch_inc_relaxed
796
797 #ifndef atomic64_fetch_inc
798 #define atomic64_fetch_inc(v)           atomic64_fetch_add(1, (v))
799 #define atomic64_fetch_inc_relaxed(v)   atomic64_fetch_add_relaxed(1, (v))
800 #define atomic64_fetch_inc_acquire(v)   atomic64_fetch_add_acquire(1, (v))
801 #define atomic64_fetch_inc_release(v)   atomic64_fetch_add_release(1, (v))
802 #else /* atomic64_fetch_inc */
803 #define atomic64_fetch_inc_relaxed      atomic64_fetch_inc
804 #define atomic64_fetch_inc_acquire      atomic64_fetch_inc
805 #define atomic64_fetch_inc_release      atomic64_fetch_inc
806 #endif /* atomic64_fetch_inc */
807
808 #else /* atomic64_fetch_inc_relaxed */
809
810 #ifndef atomic64_fetch_inc_acquire
811 #define atomic64_fetch_inc_acquire(...)                                 \
812         __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
813 #endif
814
815 #ifndef atomic64_fetch_inc_release
816 #define atomic64_fetch_inc_release(...)                                 \
817         __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
818 #endif
819
820 #ifndef atomic64_fetch_inc
821 #define atomic64_fetch_inc(...)                                         \
822         __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
823 #endif
824 #endif /* atomic64_fetch_inc_relaxed */
825
826 /* atomic64_fetch_sub_relaxed */
827 #ifndef atomic64_fetch_sub_relaxed
828 #define atomic64_fetch_sub_relaxed      atomic64_fetch_sub
829 #define atomic64_fetch_sub_acquire      atomic64_fetch_sub
830 #define atomic64_fetch_sub_release      atomic64_fetch_sub
831
832 #else /* atomic64_fetch_sub_relaxed */
833
834 #ifndef atomic64_fetch_sub_acquire
835 #define atomic64_fetch_sub_acquire(...)                                 \
836         __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
837 #endif
838
839 #ifndef atomic64_fetch_sub_release
840 #define atomic64_fetch_sub_release(...)                                 \
841         __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
842 #endif
843
844 #ifndef atomic64_fetch_sub
845 #define atomic64_fetch_sub(...)                                         \
846         __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
847 #endif
848 #endif /* atomic64_fetch_sub_relaxed */
849
850 /* atomic64_fetch_dec_relaxed */
851 #ifndef atomic64_fetch_dec_relaxed
852
853 #ifndef atomic64_fetch_dec
854 #define atomic64_fetch_dec(v)           atomic64_fetch_sub(1, (v))
855 #define atomic64_fetch_dec_relaxed(v)   atomic64_fetch_sub_relaxed(1, (v))
856 #define atomic64_fetch_dec_acquire(v)   atomic64_fetch_sub_acquire(1, (v))
857 #define atomic64_fetch_dec_release(v)   atomic64_fetch_sub_release(1, (v))
858 #else /* atomic64_fetch_dec */
859 #define atomic64_fetch_dec_relaxed      atomic64_fetch_dec
860 #define atomic64_fetch_dec_acquire      atomic64_fetch_dec
861 #define atomic64_fetch_dec_release      atomic64_fetch_dec
862 #endif /* atomic64_fetch_dec */
863
864 #else /* atomic64_fetch_dec_relaxed */
865
866 #ifndef atomic64_fetch_dec_acquire
867 #define atomic64_fetch_dec_acquire(...)                                 \
868         __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
869 #endif
870
871 #ifndef atomic64_fetch_dec_release
872 #define atomic64_fetch_dec_release(...)                                 \
873         __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
874 #endif
875
876 #ifndef atomic64_fetch_dec
877 #define atomic64_fetch_dec(...)                                         \
878         __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
879 #endif
880 #endif /* atomic64_fetch_dec_relaxed */
881
882 /* atomic64_fetch_or_relaxed */
883 #ifndef atomic64_fetch_or_relaxed
884 #define atomic64_fetch_or_relaxed       atomic64_fetch_or
885 #define atomic64_fetch_or_acquire       atomic64_fetch_or
886 #define atomic64_fetch_or_release       atomic64_fetch_or
887
888 #else /* atomic64_fetch_or_relaxed */
889
890 #ifndef atomic64_fetch_or_acquire
891 #define atomic64_fetch_or_acquire(...)                                  \
892         __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
893 #endif
894
895 #ifndef atomic64_fetch_or_release
896 #define atomic64_fetch_or_release(...)                                  \
897         __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
898 #endif
899
900 #ifndef atomic64_fetch_or
901 #define atomic64_fetch_or(...)                                          \
902         __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
903 #endif
904 #endif /* atomic64_fetch_or_relaxed */
905
906 /* atomic64_fetch_and_relaxed */
907 #ifndef atomic64_fetch_and_relaxed
908 #define atomic64_fetch_and_relaxed      atomic64_fetch_and
909 #define atomic64_fetch_and_acquire      atomic64_fetch_and
910 #define atomic64_fetch_and_release      atomic64_fetch_and
911
912 #else /* atomic64_fetch_and_relaxed */
913
914 #ifndef atomic64_fetch_and_acquire
915 #define atomic64_fetch_and_acquire(...)                                 \
916         __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
917 #endif
918
919 #ifndef atomic64_fetch_and_release
920 #define atomic64_fetch_and_release(...)                                 \
921         __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
922 #endif
923
924 #ifndef atomic64_fetch_and
925 #define atomic64_fetch_and(...)                                         \
926         __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
927 #endif
928 #endif /* atomic64_fetch_and_relaxed */
929
930 #ifdef atomic64_andnot
931 /* atomic64_fetch_andnot_relaxed */
932 #ifndef atomic64_fetch_andnot_relaxed
933 #define atomic64_fetch_andnot_relaxed   atomic64_fetch_andnot
934 #define atomic64_fetch_andnot_acquire   atomic64_fetch_andnot
935 #define atomic64_fetch_andnot_release   atomic64_fetch_andnot
936
937 #else /* atomic64_fetch_andnot_relaxed */
938
939 #ifndef atomic64_fetch_andnot_acquire
940 #define atomic64_fetch_andnot_acquire(...)                                      \
941         __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
942 #endif
943
944 #ifndef atomic64_fetch_andnot_release
945 #define atomic64_fetch_andnot_release(...)                                      \
946         __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
947 #endif
948
949 #ifndef atomic64_fetch_andnot
950 #define atomic64_fetch_andnot(...)                                              \
951         __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
952 #endif
953 #endif /* atomic64_fetch_andnot_relaxed */
954 #endif /* atomic64_andnot */
955
956 /* atomic64_fetch_xor_relaxed */
957 #ifndef atomic64_fetch_xor_relaxed
958 #define atomic64_fetch_xor_relaxed      atomic64_fetch_xor
959 #define atomic64_fetch_xor_acquire      atomic64_fetch_xor
960 #define atomic64_fetch_xor_release      atomic64_fetch_xor
961
962 #else /* atomic64_fetch_xor_relaxed */
963
964 #ifndef atomic64_fetch_xor_acquire
965 #define atomic64_fetch_xor_acquire(...)                                 \
966         __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
967 #endif
968
969 #ifndef atomic64_fetch_xor_release
970 #define atomic64_fetch_xor_release(...)                                 \
971         __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
972 #endif
973
974 #ifndef atomic64_fetch_xor
975 #define atomic64_fetch_xor(...)                                         \
976         __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
977 #endif
978 #endif /* atomic64_fetch_xor_relaxed */
979
980
981 /* atomic64_xchg_relaxed */
982 #ifndef atomic64_xchg_relaxed
983 #define  atomic64_xchg_relaxed          atomic64_xchg
984 #define  atomic64_xchg_acquire          atomic64_xchg
985 #define  atomic64_xchg_release          atomic64_xchg
986
987 #else /* atomic64_xchg_relaxed */
988
989 #ifndef atomic64_xchg_acquire
990 #define  atomic64_xchg_acquire(...)                                     \
991         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
992 #endif
993
994 #ifndef atomic64_xchg_release
995 #define  atomic64_xchg_release(...)                                     \
996         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
997 #endif
998
999 #ifndef atomic64_xchg
1000 #define  atomic64_xchg(...)                                             \
1001         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
1002 #endif
1003 #endif /* atomic64_xchg_relaxed */
1004
1005 /* atomic64_cmpxchg_relaxed */
1006 #ifndef atomic64_cmpxchg_relaxed
1007 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
1008 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
1009 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
1010
1011 #else /* atomic64_cmpxchg_relaxed */
1012
1013 #ifndef atomic64_cmpxchg_acquire
1014 #define  atomic64_cmpxchg_acquire(...)                                  \
1015         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1016 #endif
1017
1018 #ifndef atomic64_cmpxchg_release
1019 #define  atomic64_cmpxchg_release(...)                                  \
1020         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1021 #endif
1022
1023 #ifndef atomic64_cmpxchg
1024 #define  atomic64_cmpxchg(...)                                          \
1025         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1026 #endif
1027 #endif /* atomic64_cmpxchg_relaxed */
1028
1029 #ifndef atomic64_try_cmpxchg
1030
1031 #define __atomic64_try_cmpxchg(type, _p, _po, _n)                       \
1032 ({                                                                      \
1033         typeof(_po) __po = (_po);                                       \
1034         typeof(*(_po)) __r, __o = *__po;                                \
1035         __r = atomic64_cmpxchg##type((_p), __o, (_n));                  \
1036         if (unlikely(__r != __o))                                       \
1037                 *__po = __r;                                            \
1038         likely(__r == __o);                                             \
1039 })
1040
1041 #define atomic64_try_cmpxchg(_p, _po, _n)               __atomic64_try_cmpxchg(, _p, _po, _n)
1042 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n)       __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1043 #define atomic64_try_cmpxchg_acquire(_p, _po, _n)       __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1044 #define atomic64_try_cmpxchg_release(_p, _po, _n)       __atomic64_try_cmpxchg(_release, _p, _po, _n)
1045
1046 #else /* atomic64_try_cmpxchg */
1047 #define atomic64_try_cmpxchg_relaxed    atomic64_try_cmpxchg
1048 #define atomic64_try_cmpxchg_acquire    atomic64_try_cmpxchg
1049 #define atomic64_try_cmpxchg_release    atomic64_try_cmpxchg
1050 #endif /* atomic64_try_cmpxchg */
1051
1052 #ifndef atomic64_andnot
1053 static inline void atomic64_andnot(long long i, atomic64_t *v)
1054 {
1055         atomic64_and(~i, v);
1056 }
1057
1058 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1059 {
1060         return atomic64_fetch_and(~i, v);
1061 }
1062
1063 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1064 {
1065         return atomic64_fetch_and_relaxed(~i, v);
1066 }
1067
1068 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1069 {
1070         return atomic64_fetch_and_acquire(~i, v);
1071 }
1072
1073 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1074 {
1075         return atomic64_fetch_and_release(~i, v);
1076 }
1077 #endif
1078
1079 #define atomic64_cond_read_relaxed(v, c)        smp_cond_load_relaxed(&(v)->counter, (c))
1080 #define atomic64_cond_read_acquire(v, c)        smp_cond_load_acquire(&(v)->counter, (c))
1081
1082 #include <asm-generic/atomic-long.h>
1083
1084 #endif /* _LINUX_ATOMIC_H */