include/linux/atomic.h

   1 /* Atomic operations usable in machine independent code */
   2 #ifndef _LINUX_ATOMIC_H
   3 #define _LINUX_ATOMIC_H
   4 #include <asm/atomic.h>
   5 #include <asm/barrier.h>
   6
   7 /*
   8  * Relaxed variants of xchg, cmpxchg and some atomic operations.
   9  *
  10  * We support four variants:
  11  *
  12  * - Fully ordered: The default implementation, no suffix required.
  13  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  14  * - Release: Provides RELEASE semantics, _release suffix.
  15  * - Relaxed: No ordering guarantees, _relaxed suffix.
  16  *
  17  * For compound atomics performing both a load and a store, ACQUIRE
  18  * semantics apply only to the load and RELEASE semantics only to the
  19  * store portion of the operation. Note that a failed cmpxchg_acquire
  20  * does -not- imply any memory ordering constraints.
  21  *
  22  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  23  */
  24
  25 #ifndef atomic_read_acquire
  26 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  27 #endif
  28
  29 #ifndef atomic_set_release
  30 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  31 #endif
  32
  33 /*
  34  * The idea here is to build acquire/release variants by adding explicit
  35  * barriers on top of the relaxed variant. In the case where the relaxed
  36  * variant is already fully ordered, no additional barriers are needed.
  37  *
  38  * Besides, if an arch has a special barrier for acquire/release, it could
  39  * implement its own __atomic_op_* and use the same framework for building
  40  * variants
  41  */
  42 #ifndef __atomic_op_acquire
  43 #define __atomic_op_acquire(op, args...)                                \
  44 ({                                                                      \
  45         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  46         smp_mb__after_atomic();                                         \
  47         __ret;                                                          \
  48 })
  49 #endif
  50
  51 #ifndef __atomic_op_release
  52 #define __atomic_op_release(op, args...)                                \
  53 ({                                                                      \
  54         smp_mb__before_atomic();                                        \
  55         op##_relaxed(args);                                             \
  56 })
  57 #endif
  58
  59 #ifndef __atomic_op_fence
  60 #define __atomic_op_fence(op, args...)                                  \
  61 ({                                                                      \
  62         typeof(op##_relaxed(args)) __ret;                               \
  63         smp_mb__before_atomic();                                        \
  64         __ret = op##_relaxed(args);                                     \
  65         smp_mb__after_atomic();                                         \
  66         __ret;                                                          \
  67 })
  68 #endif
  69
  70 /* atomic_add_return_relaxed */
  71 #ifndef atomic_add_return_relaxed
  72 #define  atomic_add_return_relaxed      atomic_add_return
  73 #define  atomic_add_return_acquire      atomic_add_return
  74 #define  atomic_add_return_release      atomic_add_return
  75
  76 #else /* atomic_add_return_relaxed */
  77
  78 #ifndef atomic_add_return_acquire
  79 #define  atomic_add_return_acquire(...)                                 \
  80         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  81 #endif
  82
  83 #ifndef atomic_add_return_release
  84 #define  atomic_add_return_release(...)                                 \
  85         __atomic_op_release(atomic_add_return, __VA_ARGS__)
  86 #endif
  87
  88 #ifndef atomic_add_return
  89 #define  atomic_add_return(...)                                         \
  90         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  91 #endif
  92 #endif /* atomic_add_return_relaxed */
  93
  94 /* atomic_inc_return_relaxed */
  95 #ifndef atomic_inc_return_relaxed
  96 #define  atomic_inc_return_relaxed      atomic_inc_return
  97 #define  atomic_inc_return_acquire      atomic_inc_return
  98 #define  atomic_inc_return_release      atomic_inc_return
  99
 100 #else /* atomic_inc_return_relaxed */
 101
 102 #ifndef atomic_inc_return_acquire
 103 #define  atomic_inc_return_acquire(...)                                 \
 104         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
 105 #endif
 106
 107 #ifndef atomic_inc_return_release
 108 #define  atomic_inc_return_release(...)                                 \
 109         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 110 #endif
 111
 112 #ifndef atomic_inc_return
 113 #define  atomic_inc_return(...)                                         \
 114         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 115 #endif
 116 #endif /* atomic_inc_return_relaxed */
 117
 118 /* atomic_sub_return_relaxed */
 119 #ifndef atomic_sub_return_relaxed
 120 #define  atomic_sub_return_relaxed      atomic_sub_return
 121 #define  atomic_sub_return_acquire      atomic_sub_return
 122 #define  atomic_sub_return_release      atomic_sub_return
 123
 124 #else /* atomic_sub_return_relaxed */
 125
 126 #ifndef atomic_sub_return_acquire
 127 #define  atomic_sub_return_acquire(...)                                 \
 128         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 129 #endif
 130
 131 #ifndef atomic_sub_return_release
 132 #define  atomic_sub_return_release(...)                                 \
 133         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 134 #endif
 135
 136 #ifndef atomic_sub_return
 137 #define  atomic_sub_return(...)                                         \
 138         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 139 #endif
 140 #endif /* atomic_sub_return_relaxed */
 141
 142 /* atomic_dec_return_relaxed */
 143 #ifndef atomic_dec_return_relaxed
 144 #define  atomic_dec_return_relaxed      atomic_dec_return
 145 #define  atomic_dec_return_acquire      atomic_dec_return
 146 #define  atomic_dec_return_release      atomic_dec_return
 147
 148 #else /* atomic_dec_return_relaxed */
 149
 150 #ifndef atomic_dec_return_acquire
 151 #define  atomic_dec_return_acquire(...)                                 \
 152         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 153 #endif
 154
 155 #ifndef atomic_dec_return_release
 156 #define  atomic_dec_return_release(...)                                 \
 157         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 158 #endif
 159
 160 #ifndef atomic_dec_return
 161 #define  atomic_dec_return(...)                                         \
 162         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 163 #endif
 164 #endif /* atomic_dec_return_relaxed */
 165
 166
 167 /* atomic_fetch_add_relaxed */
 168 #ifndef atomic_fetch_add_relaxed
 169 #define atomic_fetch_add_relaxed        atomic_fetch_add
 170 #define atomic_fetch_add_acquire        atomic_fetch_add
 171 #define atomic_fetch_add_release        atomic_fetch_add
 172
 173 #else /* atomic_fetch_add_relaxed */
 174
 175 #ifndef atomic_fetch_add_acquire
 176 #define atomic_fetch_add_acquire(...)                                   \
 177         __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
 178 #endif
 179
 180 #ifndef atomic_fetch_add_release
 181 #define atomic_fetch_add_release(...)                                   \
 182         __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
 183 #endif
 184
 185 #ifndef atomic_fetch_add
 186 #define atomic_fetch_add(...)                                           \
 187         __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
 188 #endif
 189 #endif /* atomic_fetch_add_relaxed */
 190
 191 /* atomic_fetch_inc_relaxed */
 192 #ifndef atomic_fetch_inc_relaxed
 193
 194 #ifndef atomic_fetch_inc
 195 #define atomic_fetch_inc(v)             atomic_fetch_add(1, (v))
 196 #define atomic_fetch_inc_relaxed(v)     atomic_fetch_add_relaxed(1, (v))
 197 #define atomic_fetch_inc_acquire(v)     atomic_fetch_add_acquire(1, (v))
 198 #define atomic_fetch_inc_release(v)     atomic_fetch_add_release(1, (v))
 199 #else /* atomic_fetch_inc */
 200 #define atomic_fetch_inc_relaxed        atomic_fetch_inc
 201 #define atomic_fetch_inc_acquire        atomic_fetch_inc
 202 #define atomic_fetch_inc_release        atomic_fetch_inc
 203 #endif /* atomic_fetch_inc */
 204
 205 #else /* atomic_fetch_inc_relaxed */
 206
 207 #ifndef atomic_fetch_inc_acquire
 208 #define atomic_fetch_inc_acquire(...)                                   \
 209         __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
 210 #endif
 211
 212 #ifndef atomic_fetch_inc_release
 213 #define atomic_fetch_inc_release(...)                                   \
 214         __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
 215 #endif
 216
 217 #ifndef atomic_fetch_inc
 218 #define atomic_fetch_inc(...)                                           \
 219         __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
 220 #endif
 221 #endif /* atomic_fetch_inc_relaxed */
 222
 223 /* atomic_fetch_sub_relaxed */
 224 #ifndef atomic_fetch_sub_relaxed
 225 #define atomic_fetch_sub_relaxed        atomic_fetch_sub
 226 #define atomic_fetch_sub_acquire        atomic_fetch_sub
 227 #define atomic_fetch_sub_release        atomic_fetch_sub
 228
 229 #else /* atomic_fetch_sub_relaxed */
 230
 231 #ifndef atomic_fetch_sub_acquire
 232 #define atomic_fetch_sub_acquire(...)                                   \
 233         __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
 234 #endif
 235
 236 #ifndef atomic_fetch_sub_release
 237 #define atomic_fetch_sub_release(...)                                   \
 238         __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
 239 #endif
 240
 241 #ifndef atomic_fetch_sub
 242 #define atomic_fetch_sub(...)                                           \
 243         __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
 244 #endif
 245 #endif /* atomic_fetch_sub_relaxed */
 246
 247 /* atomic_fetch_dec_relaxed */
 248 #ifndef atomic_fetch_dec_relaxed
 249
 250 #ifndef atomic_fetch_dec
 251 #define atomic_fetch_dec(v)             atomic_fetch_sub(1, (v))
 252 #define atomic_fetch_dec_relaxed(v)     atomic_fetch_sub_relaxed(1, (v))
 253 #define atomic_fetch_dec_acquire(v)     atomic_fetch_sub_acquire(1, (v))
 254 #define atomic_fetch_dec_release(v)     atomic_fetch_sub_release(1, (v))
 255 #else /* atomic_fetch_dec */
 256 #define atomic_fetch_dec_relaxed        atomic_fetch_dec
 257 #define atomic_fetch_dec_acquire        atomic_fetch_dec
 258 #define atomic_fetch_dec_release        atomic_fetch_dec
 259 #endif /* atomic_fetch_dec */
 260
 261 #else /* atomic_fetch_dec_relaxed */
 262
 263 #ifndef atomic_fetch_dec_acquire
 264 #define atomic_fetch_dec_acquire(...)                                   \
 265         __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 266 #endif
 267
 268 #ifndef atomic_fetch_dec_release
 269 #define atomic_fetch_dec_release(...)                                   \
 270         __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 271 #endif
 272
 273 #ifndef atomic_fetch_dec
 274 #define atomic_fetch_dec(...)                                           \
 275         __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 276 #endif
 277 #endif /* atomic_fetch_dec_relaxed */
 278
 279 /* atomic_fetch_or_relaxed */
 280 #ifndef atomic_fetch_or_relaxed
 281 #define atomic_fetch_or_relaxed atomic_fetch_or
 282 #define atomic_fetch_or_acquire atomic_fetch_or
 283 #define atomic_fetch_or_release atomic_fetch_or
 284
 285 #else /* atomic_fetch_or_relaxed */
 286
 287 #ifndef atomic_fetch_or_acquire
 288 #define atomic_fetch_or_acquire(...)                                    \
 289         __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
 290 #endif
 291
 292 #ifndef atomic_fetch_or_release
 293 #define atomic_fetch_or_release(...)                                    \
 294         __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
 295 #endif
 296
 297 #ifndef atomic_fetch_or
 298 #define atomic_fetch_or(...)                                            \
 299         __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
 300 #endif
 301 #endif /* atomic_fetch_or_relaxed */
 302
 303 /* atomic_fetch_and_relaxed */
 304 #ifndef atomic_fetch_and_relaxed
 305 #define atomic_fetch_and_relaxed        atomic_fetch_and
 306 #define atomic_fetch_and_acquire        atomic_fetch_and
 307 #define atomic_fetch_and_release        atomic_fetch_and
 308
 309 #else /* atomic_fetch_and_relaxed */
 310
 311 #ifndef atomic_fetch_and_acquire
 312 #define atomic_fetch_and_acquire(...)                                   \
 313         __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
 314 #endif
 315
 316 #ifndef atomic_fetch_and_release
 317 #define atomic_fetch_and_release(...)                                   \
 318         __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
 319 #endif
 320
 321 #ifndef atomic_fetch_and
 322 #define atomic_fetch_and(...)                                           \
 323         __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
 324 #endif
 325 #endif /* atomic_fetch_and_relaxed */
 326
 327 #ifdef atomic_andnot
 328 /* atomic_fetch_andnot_relaxed */
 329 #ifndef atomic_fetch_andnot_relaxed
 330 #define atomic_fetch_andnot_relaxed     atomic_fetch_andnot
 331 #define atomic_fetch_andnot_acquire     atomic_fetch_andnot
 332 #define atomic_fetch_andnot_release     atomic_fetch_andnot
 333
 334 #else /* atomic_fetch_andnot_relaxed */
 335
 336 #ifndef atomic_fetch_andnot_acquire
 337 #define atomic_fetch_andnot_acquire(...)                                        \
 338         __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
 339 #endif
 340
 341 #ifndef atomic_fetch_andnot_release
 342 #define atomic_fetch_andnot_release(...)                                        \
 343         __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
 344 #endif
 345
 346 #ifndef atomic_fetch_andnot
 347 #define atomic_fetch_andnot(...)                                                \
 348         __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
 349 #endif
 350 #endif /* atomic_fetch_andnot_relaxed */
 351 #endif /* atomic_andnot */
 352
 353 /* atomic_fetch_xor_relaxed */
 354 #ifndef atomic_fetch_xor_relaxed
 355 #define atomic_fetch_xor_relaxed        atomic_fetch_xor
 356 #define atomic_fetch_xor_acquire        atomic_fetch_xor
 357 #define atomic_fetch_xor_release        atomic_fetch_xor
 358
 359 #else /* atomic_fetch_xor_relaxed */
 360
 361 #ifndef atomic_fetch_xor_acquire
 362 #define atomic_fetch_xor_acquire(...)                                   \
 363         __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
 364 #endif
 365
 366 #ifndef atomic_fetch_xor_release
 367 #define atomic_fetch_xor_release(...)                                   \
 368         __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
 369 #endif
 370
 371 #ifndef atomic_fetch_xor
 372 #define atomic_fetch_xor(...)                                           \
 373         __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
 374 #endif
 375 #endif /* atomic_fetch_xor_relaxed */
 376
 377
 378 /* atomic_xchg_relaxed */
 379 #ifndef atomic_xchg_relaxed
 380 #define  atomic_xchg_relaxed            atomic_xchg
 381 #define  atomic_xchg_acquire            atomic_xchg
 382 #define  atomic_xchg_release            atomic_xchg
 383
 384 #else /* atomic_xchg_relaxed */
 385
 386 #ifndef atomic_xchg_acquire
 387 #define  atomic_xchg_acquire(...)                                       \
 388         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 389 #endif
 390
 391 #ifndef atomic_xchg_release
 392 #define  atomic_xchg_release(...)                                       \
 393         __atomic_op_release(atomic_xchg, __VA_ARGS__)
 394 #endif
 395
 396 #ifndef atomic_xchg
 397 #define  atomic_xchg(...)                                               \
 398         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 399 #endif
 400 #endif /* atomic_xchg_relaxed */
 401
 402 /* atomic_cmpxchg_relaxed */
 403 #ifndef atomic_cmpxchg_relaxed
 404 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 405 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
 406 #define  atomic_cmpxchg_release         atomic_cmpxchg
 407
 408 #else /* atomic_cmpxchg_relaxed */
 409
 410 #ifndef atomic_cmpxchg_acquire
 411 #define  atomic_cmpxchg_acquire(...)                                    \
 412         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 413 #endif
 414
 415 #ifndef atomic_cmpxchg_release
 416 #define  atomic_cmpxchg_release(...)                                    \
 417         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 418 #endif
 419
 420 #ifndef atomic_cmpxchg
 421 #define  atomic_cmpxchg(...)                                            \
 422         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 423 #endif
 424 #endif /* atomic_cmpxchg_relaxed */
 425
 426 /* cmpxchg_relaxed */
 427 #ifndef cmpxchg_relaxed
 428 #define  cmpxchg_relaxed                cmpxchg
 429 #define  cmpxchg_acquire                cmpxchg
 430 #define  cmpxchg_release                cmpxchg
 431
 432 #else /* cmpxchg_relaxed */
 433
 434 #ifndef cmpxchg_acquire
 435 #define  cmpxchg_acquire(...)                                           \
 436         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 437 #endif
 438
 439 #ifndef cmpxchg_release
 440 #define  cmpxchg_release(...)                                           \
 441         __atomic_op_release(cmpxchg, __VA_ARGS__)
 442 #endif
 443
 444 #ifndef cmpxchg
 445 #define  cmpxchg(...)                                                   \
 446         __atomic_op_fence(cmpxchg, __VA_ARGS__)
 447 #endif
 448 #endif /* cmpxchg_relaxed */
 449
 450 /* cmpxchg64_relaxed */
 451 #ifndef cmpxchg64_relaxed
 452 #define  cmpxchg64_relaxed              cmpxchg64
 453 #define  cmpxchg64_acquire              cmpxchg64
 454 #define  cmpxchg64_release              cmpxchg64
 455
 456 #else /* cmpxchg64_relaxed */
 457
 458 #ifndef cmpxchg64_acquire
 459 #define  cmpxchg64_acquire(...)                                         \
 460         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 461 #endif
 462
 463 #ifndef cmpxchg64_release
 464 #define  cmpxchg64_release(...)                                         \
 465         __atomic_op_release(cmpxchg64, __VA_ARGS__)
 466 #endif
 467
 468 #ifndef cmpxchg64
 469 #define  cmpxchg64(...)                                                 \
 470         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 471 #endif
 472 #endif /* cmpxchg64_relaxed */
 473
 474 /* xchg_relaxed */
 475 #ifndef xchg_relaxed
 476 #define  xchg_relaxed                   xchg
 477 #define  xchg_acquire                   xchg
 478 #define  xchg_release                   xchg
 479
 480 #else /* xchg_relaxed */
 481
 482 #ifndef xchg_acquire
 483 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 484 #endif
 485
 486 #ifndef xchg_release
 487 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 488 #endif
 489
 490 #ifndef xchg
 491 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 492 #endif
 493 #endif /* xchg_relaxed */
 494
 495 /**
 496  * atomic_add_unless - add unless the number is already a given value
 497  * @v: pointer of type atomic_t
 498  * @a: the amount to add to v...
 499  * @u: ...unless v is equal to u.
 500  *
 501  * Atomically adds @a to @v, so long as @v was not already @u.
 502  * Returns non-zero if @v was not @u, and zero otherwise.
 503  */
 504 static inline int atomic_add_unless(atomic_t *v, int a, int u)
 505 {
 506         return __atomic_add_unless(v, a, u) != u;
 507 }
 508
 509 /**
 510  * atomic_inc_not_zero - increment unless the number is zero
 511  * @v: pointer of type atomic_t
 512  *
 513  * Atomically increments @v by 1, so long as @v is non-zero.
 514  * Returns non-zero if @v was non-zero, and zero otherwise.
 515  */
 516 #ifndef atomic_inc_not_zero
 517 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 518 #endif
 519
 520 #ifndef atomic_andnot
 521 static inline void atomic_andnot(int i, atomic_t *v)
 522 {
 523         atomic_and(~i, v);
 524 }
 525
 526 static inline int atomic_fetch_andnot(int i, atomic_t *v)
 527 {
 528         return atomic_fetch_and(~i, v);
 529 }
 530
 531 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 532 {
 533         return atomic_fetch_and_relaxed(~i, v);
 534 }
 535
 536 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
 537 {
 538         return atomic_fetch_and_acquire(~i, v);
 539 }
 540
 541 static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
 542 {
 543         return atomic_fetch_and_release(~i, v);
 544 }
 545 #endif
 546
 547 /**
 548  * atomic_inc_not_zero_hint - increment if not null
 549  * @v: pointer of type atomic_t
 550  * @hint: probable value of the atomic before the increment
 551  *
 552  * This version of atomic_inc_not_zero() gives a hint of probable
 553  * value of the atomic. This helps processor to not read the memory
 554  * before doing the atomic read/modify/write cycle, lowering
 555  * number of bus transactions on some arches.
 556  *
 557  * Returns: 0 if increment was not done, 1 otherwise.
 558  */
 559 #ifndef atomic_inc_not_zero_hint
 560 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 561 {
 562         int val, c = hint;
 563
 564         /* sanity test, should be removed by compiler if hint is a constant */
 565         if (!hint)
 566                 return atomic_inc_not_zero(v);
 567
 568         do {
 569                 val = atomic_cmpxchg(v, c, c + 1);
 570                 if (val == c)
 571                         return 1;
 572                 c = val;
 573         } while (c);
 574
 575         return 0;
 576 }
 577 #endif
 578
 579 #ifndef atomic_inc_unless_negative
 580 static inline int atomic_inc_unless_negative(atomic_t *p)
 581 {
 582         int v, v1;
 583         for (v = 0; v >= 0; v = v1) {
 584                 v1 = atomic_cmpxchg(p, v, v + 1);
 585                 if (likely(v1 == v))
 586                         return 1;
 587         }
 588         return 0;
 589 }
 590 #endif
 591
 592 #ifndef atomic_dec_unless_positive
 593 static inline int atomic_dec_unless_positive(atomic_t *p)
 594 {
 595         int v, v1;
 596         for (v = 0; v <= 0; v = v1) {
 597                 v1 = atomic_cmpxchg(p, v, v - 1);
 598                 if (likely(v1 == v))
 599                         return 1;
 600         }
 601         return 0;
 602 }
 603 #endif
 604
 605 /*
 606  * atomic_dec_if_positive - decrement by 1 if old value positive
 607  * @v: pointer of type atomic_t
 608  *
 609  * The function returns the old value of *v minus 1, even if
 610  * the atomic variable, v, was not decremented.
 611  */
 612 #ifndef atomic_dec_if_positive
 613 static inline int atomic_dec_if_positive(atomic_t *v)
 614 {
 615         int c, old, dec;
 616         c = atomic_read(v);
 617         for (;;) {
 618                 dec = c - 1;
 619                 if (unlikely(dec < 0))
 620                         break;
 621                 old = atomic_cmpxchg((v), c, dec);
 622                 if (likely(old == c))
 623                         break;
 624                 c = old;
 625         }
 626         return dec;
 627 }
 628 #endif
 629
 630 #ifdef CONFIG_GENERIC_ATOMIC64
 631 #include <asm-generic/atomic64.h>
 632 #endif
 633
 634 #ifndef atomic64_read_acquire
 635 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 636 #endif
 637
 638 #ifndef atomic64_set_release
 639 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 640 #endif
 641
 642 /* atomic64_add_return_relaxed */
 643 #ifndef atomic64_add_return_relaxed
 644 #define  atomic64_add_return_relaxed    atomic64_add_return
 645 #define  atomic64_add_return_acquire    atomic64_add_return
 646 #define  atomic64_add_return_release    atomic64_add_return
 647
 648 #else /* atomic64_add_return_relaxed */
 649
 650 #ifndef atomic64_add_return_acquire
 651 #define  atomic64_add_return_acquire(...)                               \
 652         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 653 #endif
 654
 655 #ifndef atomic64_add_return_release
 656 #define  atomic64_add_return_release(...)                               \
 657         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 658 #endif
 659
 660 #ifndef atomic64_add_return
 661 #define  atomic64_add_return(...)                                       \
 662         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 663 #endif
 664 #endif /* atomic64_add_return_relaxed */
 665
 666 /* atomic64_inc_return_relaxed */
 667 #ifndef atomic64_inc_return_relaxed
 668 #define  atomic64_inc_return_relaxed    atomic64_inc_return
 669 #define  atomic64_inc_return_acquire    atomic64_inc_return
 670 #define  atomic64_inc_return_release    atomic64_inc_return
 671
 672 #else /* atomic64_inc_return_relaxed */
 673
 674 #ifndef atomic64_inc_return_acquire
 675 #define  atomic64_inc_return_acquire(...)                               \
 676         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 677 #endif
 678
 679 #ifndef atomic64_inc_return_release
 680 #define  atomic64_inc_return_release(...)                               \
 681         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 682 #endif
 683
 684 #ifndef atomic64_inc_return
 685 #define  atomic64_inc_return(...)                                       \
 686         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 687 #endif
 688 #endif /* atomic64_inc_return_relaxed */
 689
 690
 691 /* atomic64_sub_return_relaxed */
 692 #ifndef atomic64_sub_return_relaxed
 693 #define  atomic64_sub_return_relaxed    atomic64_sub_return
 694 #define  atomic64_sub_return_acquire    atomic64_sub_return
 695 #define  atomic64_sub_return_release    atomic64_sub_return
 696
 697 #else /* atomic64_sub_return_relaxed */
 698
 699 #ifndef atomic64_sub_return_acquire
 700 #define  atomic64_sub_return_acquire(...)                               \
 701         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 702 #endif
 703
 704 #ifndef atomic64_sub_return_release
 705 #define  atomic64_sub_return_release(...)                               \
 706         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 707 #endif
 708
 709 #ifndef atomic64_sub_return
 710 #define  atomic64_sub_return(...)                                       \
 711         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 712 #endif
 713 #endif /* atomic64_sub_return_relaxed */
 714
 715 /* atomic64_dec_return_relaxed */
 716 #ifndef atomic64_dec_return_relaxed
 717 #define  atomic64_dec_return_relaxed    atomic64_dec_return
 718 #define  atomic64_dec_return_acquire    atomic64_dec_return
 719 #define  atomic64_dec_return_release    atomic64_dec_return
 720
 721 #else /* atomic64_dec_return_relaxed */
 722
 723 #ifndef atomic64_dec_return_acquire
 724 #define  atomic64_dec_return_acquire(...)                               \
 725         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 726 #endif
 727
 728 #ifndef atomic64_dec_return_release
 729 #define  atomic64_dec_return_release(...)                               \
 730         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 731 #endif
 732
 733 #ifndef atomic64_dec_return
 734 #define  atomic64_dec_return(...)                                       \
 735         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 736 #endif
 737 #endif /* atomic64_dec_return_relaxed */
 738
 739
 740 /* atomic64_fetch_add_relaxed */
 741 #ifndef atomic64_fetch_add_relaxed
 742 #define atomic64_fetch_add_relaxed      atomic64_fetch_add
 743 #define atomic64_fetch_add_acquire      atomic64_fetch_add
 744 #define atomic64_fetch_add_release      atomic64_fetch_add
 745
 746 #else /* atomic64_fetch_add_relaxed */
 747
 748 #ifndef atomic64_fetch_add_acquire
 749 #define atomic64_fetch_add_acquire(...)                                 \
 750         __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
 751 #endif
 752
 753 #ifndef atomic64_fetch_add_release
 754 #define atomic64_fetch_add_release(...)                                 \
 755         __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
 756 #endif
 757
 758 #ifndef atomic64_fetch_add
 759 #define atomic64_fetch_add(...)                                         \
 760         __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
 761 #endif
 762 #endif /* atomic64_fetch_add_relaxed */
 763
 764 /* atomic64_fetch_inc_relaxed */
 765 #ifndef atomic64_fetch_inc_relaxed
 766
 767 #ifndef atomic64_fetch_inc
 768 #define atomic64_fetch_inc(v)           atomic64_fetch_add(1, (v))
 769 #define atomic64_fetch_inc_relaxed(v)   atomic64_fetch_add_relaxed(1, (v))
 770 #define atomic64_fetch_inc_acquire(v)   atomic64_fetch_add_acquire(1, (v))
 771 #define atomic64_fetch_inc_release(v)   atomic64_fetch_add_release(1, (v))
 772 #else /* atomic64_fetch_inc */
 773 #define atomic64_fetch_inc_relaxed      atomic64_fetch_inc
 774 #define atomic64_fetch_inc_acquire      atomic64_fetch_inc
 775 #define atomic64_fetch_inc_release      atomic64_fetch_inc
 776 #endif /* atomic64_fetch_inc */
 777
 778 #else /* atomic64_fetch_inc_relaxed */
 779
 780 #ifndef atomic64_fetch_inc_acquire
 781 #define atomic64_fetch_inc_acquire(...)                                 \
 782         __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
 783 #endif
 784
 785 #ifndef atomic64_fetch_inc_release
 786 #define atomic64_fetch_inc_release(...)                                 \
 787         __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
 788 #endif
 789
 790 #ifndef atomic64_fetch_inc
 791 #define atomic64_fetch_inc(...)                                         \
 792         __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
 793 #endif
 794 #endif /* atomic64_fetch_inc_relaxed */
 795
 796 /* atomic64_fetch_sub_relaxed */
 797 #ifndef atomic64_fetch_sub_relaxed
 798 #define atomic64_fetch_sub_relaxed      atomic64_fetch_sub
 799 #define atomic64_fetch_sub_acquire      atomic64_fetch_sub
 800 #define atomic64_fetch_sub_release      atomic64_fetch_sub
 801
 802 #else /* atomic64_fetch_sub_relaxed */
 803
 804 #ifndef atomic64_fetch_sub_acquire
 805 #define atomic64_fetch_sub_acquire(...)                                 \
 806         __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
 807 #endif
 808
 809 #ifndef atomic64_fetch_sub_release
 810 #define atomic64_fetch_sub_release(...)                                 \
 811         __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
 812 #endif
 813
 814 #ifndef atomic64_fetch_sub
 815 #define atomic64_fetch_sub(...)                                         \
 816         __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
 817 #endif
 818 #endif /* atomic64_fetch_sub_relaxed */
 819
 820 /* atomic64_fetch_dec_relaxed */
 821 #ifndef atomic64_fetch_dec_relaxed
 822
 823 #ifndef atomic64_fetch_dec
 824 #define atomic64_fetch_dec(v)           atomic64_fetch_sub(1, (v))
 825 #define atomic64_fetch_dec_relaxed(v)   atomic64_fetch_sub_relaxed(1, (v))
 826 #define atomic64_fetch_dec_acquire(v)   atomic64_fetch_sub_acquire(1, (v))
 827 #define atomic64_fetch_dec_release(v)   atomic64_fetch_sub_release(1, (v))
 828 #else /* atomic64_fetch_dec */
 829 #define atomic64_fetch_dec_relaxed      atomic64_fetch_dec
 830 #define atomic64_fetch_dec_acquire      atomic64_fetch_dec
 831 #define atomic64_fetch_dec_release      atomic64_fetch_dec
 832 #endif /* atomic64_fetch_dec */
 833
 834 #else /* atomic64_fetch_dec_relaxed */
 835
 836 #ifndef atomic64_fetch_dec_acquire
 837 #define atomic64_fetch_dec_acquire(...)                                 \
 838         __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
 839 #endif
 840
 841 #ifndef atomic64_fetch_dec_release
 842 #define atomic64_fetch_dec_release(...)                                 \
 843         __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
 844 #endif
 845
 846 #ifndef atomic64_fetch_dec
 847 #define atomic64_fetch_dec(...)                                         \
 848         __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
 849 #endif
 850 #endif /* atomic64_fetch_dec_relaxed */
 851
 852 /* atomic64_fetch_or_relaxed */
 853 #ifndef atomic64_fetch_or_relaxed
 854 #define atomic64_fetch_or_relaxed       atomic64_fetch_or
 855 #define atomic64_fetch_or_acquire       atomic64_fetch_or
 856 #define atomic64_fetch_or_release       atomic64_fetch_or
 857
 858 #else /* atomic64_fetch_or_relaxed */
 859
 860 #ifndef atomic64_fetch_or_acquire
 861 #define atomic64_fetch_or_acquire(...)                                  \
 862         __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
 863 #endif
 864
 865 #ifndef atomic64_fetch_or_release
 866 #define atomic64_fetch_or_release(...)                                  \
 867         __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
 868 #endif
 869
 870 #ifndef atomic64_fetch_or
 871 #define atomic64_fetch_or(...)                                          \
 872         __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
 873 #endif
 874 #endif /* atomic64_fetch_or_relaxed */
 875
 876 /* atomic64_fetch_and_relaxed */
 877 #ifndef atomic64_fetch_and_relaxed
 878 #define atomic64_fetch_and_relaxed      atomic64_fetch_and
 879 #define atomic64_fetch_and_acquire      atomic64_fetch_and
 880 #define atomic64_fetch_and_release      atomic64_fetch_and
 881
 882 #else /* atomic64_fetch_and_relaxed */
 883
 884 #ifndef atomic64_fetch_and_acquire
 885 #define atomic64_fetch_and_acquire(...)                                 \
 886         __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
 887 #endif
 888
 889 #ifndef atomic64_fetch_and_release
 890 #define atomic64_fetch_and_release(...)                                 \
 891         __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
 892 #endif
 893
 894 #ifndef atomic64_fetch_and
 895 #define atomic64_fetch_and(...)                                         \
 896         __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
 897 #endif
 898 #endif /* atomic64_fetch_and_relaxed */
 899
 900 #ifdef atomic64_andnot
 901 /* atomic64_fetch_andnot_relaxed */
 902 #ifndef atomic64_fetch_andnot_relaxed
 903 #define atomic64_fetch_andnot_relaxed   atomic64_fetch_andnot
 904 #define atomic64_fetch_andnot_acquire   atomic64_fetch_andnot
 905 #define atomic64_fetch_andnot_release   atomic64_fetch_andnot
 906
 907 #else /* atomic64_fetch_andnot_relaxed */
 908
 909 #ifndef atomic64_fetch_andnot_acquire
 910 #define atomic64_fetch_andnot_acquire(...)                                      \
 911         __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
 912 #endif
 913
 914 #ifndef atomic64_fetch_andnot_release
 915 #define atomic64_fetch_andnot_release(...)                                      \
 916         __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
 917 #endif
 918
 919 #ifndef atomic64_fetch_andnot
 920 #define atomic64_fetch_andnot(...)                                              \
 921         __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
 922 #endif
 923 #endif /* atomic64_fetch_andnot_relaxed */
 924 #endif /* atomic64_andnot */
 925
 926 /* atomic64_fetch_xor_relaxed */
 927 #ifndef atomic64_fetch_xor_relaxed
 928 #define atomic64_fetch_xor_relaxed      atomic64_fetch_xor
 929 #define atomic64_fetch_xor_acquire      atomic64_fetch_xor
 930 #define atomic64_fetch_xor_release      atomic64_fetch_xor
 931
 932 #else /* atomic64_fetch_xor_relaxed */
 933
 934 #ifndef atomic64_fetch_xor_acquire
 935 #define atomic64_fetch_xor_acquire(...)                                 \
 936         __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
 937 #endif
 938
 939 #ifndef atomic64_fetch_xor_release
 940 #define atomic64_fetch_xor_release(...)                                 \
 941         __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
 942 #endif
 943
 944 #ifndef atomic64_fetch_xor
 945 #define atomic64_fetch_xor(...)                                         \
 946         __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
 947 #endif
 948 #endif /* atomic64_fetch_xor_relaxed */
 949
 950
 951 /* atomic64_xchg_relaxed */
 952 #ifndef atomic64_xchg_relaxed
 953 #define  atomic64_xchg_relaxed          atomic64_xchg
 954 #define  atomic64_xchg_acquire          atomic64_xchg
 955 #define  atomic64_xchg_release          atomic64_xchg
 956
 957 #else /* atomic64_xchg_relaxed */
 958
 959 #ifndef atomic64_xchg_acquire
 960 #define  atomic64_xchg_acquire(...)                                     \
 961         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 962 #endif
 963
 964 #ifndef atomic64_xchg_release
 965 #define  atomic64_xchg_release(...)                                     \
 966         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 967 #endif
 968
 969 #ifndef atomic64_xchg
 970 #define  atomic64_xchg(...)                                             \
 971         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
 972 #endif
 973 #endif /* atomic64_xchg_relaxed */
 974
 975 /* atomic64_cmpxchg_relaxed */
 976 #ifndef atomic64_cmpxchg_relaxed
 977 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
 978 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
 979 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
 980
 981 #else /* atomic64_cmpxchg_relaxed */
 982
 983 #ifndef atomic64_cmpxchg_acquire
 984 #define  atomic64_cmpxchg_acquire(...)                                  \
 985         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
 986 #endif
 987
 988 #ifndef atomic64_cmpxchg_release
 989 #define  atomic64_cmpxchg_release(...)                                  \
 990         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
 991 #endif
 992
 993 #ifndef atomic64_cmpxchg
 994 #define  atomic64_cmpxchg(...)                                          \
 995         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
 996 #endif
 997 #endif /* atomic64_cmpxchg_relaxed */
 998
 999 #ifndef atomic64_andnot
1000 static inline void atomic64_andnot(long long i, atomic64_t *v)
1001 {
1002         atomic64_and(~i, v);
1003 }
1004
1005 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1006 {
1007         return atomic64_fetch_and(~i, v);
1008 }
1009
1010 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1011 {
1012         return atomic64_fetch_and_relaxed(~i, v);
1013 }
1014
1015 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1016 {
1017         return atomic64_fetch_and_acquire(~i, v);
1018 }
1019
1020 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1021 {
1022         return atomic64_fetch_and_release(~i, v);
1023 }
1024 #endif
1025
1026 #include <asm-generic/atomic-long.h>
1027
1028 #endif /* _LINUX_ATOMIC_H */