include/linux/atomic-fallback.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-fallback.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 #ifndef _LINUX_ATOMIC_FALLBACK_H
   7 #define _LINUX_ATOMIC_FALLBACK_H
   8
   9 #ifndef xchg_relaxed
  10 #define xchg_relaxed            xchg
  11 #define xchg_acquire            xchg
  12 #define xchg_release            xchg
  13 #else /* xchg_relaxed */
  14
  15 #ifndef xchg_acquire
  16 #define xchg_acquire(...) \
  17         __atomic_op_acquire(xchg, __VA_ARGS__)
  18 #endif
  19
  20 #ifndef xchg_release
  21 #define xchg_release(...) \
  22         __atomic_op_release(xchg, __VA_ARGS__)
  23 #endif
  24
  25 #ifndef xchg
  26 #define xchg(...) \
  27         __atomic_op_fence(xchg, __VA_ARGS__)
  28 #endif
  29
  30 #endif /* xchg_relaxed */
  31
  32 #ifndef cmpxchg_relaxed
  33 #define cmpxchg_relaxed         cmpxchg
  34 #define cmpxchg_acquire         cmpxchg
  35 #define cmpxchg_release         cmpxchg
  36 #else /* cmpxchg_relaxed */
  37
  38 #ifndef cmpxchg_acquire
  39 #define cmpxchg_acquire(...) \
  40         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
  41 #endif
  42
  43 #ifndef cmpxchg_release
  44 #define cmpxchg_release(...) \
  45         __atomic_op_release(cmpxchg, __VA_ARGS__)
  46 #endif
  47
  48 #ifndef cmpxchg
  49 #define cmpxchg(...) \
  50         __atomic_op_fence(cmpxchg, __VA_ARGS__)
  51 #endif
  52
  53 #endif /* cmpxchg_relaxed */
  54
  55 #ifndef cmpxchg64_relaxed
  56 #define cmpxchg64_relaxed               cmpxchg64
  57 #define cmpxchg64_acquire               cmpxchg64
  58 #define cmpxchg64_release               cmpxchg64
  59 #else /* cmpxchg64_relaxed */
  60
  61 #ifndef cmpxchg64_acquire
  62 #define cmpxchg64_acquire(...) \
  63         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
  64 #endif
  65
  66 #ifndef cmpxchg64_release
  67 #define cmpxchg64_release(...) \
  68         __atomic_op_release(cmpxchg64, __VA_ARGS__)
  69 #endif
  70
  71 #ifndef cmpxchg64
  72 #define cmpxchg64(...) \
  73         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
  74 #endif
  75
  76 #endif /* cmpxchg64_relaxed */
  77
  78 #ifndef atomic_read_acquire
  79 static inline int
  80 atomic_read_acquire(const atomic_t *v)
  81 {
  82         return smp_load_acquire(&(v)->counter);
  83 }
  84 #define atomic_read_acquire atomic_read_acquire
  85 #endif
  86
  87 #ifndef atomic_set_release
  88 static inline void
  89 atomic_set_release(atomic_t *v, int i)
  90 {
  91         smp_store_release(&(v)->counter, i);
  92 }
  93 #define atomic_set_release atomic_set_release
  94 #endif
  95
  96 #ifndef atomic_add_return_relaxed
  97 #define atomic_add_return_acquire atomic_add_return
  98 #define atomic_add_return_release atomic_add_return
  99 #define atomic_add_return_relaxed atomic_add_return
 100 #else /* atomic_add_return_relaxed */
 101
 102 #ifndef atomic_add_return_acquire
 103 static inline int
 104 atomic_add_return_acquire(int i, atomic_t *v)
 105 {
 106         int ret = atomic_add_return_relaxed(i, v);
 107         __atomic_acquire_fence();
 108         return ret;
 109 }
 110 #define atomic_add_return_acquire atomic_add_return_acquire
 111 #endif
 112
 113 #ifndef atomic_add_return_release
 114 static inline int
 115 atomic_add_return_release(int i, atomic_t *v)
 116 {
 117         __atomic_release_fence();
 118         return atomic_add_return_relaxed(i, v);
 119 }
 120 #define atomic_add_return_release atomic_add_return_release
 121 #endif
 122
 123 #ifndef atomic_add_return
 124 static inline int
 125 atomic_add_return(int i, atomic_t *v)
 126 {
 127         int ret;
 128         __atomic_pre_full_fence();
 129         ret = atomic_add_return_relaxed(i, v);
 130         __atomic_post_full_fence();
 131         return ret;
 132 }
 133 #define atomic_add_return atomic_add_return
 134 #endif
 135
 136 #endif /* atomic_add_return_relaxed */
 137
 138 #ifndef atomic_fetch_add_relaxed
 139 #define atomic_fetch_add_acquire atomic_fetch_add
 140 #define atomic_fetch_add_release atomic_fetch_add
 141 #define atomic_fetch_add_relaxed atomic_fetch_add
 142 #else /* atomic_fetch_add_relaxed */
 143
 144 #ifndef atomic_fetch_add_acquire
 145 static inline int
 146 atomic_fetch_add_acquire(int i, atomic_t *v)
 147 {
 148         int ret = atomic_fetch_add_relaxed(i, v);
 149         __atomic_acquire_fence();
 150         return ret;
 151 }
 152 #define atomic_fetch_add_acquire atomic_fetch_add_acquire
 153 #endif
 154
 155 #ifndef atomic_fetch_add_release
 156 static inline int
 157 atomic_fetch_add_release(int i, atomic_t *v)
 158 {
 159         __atomic_release_fence();
 160         return atomic_fetch_add_relaxed(i, v);
 161 }
 162 #define atomic_fetch_add_release atomic_fetch_add_release
 163 #endif
 164
 165 #ifndef atomic_fetch_add
 166 static inline int
 167 atomic_fetch_add(int i, atomic_t *v)
 168 {
 169         int ret;
 170         __atomic_pre_full_fence();
 171         ret = atomic_fetch_add_relaxed(i, v);
 172         __atomic_post_full_fence();
 173         return ret;
 174 }
 175 #define atomic_fetch_add atomic_fetch_add
 176 #endif
 177
 178 #endif /* atomic_fetch_add_relaxed */
 179
 180 #ifndef atomic_sub_return_relaxed
 181 #define atomic_sub_return_acquire atomic_sub_return
 182 #define atomic_sub_return_release atomic_sub_return
 183 #define atomic_sub_return_relaxed atomic_sub_return
 184 #else /* atomic_sub_return_relaxed */
 185
 186 #ifndef atomic_sub_return_acquire
 187 static inline int
 188 atomic_sub_return_acquire(int i, atomic_t *v)
 189 {
 190         int ret = atomic_sub_return_relaxed(i, v);
 191         __atomic_acquire_fence();
 192         return ret;
 193 }
 194 #define atomic_sub_return_acquire atomic_sub_return_acquire
 195 #endif
 196
 197 #ifndef atomic_sub_return_release
 198 static inline int
 199 atomic_sub_return_release(int i, atomic_t *v)
 200 {
 201         __atomic_release_fence();
 202         return atomic_sub_return_relaxed(i, v);
 203 }
 204 #define atomic_sub_return_release atomic_sub_return_release
 205 #endif
 206
 207 #ifndef atomic_sub_return
 208 static inline int
 209 atomic_sub_return(int i, atomic_t *v)
 210 {
 211         int ret;
 212         __atomic_pre_full_fence();
 213         ret = atomic_sub_return_relaxed(i, v);
 214         __atomic_post_full_fence();
 215         return ret;
 216 }
 217 #define atomic_sub_return atomic_sub_return
 218 #endif
 219
 220 #endif /* atomic_sub_return_relaxed */
 221
 222 #ifndef atomic_fetch_sub_relaxed
 223 #define atomic_fetch_sub_acquire atomic_fetch_sub
 224 #define atomic_fetch_sub_release atomic_fetch_sub
 225 #define atomic_fetch_sub_relaxed atomic_fetch_sub
 226 #else /* atomic_fetch_sub_relaxed */
 227
 228 #ifndef atomic_fetch_sub_acquire
 229 static inline int
 230 atomic_fetch_sub_acquire(int i, atomic_t *v)
 231 {
 232         int ret = atomic_fetch_sub_relaxed(i, v);
 233         __atomic_acquire_fence();
 234         return ret;
 235 }
 236 #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
 237 #endif
 238
 239 #ifndef atomic_fetch_sub_release
 240 static inline int
 241 atomic_fetch_sub_release(int i, atomic_t *v)
 242 {
 243         __atomic_release_fence();
 244         return atomic_fetch_sub_relaxed(i, v);
 245 }
 246 #define atomic_fetch_sub_release atomic_fetch_sub_release
 247 #endif
 248
 249 #ifndef atomic_fetch_sub
 250 static inline int
 251 atomic_fetch_sub(int i, atomic_t *v)
 252 {
 253         int ret;
 254         __atomic_pre_full_fence();
 255         ret = atomic_fetch_sub_relaxed(i, v);
 256         __atomic_post_full_fence();
 257         return ret;
 258 }
 259 #define atomic_fetch_sub atomic_fetch_sub
 260 #endif
 261
 262 #endif /* atomic_fetch_sub_relaxed */
 263
 264 #ifndef atomic_inc
 265 static inline void
 266 atomic_inc(atomic_t *v)
 267 {
 268         atomic_add(1, v);
 269 }
 270 #define atomic_inc atomic_inc
 271 #endif
 272
 273 #ifndef atomic_inc_return_relaxed
 274 #ifdef atomic_inc_return
 275 #define atomic_inc_return_acquire atomic_inc_return
 276 #define atomic_inc_return_release atomic_inc_return
 277 #define atomic_inc_return_relaxed atomic_inc_return
 278 #endif /* atomic_inc_return */
 279
 280 #ifndef atomic_inc_return
 281 static inline int
 282 atomic_inc_return(atomic_t *v)
 283 {
 284         return atomic_add_return(1, v);
 285 }
 286 #define atomic_inc_return atomic_inc_return
 287 #endif
 288
 289 #ifndef atomic_inc_return_acquire
 290 static inline int
 291 atomic_inc_return_acquire(atomic_t *v)
 292 {
 293         return atomic_add_return_acquire(1, v);
 294 }
 295 #define atomic_inc_return_acquire atomic_inc_return_acquire
 296 #endif
 297
 298 #ifndef atomic_inc_return_release
 299 static inline int
 300 atomic_inc_return_release(atomic_t *v)
 301 {
 302         return atomic_add_return_release(1, v);
 303 }
 304 #define atomic_inc_return_release atomic_inc_return_release
 305 #endif
 306
 307 #ifndef atomic_inc_return_relaxed
 308 static inline int
 309 atomic_inc_return_relaxed(atomic_t *v)
 310 {
 311         return atomic_add_return_relaxed(1, v);
 312 }
 313 #define atomic_inc_return_relaxed atomic_inc_return_relaxed
 314 #endif
 315
 316 #else /* atomic_inc_return_relaxed */
 317
 318 #ifndef atomic_inc_return_acquire
 319 static inline int
 320 atomic_inc_return_acquire(atomic_t *v)
 321 {
 322         int ret = atomic_inc_return_relaxed(v);
 323         __atomic_acquire_fence();
 324         return ret;
 325 }
 326 #define atomic_inc_return_acquire atomic_inc_return_acquire
 327 #endif
 328
 329 #ifndef atomic_inc_return_release
 330 static inline int
 331 atomic_inc_return_release(atomic_t *v)
 332 {
 333         __atomic_release_fence();
 334         return atomic_inc_return_relaxed(v);
 335 }
 336 #define atomic_inc_return_release atomic_inc_return_release
 337 #endif
 338
 339 #ifndef atomic_inc_return
 340 static inline int
 341 atomic_inc_return(atomic_t *v)
 342 {
 343         int ret;
 344         __atomic_pre_full_fence();
 345         ret = atomic_inc_return_relaxed(v);
 346         __atomic_post_full_fence();
 347         return ret;
 348 }
 349 #define atomic_inc_return atomic_inc_return
 350 #endif
 351
 352 #endif /* atomic_inc_return_relaxed */
 353
 354 #ifndef atomic_fetch_inc_relaxed
 355 #ifdef atomic_fetch_inc
 356 #define atomic_fetch_inc_acquire atomic_fetch_inc
 357 #define atomic_fetch_inc_release atomic_fetch_inc
 358 #define atomic_fetch_inc_relaxed atomic_fetch_inc
 359 #endif /* atomic_fetch_inc */
 360
 361 #ifndef atomic_fetch_inc
 362 static inline int
 363 atomic_fetch_inc(atomic_t *v)
 364 {
 365         return atomic_fetch_add(1, v);
 366 }
 367 #define atomic_fetch_inc atomic_fetch_inc
 368 #endif
 369
 370 #ifndef atomic_fetch_inc_acquire
 371 static inline int
 372 atomic_fetch_inc_acquire(atomic_t *v)
 373 {
 374         return atomic_fetch_add_acquire(1, v);
 375 }
 376 #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
 377 #endif
 378
 379 #ifndef atomic_fetch_inc_release
 380 static inline int
 381 atomic_fetch_inc_release(atomic_t *v)
 382 {
 383         return atomic_fetch_add_release(1, v);
 384 }
 385 #define atomic_fetch_inc_release atomic_fetch_inc_release
 386 #endif
 387
 388 #ifndef atomic_fetch_inc_relaxed
 389 static inline int
 390 atomic_fetch_inc_relaxed(atomic_t *v)
 391 {
 392         return atomic_fetch_add_relaxed(1, v);
 393 }
 394 #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
 395 #endif
 396
 397 #else /* atomic_fetch_inc_relaxed */
 398
 399 #ifndef atomic_fetch_inc_acquire
 400 static inline int
 401 atomic_fetch_inc_acquire(atomic_t *v)
 402 {
 403         int ret = atomic_fetch_inc_relaxed(v);
 404         __atomic_acquire_fence();
 405         return ret;
 406 }
 407 #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
 408 #endif
 409
 410 #ifndef atomic_fetch_inc_release
 411 static inline int
 412 atomic_fetch_inc_release(atomic_t *v)
 413 {
 414         __atomic_release_fence();
 415         return atomic_fetch_inc_relaxed(v);
 416 }
 417 #define atomic_fetch_inc_release atomic_fetch_inc_release
 418 #endif
 419
 420 #ifndef atomic_fetch_inc
 421 static inline int
 422 atomic_fetch_inc(atomic_t *v)
 423 {
 424         int ret;
 425         __atomic_pre_full_fence();
 426         ret = atomic_fetch_inc_relaxed(v);
 427         __atomic_post_full_fence();
 428         return ret;
 429 }
 430 #define atomic_fetch_inc atomic_fetch_inc
 431 #endif
 432
 433 #endif /* atomic_fetch_inc_relaxed */
 434
 435 #ifndef atomic_dec
 436 static inline void
 437 atomic_dec(atomic_t *v)
 438 {
 439         atomic_sub(1, v);
 440 }
 441 #define atomic_dec atomic_dec
 442 #endif
 443
 444 #ifndef atomic_dec_return_relaxed
 445 #ifdef atomic_dec_return
 446 #define atomic_dec_return_acquire atomic_dec_return
 447 #define atomic_dec_return_release atomic_dec_return
 448 #define atomic_dec_return_relaxed atomic_dec_return
 449 #endif /* atomic_dec_return */
 450
 451 #ifndef atomic_dec_return
 452 static inline int
 453 atomic_dec_return(atomic_t *v)
 454 {
 455         return atomic_sub_return(1, v);
 456 }
 457 #define atomic_dec_return atomic_dec_return
 458 #endif
 459
 460 #ifndef atomic_dec_return_acquire
 461 static inline int
 462 atomic_dec_return_acquire(atomic_t *v)
 463 {
 464         return atomic_sub_return_acquire(1, v);
 465 }
 466 #define atomic_dec_return_acquire atomic_dec_return_acquire
 467 #endif
 468
 469 #ifndef atomic_dec_return_release
 470 static inline int
 471 atomic_dec_return_release(atomic_t *v)
 472 {
 473         return atomic_sub_return_release(1, v);
 474 }
 475 #define atomic_dec_return_release atomic_dec_return_release
 476 #endif
 477
 478 #ifndef atomic_dec_return_relaxed
 479 static inline int
 480 atomic_dec_return_relaxed(atomic_t *v)
 481 {
 482         return atomic_sub_return_relaxed(1, v);
 483 }
 484 #define atomic_dec_return_relaxed atomic_dec_return_relaxed
 485 #endif
 486
 487 #else /* atomic_dec_return_relaxed */
 488
 489 #ifndef atomic_dec_return_acquire
 490 static inline int
 491 atomic_dec_return_acquire(atomic_t *v)
 492 {
 493         int ret = atomic_dec_return_relaxed(v);
 494         __atomic_acquire_fence();
 495         return ret;
 496 }
 497 #define atomic_dec_return_acquire atomic_dec_return_acquire
 498 #endif
 499
 500 #ifndef atomic_dec_return_release
 501 static inline int
 502 atomic_dec_return_release(atomic_t *v)
 503 {
 504         __atomic_release_fence();
 505         return atomic_dec_return_relaxed(v);
 506 }
 507 #define atomic_dec_return_release atomic_dec_return_release
 508 #endif
 509
 510 #ifndef atomic_dec_return
 511 static inline int
 512 atomic_dec_return(atomic_t *v)
 513 {
 514         int ret;
 515         __atomic_pre_full_fence();
 516         ret = atomic_dec_return_relaxed(v);
 517         __atomic_post_full_fence();
 518         return ret;
 519 }
 520 #define atomic_dec_return atomic_dec_return
 521 #endif
 522
 523 #endif /* atomic_dec_return_relaxed */
 524
 525 #ifndef atomic_fetch_dec_relaxed
 526 #ifdef atomic_fetch_dec
 527 #define atomic_fetch_dec_acquire atomic_fetch_dec
 528 #define atomic_fetch_dec_release atomic_fetch_dec
 529 #define atomic_fetch_dec_relaxed atomic_fetch_dec
 530 #endif /* atomic_fetch_dec */
 531
 532 #ifndef atomic_fetch_dec
 533 static inline int
 534 atomic_fetch_dec(atomic_t *v)
 535 {
 536         return atomic_fetch_sub(1, v);
 537 }
 538 #define atomic_fetch_dec atomic_fetch_dec
 539 #endif
 540
 541 #ifndef atomic_fetch_dec_acquire
 542 static inline int
 543 atomic_fetch_dec_acquire(atomic_t *v)
 544 {
 545         return atomic_fetch_sub_acquire(1, v);
 546 }
 547 #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
 548 #endif
 549
 550 #ifndef atomic_fetch_dec_release
 551 static inline int
 552 atomic_fetch_dec_release(atomic_t *v)
 553 {
 554         return atomic_fetch_sub_release(1, v);
 555 }
 556 #define atomic_fetch_dec_release atomic_fetch_dec_release
 557 #endif
 558
 559 #ifndef atomic_fetch_dec_relaxed
 560 static inline int
 561 atomic_fetch_dec_relaxed(atomic_t *v)
 562 {
 563         return atomic_fetch_sub_relaxed(1, v);
 564 }
 565 #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
 566 #endif
 567
 568 #else /* atomic_fetch_dec_relaxed */
 569
 570 #ifndef atomic_fetch_dec_acquire
 571 static inline int
 572 atomic_fetch_dec_acquire(atomic_t *v)
 573 {
 574         int ret = atomic_fetch_dec_relaxed(v);
 575         __atomic_acquire_fence();
 576         return ret;
 577 }
 578 #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
 579 #endif
 580
 581 #ifndef atomic_fetch_dec_release
 582 static inline int
 583 atomic_fetch_dec_release(atomic_t *v)
 584 {
 585         __atomic_release_fence();
 586         return atomic_fetch_dec_relaxed(v);
 587 }
 588 #define atomic_fetch_dec_release atomic_fetch_dec_release
 589 #endif
 590
 591 #ifndef atomic_fetch_dec
 592 static inline int
 593 atomic_fetch_dec(atomic_t *v)
 594 {
 595         int ret;
 596         __atomic_pre_full_fence();
 597         ret = atomic_fetch_dec_relaxed(v);
 598         __atomic_post_full_fence();
 599         return ret;
 600 }
 601 #define atomic_fetch_dec atomic_fetch_dec
 602 #endif
 603
 604 #endif /* atomic_fetch_dec_relaxed */
 605
 606 #ifndef atomic_fetch_and_relaxed
 607 #define atomic_fetch_and_acquire atomic_fetch_and
 608 #define atomic_fetch_and_release atomic_fetch_and
 609 #define atomic_fetch_and_relaxed atomic_fetch_and
 610 #else /* atomic_fetch_and_relaxed */
 611
 612 #ifndef atomic_fetch_and_acquire
 613 static inline int
 614 atomic_fetch_and_acquire(int i, atomic_t *v)
 615 {
 616         int ret = atomic_fetch_and_relaxed(i, v);
 617         __atomic_acquire_fence();
 618         return ret;
 619 }
 620 #define atomic_fetch_and_acquire atomic_fetch_and_acquire
 621 #endif
 622
 623 #ifndef atomic_fetch_and_release
 624 static inline int
 625 atomic_fetch_and_release(int i, atomic_t *v)
 626 {
 627         __atomic_release_fence();
 628         return atomic_fetch_and_relaxed(i, v);
 629 }
 630 #define atomic_fetch_and_release atomic_fetch_and_release
 631 #endif
 632
 633 #ifndef atomic_fetch_and
 634 static inline int
 635 atomic_fetch_and(int i, atomic_t *v)
 636 {
 637         int ret;
 638         __atomic_pre_full_fence();
 639         ret = atomic_fetch_and_relaxed(i, v);
 640         __atomic_post_full_fence();
 641         return ret;
 642 }
 643 #define atomic_fetch_and atomic_fetch_and
 644 #endif
 645
 646 #endif /* atomic_fetch_and_relaxed */
 647
 648 #ifndef atomic_andnot
 649 static inline void
 650 atomic_andnot(int i, atomic_t *v)
 651 {
 652         atomic_and(~i, v);
 653 }
 654 #define atomic_andnot atomic_andnot
 655 #endif
 656
 657 #ifndef atomic_fetch_andnot_relaxed
 658 #ifdef atomic_fetch_andnot
 659 #define atomic_fetch_andnot_acquire atomic_fetch_andnot
 660 #define atomic_fetch_andnot_release atomic_fetch_andnot
 661 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot
 662 #endif /* atomic_fetch_andnot */
 663
 664 #ifndef atomic_fetch_andnot
 665 static inline int
 666 atomic_fetch_andnot(int i, atomic_t *v)
 667 {
 668         return atomic_fetch_and(~i, v);
 669 }
 670 #define atomic_fetch_andnot atomic_fetch_andnot
 671 #endif
 672
 673 #ifndef atomic_fetch_andnot_acquire
 674 static inline int
 675 atomic_fetch_andnot_acquire(int i, atomic_t *v)
 676 {
 677         return atomic_fetch_and_acquire(~i, v);
 678 }
 679 #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
 680 #endif
 681
 682 #ifndef atomic_fetch_andnot_release
 683 static inline int
 684 atomic_fetch_andnot_release(int i, atomic_t *v)
 685 {
 686         return atomic_fetch_and_release(~i, v);
 687 }
 688 #define atomic_fetch_andnot_release atomic_fetch_andnot_release
 689 #endif
 690
 691 #ifndef atomic_fetch_andnot_relaxed
 692 static inline int
 693 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 694 {
 695         return atomic_fetch_and_relaxed(~i, v);
 696 }
 697 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
 698 #endif
 699
 700 #else /* atomic_fetch_andnot_relaxed */
 701
 702 #ifndef atomic_fetch_andnot_acquire
 703 static inline int
 704 atomic_fetch_andnot_acquire(int i, atomic_t *v)
 705 {
 706         int ret = atomic_fetch_andnot_relaxed(i, v);
 707         __atomic_acquire_fence();
 708         return ret;
 709 }
 710 #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
 711 #endif
 712
 713 #ifndef atomic_fetch_andnot_release
 714 static inline int
 715 atomic_fetch_andnot_release(int i, atomic_t *v)
 716 {
 717         __atomic_release_fence();
 718         return atomic_fetch_andnot_relaxed(i, v);
 719 }
 720 #define atomic_fetch_andnot_release atomic_fetch_andnot_release
 721 #endif
 722
 723 #ifndef atomic_fetch_andnot
 724 static inline int
 725 atomic_fetch_andnot(int i, atomic_t *v)
 726 {
 727         int ret;
 728         __atomic_pre_full_fence();
 729         ret = atomic_fetch_andnot_relaxed(i, v);
 730         __atomic_post_full_fence();
 731         return ret;
 732 }
 733 #define atomic_fetch_andnot atomic_fetch_andnot
 734 #endif
 735
 736 #endif /* atomic_fetch_andnot_relaxed */
 737
 738 #ifndef atomic_fetch_or_relaxed
 739 #define atomic_fetch_or_acquire atomic_fetch_or
 740 #define atomic_fetch_or_release atomic_fetch_or
 741 #define atomic_fetch_or_relaxed atomic_fetch_or
 742 #else /* atomic_fetch_or_relaxed */
 743
 744 #ifndef atomic_fetch_or_acquire
 745 static inline int
 746 atomic_fetch_or_acquire(int i, atomic_t *v)
 747 {
 748         int ret = atomic_fetch_or_relaxed(i, v);
 749         __atomic_acquire_fence();
 750         return ret;
 751 }
 752 #define atomic_fetch_or_acquire atomic_fetch_or_acquire
 753 #endif
 754
 755 #ifndef atomic_fetch_or_release
 756 static inline int
 757 atomic_fetch_or_release(int i, atomic_t *v)
 758 {
 759         __atomic_release_fence();
 760         return atomic_fetch_or_relaxed(i, v);
 761 }
 762 #define atomic_fetch_or_release atomic_fetch_or_release
 763 #endif
 764
 765 #ifndef atomic_fetch_or
 766 static inline int
 767 atomic_fetch_or(int i, atomic_t *v)
 768 {
 769         int ret;
 770         __atomic_pre_full_fence();
 771         ret = atomic_fetch_or_relaxed(i, v);
 772         __atomic_post_full_fence();
 773         return ret;
 774 }
 775 #define atomic_fetch_or atomic_fetch_or
 776 #endif
 777
 778 #endif /* atomic_fetch_or_relaxed */
 779
 780 #ifndef atomic_fetch_xor_relaxed
 781 #define atomic_fetch_xor_acquire atomic_fetch_xor
 782 #define atomic_fetch_xor_release atomic_fetch_xor
 783 #define atomic_fetch_xor_relaxed atomic_fetch_xor
 784 #else /* atomic_fetch_xor_relaxed */
 785
 786 #ifndef atomic_fetch_xor_acquire
 787 static inline int
 788 atomic_fetch_xor_acquire(int i, atomic_t *v)
 789 {
 790         int ret = atomic_fetch_xor_relaxed(i, v);
 791         __atomic_acquire_fence();
 792         return ret;
 793 }
 794 #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
 795 #endif
 796
 797 #ifndef atomic_fetch_xor_release
 798 static inline int
 799 atomic_fetch_xor_release(int i, atomic_t *v)
 800 {
 801         __atomic_release_fence();
 802         return atomic_fetch_xor_relaxed(i, v);
 803 }
 804 #define atomic_fetch_xor_release atomic_fetch_xor_release
 805 #endif
 806
 807 #ifndef atomic_fetch_xor
 808 static inline int
 809 atomic_fetch_xor(int i, atomic_t *v)
 810 {
 811         int ret;
 812         __atomic_pre_full_fence();
 813         ret = atomic_fetch_xor_relaxed(i, v);
 814         __atomic_post_full_fence();
 815         return ret;
 816 }
 817 #define atomic_fetch_xor atomic_fetch_xor
 818 #endif
 819
 820 #endif /* atomic_fetch_xor_relaxed */
 821
 822 #ifndef atomic_xchg_relaxed
 823 #define atomic_xchg_acquire atomic_xchg
 824 #define atomic_xchg_release atomic_xchg
 825 #define atomic_xchg_relaxed atomic_xchg
 826 #else /* atomic_xchg_relaxed */
 827
 828 #ifndef atomic_xchg_acquire
 829 static inline int
 830 atomic_xchg_acquire(atomic_t *v, int i)
 831 {
 832         int ret = atomic_xchg_relaxed(v, i);
 833         __atomic_acquire_fence();
 834         return ret;
 835 }
 836 #define atomic_xchg_acquire atomic_xchg_acquire
 837 #endif
 838
 839 #ifndef atomic_xchg_release
 840 static inline int
 841 atomic_xchg_release(atomic_t *v, int i)
 842 {
 843         __atomic_release_fence();
 844         return atomic_xchg_relaxed(v, i);
 845 }
 846 #define atomic_xchg_release atomic_xchg_release
 847 #endif
 848
 849 #ifndef atomic_xchg
 850 static inline int
 851 atomic_xchg(atomic_t *v, int i)
 852 {
 853         int ret;
 854         __atomic_pre_full_fence();
 855         ret = atomic_xchg_relaxed(v, i);
 856         __atomic_post_full_fence();
 857         return ret;
 858 }
 859 #define atomic_xchg atomic_xchg
 860 #endif
 861
 862 #endif /* atomic_xchg_relaxed */
 863
 864 #ifndef atomic_cmpxchg_relaxed
 865 #define atomic_cmpxchg_acquire atomic_cmpxchg
 866 #define atomic_cmpxchg_release atomic_cmpxchg
 867 #define atomic_cmpxchg_relaxed atomic_cmpxchg
 868 #else /* atomic_cmpxchg_relaxed */
 869
 870 #ifndef atomic_cmpxchg_acquire
 871 static inline int
 872 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
 873 {
 874         int ret = atomic_cmpxchg_relaxed(v, old, new);
 875         __atomic_acquire_fence();
 876         return ret;
 877 }
 878 #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
 879 #endif
 880
 881 #ifndef atomic_cmpxchg_release
 882 static inline int
 883 atomic_cmpxchg_release(atomic_t *v, int old, int new)
 884 {
 885         __atomic_release_fence();
 886         return atomic_cmpxchg_relaxed(v, old, new);
 887 }
 888 #define atomic_cmpxchg_release atomic_cmpxchg_release
 889 #endif
 890
 891 #ifndef atomic_cmpxchg
 892 static inline int
 893 atomic_cmpxchg(atomic_t *v, int old, int new)
 894 {
 895         int ret;
 896         __atomic_pre_full_fence();
 897         ret = atomic_cmpxchg_relaxed(v, old, new);
 898         __atomic_post_full_fence();
 899         return ret;
 900 }
 901 #define atomic_cmpxchg atomic_cmpxchg
 902 #endif
 903
 904 #endif /* atomic_cmpxchg_relaxed */
 905
 906 #ifndef atomic_try_cmpxchg_relaxed
 907 #ifdef atomic_try_cmpxchg
 908 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
 909 #define atomic_try_cmpxchg_release atomic_try_cmpxchg
 910 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
 911 #endif /* atomic_try_cmpxchg */
 912
 913 #ifndef atomic_try_cmpxchg
 914 static inline bool
 915 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 916 {
 917         int r, o = *old;
 918         r = atomic_cmpxchg(v, o, new);
 919         if (unlikely(r != o))
 920                 *old = r;
 921         return likely(r == o);
 922 }
 923 #define atomic_try_cmpxchg atomic_try_cmpxchg
 924 #endif
 925
 926 #ifndef atomic_try_cmpxchg_acquire
 927 static inline bool
 928 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 929 {
 930         int r, o = *old;
 931         r = atomic_cmpxchg_acquire(v, o, new);
 932         if (unlikely(r != o))
 933                 *old = r;
 934         return likely(r == o);
 935 }
 936 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
 937 #endif
 938
 939 #ifndef atomic_try_cmpxchg_release
 940 static inline bool
 941 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 942 {
 943         int r, o = *old;
 944         r = atomic_cmpxchg_release(v, o, new);
 945         if (unlikely(r != o))
 946                 *old = r;
 947         return likely(r == o);
 948 }
 949 #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
 950 #endif
 951
 952 #ifndef atomic_try_cmpxchg_relaxed
 953 static inline bool
 954 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
 955 {
 956         int r, o = *old;
 957         r = atomic_cmpxchg_relaxed(v, o, new);
 958         if (unlikely(r != o))
 959                 *old = r;
 960         return likely(r == o);
 961 }
 962 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
 963 #endif
 964
 965 #else /* atomic_try_cmpxchg_relaxed */
 966
 967 #ifndef atomic_try_cmpxchg_acquire
 968 static inline bool
 969 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 970 {
 971         bool ret = atomic_try_cmpxchg_relaxed(v, old, new);
 972         __atomic_acquire_fence();
 973         return ret;
 974 }
 975 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
 976 #endif
 977
 978 #ifndef atomic_try_cmpxchg_release
 979 static inline bool
 980 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 981 {
 982         __atomic_release_fence();
 983         return atomic_try_cmpxchg_relaxed(v, old, new);
 984 }
 985 #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
 986 #endif
 987
 988 #ifndef atomic_try_cmpxchg
 989 static inline bool
 990 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 991 {
 992         bool ret;
 993         __atomic_pre_full_fence();
 994         ret = atomic_try_cmpxchg_relaxed(v, old, new);
 995         __atomic_post_full_fence();
 996         return ret;
 997 }
 998 #define atomic_try_cmpxchg atomic_try_cmpxchg
 999 #endif
1000
1001 #endif /* atomic_try_cmpxchg_relaxed */
1002
1003 #ifndef atomic_sub_and_test
1004 /**
1005  * atomic_sub_and_test - subtract value from variable and test result
1006  * @i: integer value to subtract
1007  * @v: pointer of type atomic_t
1008  *
1009  * Atomically subtracts @i from @v and returns
1010  * true if the result is zero, or false for all
1011  * other cases.
1012  */
1013 static inline bool
1014 atomic_sub_and_test(int i, atomic_t *v)
1015 {
1016         return atomic_sub_return(i, v) == 0;
1017 }
1018 #define atomic_sub_and_test atomic_sub_and_test
1019 #endif
1020
1021 #ifndef atomic_dec_and_test
1022 /**
1023  * atomic_dec_and_test - decrement and test
1024  * @v: pointer of type atomic_t
1025  *
1026  * Atomically decrements @v by 1 and
1027  * returns true if the result is 0, or false for all other
1028  * cases.
1029  */
1030 static inline bool
1031 atomic_dec_and_test(atomic_t *v)
1032 {
1033         return atomic_dec_return(v) == 0;
1034 }
1035 #define atomic_dec_and_test atomic_dec_and_test
1036 #endif
1037
1038 #ifndef atomic_inc_and_test
1039 /**
1040  * atomic_inc_and_test - increment and test
1041  * @v: pointer of type atomic_t
1042  *
1043  * Atomically increments @v by 1
1044  * and returns true if the result is zero, or false for all
1045  * other cases.
1046  */
1047 static inline bool
1048 atomic_inc_and_test(atomic_t *v)
1049 {
1050         return atomic_inc_return(v) == 0;
1051 }
1052 #define atomic_inc_and_test atomic_inc_and_test
1053 #endif
1054
1055 #ifndef atomic_add_negative
1056 /**
1057  * atomic_add_negative - add and test if negative
1058  * @i: integer value to add
1059  * @v: pointer of type atomic_t
1060  *
1061  * Atomically adds @i to @v and returns true
1062  * if the result is negative, or false when
1063  * result is greater than or equal to zero.
1064  */
1065 static inline bool
1066 atomic_add_negative(int i, atomic_t *v)
1067 {
1068         return atomic_add_return(i, v) < 0;
1069 }
1070 #define atomic_add_negative atomic_add_negative
1071 #endif
1072
1073 #ifndef atomic_fetch_add_unless
1074 /**
1075  * atomic_fetch_add_unless - add unless the number is already a given value
1076  * @v: pointer of type atomic_t
1077  * @a: the amount to add to v...
1078  * @u: ...unless v is equal to u.
1079  *
1080  * Atomically adds @a to @v, so long as @v was not already @u.
1081  * Returns original value of @v
1082  */
1083 static inline int
1084 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1085 {
1086         int c = atomic_read(v);
1087
1088         do {
1089                 if (unlikely(c == u))
1090                         break;
1091         } while (!atomic_try_cmpxchg(v, &c, c + a));
1092
1093         return c;
1094 }
1095 #define atomic_fetch_add_unless atomic_fetch_add_unless
1096 #endif
1097
1098 #ifndef atomic_add_unless
1099 /**
1100  * atomic_add_unless - add unless the number is already a given value
1101  * @v: pointer of type atomic_t
1102  * @a: the amount to add to v...
1103  * @u: ...unless v is equal to u.
1104  *
1105  * Atomically adds @a to @v, if @v was not already @u.
1106  * Returns true if the addition was done.
1107  */
1108 static inline bool
1109 atomic_add_unless(atomic_t *v, int a, int u)
1110 {
1111         return atomic_fetch_add_unless(v, a, u) != u;
1112 }
1113 #define atomic_add_unless atomic_add_unless
1114 #endif
1115
1116 #ifndef atomic_inc_not_zero
1117 /**
1118  * atomic_inc_not_zero - increment unless the number is zero
1119  * @v: pointer of type atomic_t
1120  *
1121  * Atomically increments @v by 1, if @v is non-zero.
1122  * Returns true if the increment was done.
1123  */
1124 static inline bool
1125 atomic_inc_not_zero(atomic_t *v)
1126 {
1127         return atomic_add_unless(v, 1, 0);
1128 }
1129 #define atomic_inc_not_zero atomic_inc_not_zero
1130 #endif
1131
1132 #ifndef atomic_inc_unless_negative
1133 static inline bool
1134 atomic_inc_unless_negative(atomic_t *v)
1135 {
1136         int c = atomic_read(v);
1137
1138         do {
1139                 if (unlikely(c < 0))
1140                         return false;
1141         } while (!atomic_try_cmpxchg(v, &c, c + 1));
1142
1143         return true;
1144 }
1145 #define atomic_inc_unless_negative atomic_inc_unless_negative
1146 #endif
1147
1148 #ifndef atomic_dec_unless_positive
1149 static inline bool
1150 atomic_dec_unless_positive(atomic_t *v)
1151 {
1152         int c = atomic_read(v);
1153
1154         do {
1155                 if (unlikely(c > 0))
1156                         return false;
1157         } while (!atomic_try_cmpxchg(v, &c, c - 1));
1158
1159         return true;
1160 }
1161 #define atomic_dec_unless_positive atomic_dec_unless_positive
1162 #endif
1163
1164 #ifndef atomic_dec_if_positive
1165 static inline int
1166 atomic_dec_if_positive(atomic_t *v)
1167 {
1168         int dec, c = atomic_read(v);
1169
1170         do {
1171                 dec = c - 1;
1172                 if (unlikely(dec < 0))
1173                         break;
1174         } while (!atomic_try_cmpxchg(v, &c, dec));
1175
1176         return dec;
1177 }
1178 #define atomic_dec_if_positive atomic_dec_if_positive
1179 #endif
1180
1181 #define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
1182 #define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1183
1184 #ifdef CONFIG_GENERIC_ATOMIC64
1185 #include <asm-generic/atomic64.h>
1186 #endif
1187
1188 #ifndef atomic64_read_acquire
1189 static inline s64
1190 atomic64_read_acquire(const atomic64_t *v)
1191 {
1192         return smp_load_acquire(&(v)->counter);
1193 }
1194 #define atomic64_read_acquire atomic64_read_acquire
1195 #endif
1196
1197 #ifndef atomic64_set_release
1198 static inline void
1199 atomic64_set_release(atomic64_t *v, s64 i)
1200 {
1201         smp_store_release(&(v)->counter, i);
1202 }
1203 #define atomic64_set_release atomic64_set_release
1204 #endif
1205
1206 #ifndef atomic64_add_return_relaxed
1207 #define atomic64_add_return_acquire atomic64_add_return
1208 #define atomic64_add_return_release atomic64_add_return
1209 #define atomic64_add_return_relaxed atomic64_add_return
1210 #else /* atomic64_add_return_relaxed */
1211
1212 #ifndef atomic64_add_return_acquire
1213 static inline s64
1214 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1215 {
1216         s64 ret = atomic64_add_return_relaxed(i, v);
1217         __atomic_acquire_fence();
1218         return ret;
1219 }
1220 #define atomic64_add_return_acquire atomic64_add_return_acquire
1221 #endif
1222
1223 #ifndef atomic64_add_return_release
1224 static inline s64
1225 atomic64_add_return_release(s64 i, atomic64_t *v)
1226 {
1227         __atomic_release_fence();
1228         return atomic64_add_return_relaxed(i, v);
1229 }
1230 #define atomic64_add_return_release atomic64_add_return_release
1231 #endif
1232
1233 #ifndef atomic64_add_return
1234 static inline s64
1235 atomic64_add_return(s64 i, atomic64_t *v)
1236 {
1237         s64 ret;
1238         __atomic_pre_full_fence();
1239         ret = atomic64_add_return_relaxed(i, v);
1240         __atomic_post_full_fence();
1241         return ret;
1242 }
1243 #define atomic64_add_return atomic64_add_return
1244 #endif
1245
1246 #endif /* atomic64_add_return_relaxed */
1247
1248 #ifndef atomic64_fetch_add_relaxed
1249 #define atomic64_fetch_add_acquire atomic64_fetch_add
1250 #define atomic64_fetch_add_release atomic64_fetch_add
1251 #define atomic64_fetch_add_relaxed atomic64_fetch_add
1252 #else /* atomic64_fetch_add_relaxed */
1253
1254 #ifndef atomic64_fetch_add_acquire
1255 static inline s64
1256 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1257 {
1258         s64 ret = atomic64_fetch_add_relaxed(i, v);
1259         __atomic_acquire_fence();
1260         return ret;
1261 }
1262 #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
1263 #endif
1264
1265 #ifndef atomic64_fetch_add_release
1266 static inline s64
1267 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1268 {
1269         __atomic_release_fence();
1270         return atomic64_fetch_add_relaxed(i, v);
1271 }
1272 #define atomic64_fetch_add_release atomic64_fetch_add_release
1273 #endif
1274
1275 #ifndef atomic64_fetch_add
1276 static inline s64
1277 atomic64_fetch_add(s64 i, atomic64_t *v)
1278 {
1279         s64 ret;
1280         __atomic_pre_full_fence();
1281         ret = atomic64_fetch_add_relaxed(i, v);
1282         __atomic_post_full_fence();
1283         return ret;
1284 }
1285 #define atomic64_fetch_add atomic64_fetch_add
1286 #endif
1287
1288 #endif /* atomic64_fetch_add_relaxed */
1289
1290 #ifndef atomic64_sub_return_relaxed
1291 #define atomic64_sub_return_acquire atomic64_sub_return
1292 #define atomic64_sub_return_release atomic64_sub_return
1293 #define atomic64_sub_return_relaxed atomic64_sub_return
1294 #else /* atomic64_sub_return_relaxed */
1295
1296 #ifndef atomic64_sub_return_acquire
1297 static inline s64
1298 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1299 {
1300         s64 ret = atomic64_sub_return_relaxed(i, v);
1301         __atomic_acquire_fence();
1302         return ret;
1303 }
1304 #define atomic64_sub_return_acquire atomic64_sub_return_acquire
1305 #endif
1306
1307 #ifndef atomic64_sub_return_release
1308 static inline s64
1309 atomic64_sub_return_release(s64 i, atomic64_t *v)
1310 {
1311         __atomic_release_fence();
1312         return atomic64_sub_return_relaxed(i, v);
1313 }
1314 #define atomic64_sub_return_release atomic64_sub_return_release
1315 #endif
1316
1317 #ifndef atomic64_sub_return
1318 static inline s64
1319 atomic64_sub_return(s64 i, atomic64_t *v)
1320 {
1321         s64 ret;
1322         __atomic_pre_full_fence();
1323         ret = atomic64_sub_return_relaxed(i, v);
1324         __atomic_post_full_fence();
1325         return ret;
1326 }
1327 #define atomic64_sub_return atomic64_sub_return
1328 #endif
1329
1330 #endif /* atomic64_sub_return_relaxed */
1331
1332 #ifndef atomic64_fetch_sub_relaxed
1333 #define atomic64_fetch_sub_acquire atomic64_fetch_sub
1334 #define atomic64_fetch_sub_release atomic64_fetch_sub
1335 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub
1336 #else /* atomic64_fetch_sub_relaxed */
1337
1338 #ifndef atomic64_fetch_sub_acquire
1339 static inline s64
1340 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1341 {
1342         s64 ret = atomic64_fetch_sub_relaxed(i, v);
1343         __atomic_acquire_fence();
1344         return ret;
1345 }
1346 #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
1347 #endif
1348
1349 #ifndef atomic64_fetch_sub_release
1350 static inline s64
1351 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1352 {
1353         __atomic_release_fence();
1354         return atomic64_fetch_sub_relaxed(i, v);
1355 }
1356 #define atomic64_fetch_sub_release atomic64_fetch_sub_release
1357 #endif
1358
1359 #ifndef atomic64_fetch_sub
1360 static inline s64
1361 atomic64_fetch_sub(s64 i, atomic64_t *v)
1362 {
1363         s64 ret;
1364         __atomic_pre_full_fence();
1365         ret = atomic64_fetch_sub_relaxed(i, v);
1366         __atomic_post_full_fence();
1367         return ret;
1368 }
1369 #define atomic64_fetch_sub atomic64_fetch_sub
1370 #endif
1371
1372 #endif /* atomic64_fetch_sub_relaxed */
1373
1374 #ifndef atomic64_inc
1375 static inline void
1376 atomic64_inc(atomic64_t *v)
1377 {
1378         atomic64_add(1, v);
1379 }
1380 #define atomic64_inc atomic64_inc
1381 #endif
1382
1383 #ifndef atomic64_inc_return_relaxed
1384 #ifdef atomic64_inc_return
1385 #define atomic64_inc_return_acquire atomic64_inc_return
1386 #define atomic64_inc_return_release atomic64_inc_return
1387 #define atomic64_inc_return_relaxed atomic64_inc_return
1388 #endif /* atomic64_inc_return */
1389
1390 #ifndef atomic64_inc_return
1391 static inline s64
1392 atomic64_inc_return(atomic64_t *v)
1393 {
1394         return atomic64_add_return(1, v);
1395 }
1396 #define atomic64_inc_return atomic64_inc_return
1397 #endif
1398
1399 #ifndef atomic64_inc_return_acquire
1400 static inline s64
1401 atomic64_inc_return_acquire(atomic64_t *v)
1402 {
1403         return atomic64_add_return_acquire(1, v);
1404 }
1405 #define atomic64_inc_return_acquire atomic64_inc_return_acquire
1406 #endif
1407
1408 #ifndef atomic64_inc_return_release
1409 static inline s64
1410 atomic64_inc_return_release(atomic64_t *v)
1411 {
1412         return atomic64_add_return_release(1, v);
1413 }
1414 #define atomic64_inc_return_release atomic64_inc_return_release
1415 #endif
1416
1417 #ifndef atomic64_inc_return_relaxed
1418 static inline s64
1419 atomic64_inc_return_relaxed(atomic64_t *v)
1420 {
1421         return atomic64_add_return_relaxed(1, v);
1422 }
1423 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
1424 #endif
1425
1426 #else /* atomic64_inc_return_relaxed */
1427
1428 #ifndef atomic64_inc_return_acquire
1429 static inline s64
1430 atomic64_inc_return_acquire(atomic64_t *v)
1431 {
1432         s64 ret = atomic64_inc_return_relaxed(v);
1433         __atomic_acquire_fence();
1434         return ret;
1435 }
1436 #define atomic64_inc_return_acquire atomic64_inc_return_acquire
1437 #endif
1438
1439 #ifndef atomic64_inc_return_release
1440 static inline s64
1441 atomic64_inc_return_release(atomic64_t *v)
1442 {
1443         __atomic_release_fence();
1444         return atomic64_inc_return_relaxed(v);
1445 }
1446 #define atomic64_inc_return_release atomic64_inc_return_release
1447 #endif
1448
1449 #ifndef atomic64_inc_return
1450 static inline s64
1451 atomic64_inc_return(atomic64_t *v)
1452 {
1453         s64 ret;
1454         __atomic_pre_full_fence();
1455         ret = atomic64_inc_return_relaxed(v);
1456         __atomic_post_full_fence();
1457         return ret;
1458 }
1459 #define atomic64_inc_return atomic64_inc_return
1460 #endif
1461
1462 #endif /* atomic64_inc_return_relaxed */
1463
1464 #ifndef atomic64_fetch_inc_relaxed
1465 #ifdef atomic64_fetch_inc
1466 #define atomic64_fetch_inc_acquire atomic64_fetch_inc
1467 #define atomic64_fetch_inc_release atomic64_fetch_inc
1468 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc
1469 #endif /* atomic64_fetch_inc */
1470
1471 #ifndef atomic64_fetch_inc
1472 static inline s64
1473 atomic64_fetch_inc(atomic64_t *v)
1474 {
1475         return atomic64_fetch_add(1, v);
1476 }
1477 #define atomic64_fetch_inc atomic64_fetch_inc
1478 #endif
1479
1480 #ifndef atomic64_fetch_inc_acquire
1481 static inline s64
1482 atomic64_fetch_inc_acquire(atomic64_t *v)
1483 {
1484         return atomic64_fetch_add_acquire(1, v);
1485 }
1486 #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1487 #endif
1488
1489 #ifndef atomic64_fetch_inc_release
1490 static inline s64
1491 atomic64_fetch_inc_release(atomic64_t *v)
1492 {
1493         return atomic64_fetch_add_release(1, v);
1494 }
1495 #define atomic64_fetch_inc_release atomic64_fetch_inc_release
1496 #endif
1497
1498 #ifndef atomic64_fetch_inc_relaxed
1499 static inline s64
1500 atomic64_fetch_inc_relaxed(atomic64_t *v)
1501 {
1502         return atomic64_fetch_add_relaxed(1, v);
1503 }
1504 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
1505 #endif
1506
1507 #else /* atomic64_fetch_inc_relaxed */
1508
1509 #ifndef atomic64_fetch_inc_acquire
1510 static inline s64
1511 atomic64_fetch_inc_acquire(atomic64_t *v)
1512 {
1513         s64 ret = atomic64_fetch_inc_relaxed(v);
1514         __atomic_acquire_fence();
1515         return ret;
1516 }
1517 #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1518 #endif
1519
1520 #ifndef atomic64_fetch_inc_release
1521 static inline s64
1522 atomic64_fetch_inc_release(atomic64_t *v)
1523 {
1524         __atomic_release_fence();
1525         return atomic64_fetch_inc_relaxed(v);
1526 }
1527 #define atomic64_fetch_inc_release atomic64_fetch_inc_release
1528 #endif
1529
1530 #ifndef atomic64_fetch_inc
1531 static inline s64
1532 atomic64_fetch_inc(atomic64_t *v)
1533 {
1534         s64 ret;
1535         __atomic_pre_full_fence();
1536         ret = atomic64_fetch_inc_relaxed(v);
1537         __atomic_post_full_fence();
1538         return ret;
1539 }
1540 #define atomic64_fetch_inc atomic64_fetch_inc
1541 #endif
1542
1543 #endif /* atomic64_fetch_inc_relaxed */
1544
1545 #ifndef atomic64_dec
1546 static inline void
1547 atomic64_dec(atomic64_t *v)
1548 {
1549         atomic64_sub(1, v);
1550 }
1551 #define atomic64_dec atomic64_dec
1552 #endif
1553
1554 #ifndef atomic64_dec_return_relaxed
1555 #ifdef atomic64_dec_return
1556 #define atomic64_dec_return_acquire atomic64_dec_return
1557 #define atomic64_dec_return_release atomic64_dec_return
1558 #define atomic64_dec_return_relaxed atomic64_dec_return
1559 #endif /* atomic64_dec_return */
1560
1561 #ifndef atomic64_dec_return
1562 static inline s64
1563 atomic64_dec_return(atomic64_t *v)
1564 {
1565         return atomic64_sub_return(1, v);
1566 }
1567 #define atomic64_dec_return atomic64_dec_return
1568 #endif
1569
1570 #ifndef atomic64_dec_return_acquire
1571 static inline s64
1572 atomic64_dec_return_acquire(atomic64_t *v)
1573 {
1574         return atomic64_sub_return_acquire(1, v);
1575 }
1576 #define atomic64_dec_return_acquire atomic64_dec_return_acquire
1577 #endif
1578
1579 #ifndef atomic64_dec_return_release
1580 static inline s64
1581 atomic64_dec_return_release(atomic64_t *v)
1582 {
1583         return atomic64_sub_return_release(1, v);
1584 }
1585 #define atomic64_dec_return_release atomic64_dec_return_release
1586 #endif
1587
1588 #ifndef atomic64_dec_return_relaxed
1589 static inline s64
1590 atomic64_dec_return_relaxed(atomic64_t *v)
1591 {
1592         return atomic64_sub_return_relaxed(1, v);
1593 }
1594 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
1595 #endif
1596
1597 #else /* atomic64_dec_return_relaxed */
1598
1599 #ifndef atomic64_dec_return_acquire
1600 static inline s64
1601 atomic64_dec_return_acquire(atomic64_t *v)
1602 {
1603         s64 ret = atomic64_dec_return_relaxed(v);
1604         __atomic_acquire_fence();
1605         return ret;
1606 }
1607 #define atomic64_dec_return_acquire atomic64_dec_return_acquire
1608 #endif
1609
1610 #ifndef atomic64_dec_return_release
1611 static inline s64
1612 atomic64_dec_return_release(atomic64_t *v)
1613 {
1614         __atomic_release_fence();
1615         return atomic64_dec_return_relaxed(v);
1616 }
1617 #define atomic64_dec_return_release atomic64_dec_return_release
1618 #endif
1619
1620 #ifndef atomic64_dec_return
1621 static inline s64
1622 atomic64_dec_return(atomic64_t *v)
1623 {
1624         s64 ret;
1625         __atomic_pre_full_fence();
1626         ret = atomic64_dec_return_relaxed(v);
1627         __atomic_post_full_fence();
1628         return ret;
1629 }
1630 #define atomic64_dec_return atomic64_dec_return
1631 #endif
1632
1633 #endif /* atomic64_dec_return_relaxed */
1634
1635 #ifndef atomic64_fetch_dec_relaxed
1636 #ifdef atomic64_fetch_dec
1637 #define atomic64_fetch_dec_acquire atomic64_fetch_dec
1638 #define atomic64_fetch_dec_release atomic64_fetch_dec
1639 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec
1640 #endif /* atomic64_fetch_dec */
1641
1642 #ifndef atomic64_fetch_dec
1643 static inline s64
1644 atomic64_fetch_dec(atomic64_t *v)
1645 {
1646         return atomic64_fetch_sub(1, v);
1647 }
1648 #define atomic64_fetch_dec atomic64_fetch_dec
1649 #endif
1650
1651 #ifndef atomic64_fetch_dec_acquire
1652 static inline s64
1653 atomic64_fetch_dec_acquire(atomic64_t *v)
1654 {
1655         return atomic64_fetch_sub_acquire(1, v);
1656 }
1657 #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1658 #endif
1659
1660 #ifndef atomic64_fetch_dec_release
1661 static inline s64
1662 atomic64_fetch_dec_release(atomic64_t *v)
1663 {
1664         return atomic64_fetch_sub_release(1, v);
1665 }
1666 #define atomic64_fetch_dec_release atomic64_fetch_dec_release
1667 #endif
1668
1669 #ifndef atomic64_fetch_dec_relaxed
1670 static inline s64
1671 atomic64_fetch_dec_relaxed(atomic64_t *v)
1672 {
1673         return atomic64_fetch_sub_relaxed(1, v);
1674 }
1675 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
1676 #endif
1677
1678 #else /* atomic64_fetch_dec_relaxed */
1679
1680 #ifndef atomic64_fetch_dec_acquire
1681 static inline s64
1682 atomic64_fetch_dec_acquire(atomic64_t *v)
1683 {
1684         s64 ret = atomic64_fetch_dec_relaxed(v);
1685         __atomic_acquire_fence();
1686         return ret;
1687 }
1688 #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1689 #endif
1690
1691 #ifndef atomic64_fetch_dec_release
1692 static inline s64
1693 atomic64_fetch_dec_release(atomic64_t *v)
1694 {
1695         __atomic_release_fence();
1696         return atomic64_fetch_dec_relaxed(v);
1697 }
1698 #define atomic64_fetch_dec_release atomic64_fetch_dec_release
1699 #endif
1700
1701 #ifndef atomic64_fetch_dec
1702 static inline s64
1703 atomic64_fetch_dec(atomic64_t *v)
1704 {
1705         s64 ret;
1706         __atomic_pre_full_fence();
1707         ret = atomic64_fetch_dec_relaxed(v);
1708         __atomic_post_full_fence();
1709         return ret;
1710 }
1711 #define atomic64_fetch_dec atomic64_fetch_dec
1712 #endif
1713
1714 #endif /* atomic64_fetch_dec_relaxed */
1715
1716 #ifndef atomic64_fetch_and_relaxed
1717 #define atomic64_fetch_and_acquire atomic64_fetch_and
1718 #define atomic64_fetch_and_release atomic64_fetch_and
1719 #define atomic64_fetch_and_relaxed atomic64_fetch_and
1720 #else /* atomic64_fetch_and_relaxed */
1721
1722 #ifndef atomic64_fetch_and_acquire
1723 static inline s64
1724 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1725 {
1726         s64 ret = atomic64_fetch_and_relaxed(i, v);
1727         __atomic_acquire_fence();
1728         return ret;
1729 }
1730 #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
1731 #endif
1732
1733 #ifndef atomic64_fetch_and_release
1734 static inline s64
1735 atomic64_fetch_and_release(s64 i, atomic64_t *v)
1736 {
1737         __atomic_release_fence();
1738         return atomic64_fetch_and_relaxed(i, v);
1739 }
1740 #define atomic64_fetch_and_release atomic64_fetch_and_release
1741 #endif
1742
1743 #ifndef atomic64_fetch_and
1744 static inline s64
1745 atomic64_fetch_and(s64 i, atomic64_t *v)
1746 {
1747         s64 ret;
1748         __atomic_pre_full_fence();
1749         ret = atomic64_fetch_and_relaxed(i, v);
1750         __atomic_post_full_fence();
1751         return ret;
1752 }
1753 #define atomic64_fetch_and atomic64_fetch_and
1754 #endif
1755
1756 #endif /* atomic64_fetch_and_relaxed */
1757
1758 #ifndef atomic64_andnot
1759 static inline void
1760 atomic64_andnot(s64 i, atomic64_t *v)
1761 {
1762         atomic64_and(~i, v);
1763 }
1764 #define atomic64_andnot atomic64_andnot
1765 #endif
1766
1767 #ifndef atomic64_fetch_andnot_relaxed
1768 #ifdef atomic64_fetch_andnot
1769 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
1770 #define atomic64_fetch_andnot_release atomic64_fetch_andnot
1771 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
1772 #endif /* atomic64_fetch_andnot */
1773
1774 #ifndef atomic64_fetch_andnot
1775 static inline s64
1776 atomic64_fetch_andnot(s64 i, atomic64_t *v)
1777 {
1778         return atomic64_fetch_and(~i, v);
1779 }
1780 #define atomic64_fetch_andnot atomic64_fetch_andnot
1781 #endif
1782
1783 #ifndef atomic64_fetch_andnot_acquire
1784 static inline s64
1785 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1786 {
1787         return atomic64_fetch_and_acquire(~i, v);
1788 }
1789 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1790 #endif
1791
1792 #ifndef atomic64_fetch_andnot_release
1793 static inline s64
1794 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1795 {
1796         return atomic64_fetch_and_release(~i, v);
1797 }
1798 #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1799 #endif
1800
1801 #ifndef atomic64_fetch_andnot_relaxed
1802 static inline s64
1803 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1804 {
1805         return atomic64_fetch_and_relaxed(~i, v);
1806 }
1807 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
1808 #endif
1809
1810 #else /* atomic64_fetch_andnot_relaxed */
1811
1812 #ifndef atomic64_fetch_andnot_acquire
1813 static inline s64
1814 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1815 {
1816         s64 ret = atomic64_fetch_andnot_relaxed(i, v);
1817         __atomic_acquire_fence();
1818         return ret;
1819 }
1820 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1821 #endif
1822
1823 #ifndef atomic64_fetch_andnot_release
1824 static inline s64
1825 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1826 {
1827         __atomic_release_fence();
1828         return atomic64_fetch_andnot_relaxed(i, v);
1829 }
1830 #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1831 #endif
1832
1833 #ifndef atomic64_fetch_andnot
1834 static inline s64
1835 atomic64_fetch_andnot(s64 i, atomic64_t *v)
1836 {
1837         s64 ret;
1838         __atomic_pre_full_fence();
1839         ret = atomic64_fetch_andnot_relaxed(i, v);
1840         __atomic_post_full_fence();
1841         return ret;
1842 }
1843 #define atomic64_fetch_andnot atomic64_fetch_andnot
1844 #endif
1845
1846 #endif /* atomic64_fetch_andnot_relaxed */
1847
1848 #ifndef atomic64_fetch_or_relaxed
1849 #define atomic64_fetch_or_acquire atomic64_fetch_or
1850 #define atomic64_fetch_or_release atomic64_fetch_or
1851 #define atomic64_fetch_or_relaxed atomic64_fetch_or
1852 #else /* atomic64_fetch_or_relaxed */
1853
1854 #ifndef atomic64_fetch_or_acquire
1855 static inline s64
1856 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1857 {
1858         s64 ret = atomic64_fetch_or_relaxed(i, v);
1859         __atomic_acquire_fence();
1860         return ret;
1861 }
1862 #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
1863 #endif
1864
1865 #ifndef atomic64_fetch_or_release
1866 static inline s64
1867 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1868 {
1869         __atomic_release_fence();
1870         return atomic64_fetch_or_relaxed(i, v);
1871 }
1872 #define atomic64_fetch_or_release atomic64_fetch_or_release
1873 #endif
1874
1875 #ifndef atomic64_fetch_or
1876 static inline s64
1877 atomic64_fetch_or(s64 i, atomic64_t *v)
1878 {
1879         s64 ret;
1880         __atomic_pre_full_fence();
1881         ret = atomic64_fetch_or_relaxed(i, v);
1882         __atomic_post_full_fence();
1883         return ret;
1884 }
1885 #define atomic64_fetch_or atomic64_fetch_or
1886 #endif
1887
1888 #endif /* atomic64_fetch_or_relaxed */
1889
1890 #ifndef atomic64_fetch_xor_relaxed
1891 #define atomic64_fetch_xor_acquire atomic64_fetch_xor
1892 #define atomic64_fetch_xor_release atomic64_fetch_xor
1893 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor
1894 #else /* atomic64_fetch_xor_relaxed */
1895
1896 #ifndef atomic64_fetch_xor_acquire
1897 static inline s64
1898 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1899 {
1900         s64 ret = atomic64_fetch_xor_relaxed(i, v);
1901         __atomic_acquire_fence();
1902         return ret;
1903 }
1904 #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
1905 #endif
1906
1907 #ifndef atomic64_fetch_xor_release
1908 static inline s64
1909 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1910 {
1911         __atomic_release_fence();
1912         return atomic64_fetch_xor_relaxed(i, v);
1913 }
1914 #define atomic64_fetch_xor_release atomic64_fetch_xor_release
1915 #endif
1916
1917 #ifndef atomic64_fetch_xor
1918 static inline s64
1919 atomic64_fetch_xor(s64 i, atomic64_t *v)
1920 {
1921         s64 ret;
1922         __atomic_pre_full_fence();
1923         ret = atomic64_fetch_xor_relaxed(i, v);
1924         __atomic_post_full_fence();
1925         return ret;
1926 }
1927 #define atomic64_fetch_xor atomic64_fetch_xor
1928 #endif
1929
1930 #endif /* atomic64_fetch_xor_relaxed */
1931
1932 #ifndef atomic64_xchg_relaxed
1933 #define atomic64_xchg_acquire atomic64_xchg
1934 #define atomic64_xchg_release atomic64_xchg
1935 #define atomic64_xchg_relaxed atomic64_xchg
1936 #else /* atomic64_xchg_relaxed */
1937
1938 #ifndef atomic64_xchg_acquire
1939 static inline s64
1940 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1941 {
1942         s64 ret = atomic64_xchg_relaxed(v, i);
1943         __atomic_acquire_fence();
1944         return ret;
1945 }
1946 #define atomic64_xchg_acquire atomic64_xchg_acquire
1947 #endif
1948
1949 #ifndef atomic64_xchg_release
1950 static inline s64
1951 atomic64_xchg_release(atomic64_t *v, s64 i)
1952 {
1953         __atomic_release_fence();
1954         return atomic64_xchg_relaxed(v, i);
1955 }
1956 #define atomic64_xchg_release atomic64_xchg_release
1957 #endif
1958
1959 #ifndef atomic64_xchg
1960 static inline s64
1961 atomic64_xchg(atomic64_t *v, s64 i)
1962 {
1963         s64 ret;
1964         __atomic_pre_full_fence();
1965         ret = atomic64_xchg_relaxed(v, i);
1966         __atomic_post_full_fence();
1967         return ret;
1968 }
1969 #define atomic64_xchg atomic64_xchg
1970 #endif
1971
1972 #endif /* atomic64_xchg_relaxed */
1973
1974 #ifndef atomic64_cmpxchg_relaxed
1975 #define atomic64_cmpxchg_acquire atomic64_cmpxchg
1976 #define atomic64_cmpxchg_release atomic64_cmpxchg
1977 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1978 #else /* atomic64_cmpxchg_relaxed */
1979
1980 #ifndef atomic64_cmpxchg_acquire
1981 static inline s64
1982 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1983 {
1984         s64 ret = atomic64_cmpxchg_relaxed(v, old, new);
1985         __atomic_acquire_fence();
1986         return ret;
1987 }
1988 #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
1989 #endif
1990
1991 #ifndef atomic64_cmpxchg_release
1992 static inline s64
1993 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1994 {
1995         __atomic_release_fence();
1996         return atomic64_cmpxchg_relaxed(v, old, new);
1997 }
1998 #define atomic64_cmpxchg_release atomic64_cmpxchg_release
1999 #endif
2000
2001 #ifndef atomic64_cmpxchg
2002 static inline s64
2003 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2004 {
2005         s64 ret;
2006         __atomic_pre_full_fence();
2007         ret = atomic64_cmpxchg_relaxed(v, old, new);
2008         __atomic_post_full_fence();
2009         return ret;
2010 }
2011 #define atomic64_cmpxchg atomic64_cmpxchg
2012 #endif
2013
2014 #endif /* atomic64_cmpxchg_relaxed */
2015
2016 #ifndef atomic64_try_cmpxchg_relaxed
2017 #ifdef atomic64_try_cmpxchg
2018 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
2019 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
2020 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
2021 #endif /* atomic64_try_cmpxchg */
2022
2023 #ifndef atomic64_try_cmpxchg
2024 static inline bool
2025 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2026 {
2027         s64 r, o = *old;
2028         r = atomic64_cmpxchg(v, o, new);
2029         if (unlikely(r != o))
2030                 *old = r;
2031         return likely(r == o);
2032 }
2033 #define atomic64_try_cmpxchg atomic64_try_cmpxchg
2034 #endif
2035
2036 #ifndef atomic64_try_cmpxchg_acquire
2037 static inline bool
2038 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2039 {
2040         s64 r, o = *old;
2041         r = atomic64_cmpxchg_acquire(v, o, new);
2042         if (unlikely(r != o))
2043                 *old = r;
2044         return likely(r == o);
2045 }
2046 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
2047 #endif
2048
2049 #ifndef atomic64_try_cmpxchg_release
2050 static inline bool
2051 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2052 {
2053         s64 r, o = *old;
2054         r = atomic64_cmpxchg_release(v, o, new);
2055         if (unlikely(r != o))
2056                 *old = r;
2057         return likely(r == o);
2058 }
2059 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
2060 #endif
2061
2062 #ifndef atomic64_try_cmpxchg_relaxed
2063 static inline bool
2064 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2065 {
2066         s64 r, o = *old;
2067         r = atomic64_cmpxchg_relaxed(v, o, new);
2068         if (unlikely(r != o))
2069                 *old = r;
2070         return likely(r == o);
2071 }
2072 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
2073 #endif
2074
2075 #else /* atomic64_try_cmpxchg_relaxed */
2076
2077 #ifndef atomic64_try_cmpxchg_acquire
2078 static inline bool
2079 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2080 {
2081         bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);
2082         __atomic_acquire_fence();
2083         return ret;
2084 }
2085 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
2086 #endif
2087
2088 #ifndef atomic64_try_cmpxchg_release
2089 static inline bool
2090 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2091 {
2092         __atomic_release_fence();
2093         return atomic64_try_cmpxchg_relaxed(v, old, new);
2094 }
2095 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
2096 #endif
2097
2098 #ifndef atomic64_try_cmpxchg
2099 static inline bool
2100 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2101 {
2102         bool ret;
2103         __atomic_pre_full_fence();
2104         ret = atomic64_try_cmpxchg_relaxed(v, old, new);
2105         __atomic_post_full_fence();
2106         return ret;
2107 }
2108 #define atomic64_try_cmpxchg atomic64_try_cmpxchg
2109 #endif
2110
2111 #endif /* atomic64_try_cmpxchg_relaxed */
2112
2113 #ifndef atomic64_sub_and_test
2114 /**
2115  * atomic64_sub_and_test - subtract value from variable and test result
2116  * @i: integer value to subtract
2117  * @v: pointer of type atomic64_t
2118  *
2119  * Atomically subtracts @i from @v and returns
2120  * true if the result is zero, or false for all
2121  * other cases.
2122  */
2123 static inline bool
2124 atomic64_sub_and_test(s64 i, atomic64_t *v)
2125 {
2126         return atomic64_sub_return(i, v) == 0;
2127 }
2128 #define atomic64_sub_and_test atomic64_sub_and_test
2129 #endif
2130
2131 #ifndef atomic64_dec_and_test
2132 /**
2133  * atomic64_dec_and_test - decrement and test
2134  * @v: pointer of type atomic64_t
2135  *
2136  * Atomically decrements @v by 1 and
2137  * returns true if the result is 0, or false for all other
2138  * cases.
2139  */
2140 static inline bool
2141 atomic64_dec_and_test(atomic64_t *v)
2142 {
2143         return atomic64_dec_return(v) == 0;
2144 }
2145 #define atomic64_dec_and_test atomic64_dec_and_test
2146 #endif
2147
2148 #ifndef atomic64_inc_and_test
2149 /**
2150  * atomic64_inc_and_test - increment and test
2151  * @v: pointer of type atomic64_t
2152  *
2153  * Atomically increments @v by 1
2154  * and returns true if the result is zero, or false for all
2155  * other cases.
2156  */
2157 static inline bool
2158 atomic64_inc_and_test(atomic64_t *v)
2159 {
2160         return atomic64_inc_return(v) == 0;
2161 }
2162 #define atomic64_inc_and_test atomic64_inc_and_test
2163 #endif
2164
2165 #ifndef atomic64_add_negative
2166 /**
2167  * atomic64_add_negative - add and test if negative
2168  * @i: integer value to add
2169  * @v: pointer of type atomic64_t
2170  *
2171  * Atomically adds @i to @v and returns true
2172  * if the result is negative, or false when
2173  * result is greater than or equal to zero.
2174  */
2175 static inline bool
2176 atomic64_add_negative(s64 i, atomic64_t *v)
2177 {
2178         return atomic64_add_return(i, v) < 0;
2179 }
2180 #define atomic64_add_negative atomic64_add_negative
2181 #endif
2182
2183 #ifndef atomic64_fetch_add_unless
2184 /**
2185  * atomic64_fetch_add_unless - add unless the number is already a given value
2186  * @v: pointer of type atomic64_t
2187  * @a: the amount to add to v...
2188  * @u: ...unless v is equal to u.
2189  *
2190  * Atomically adds @a to @v, so long as @v was not already @u.
2191  * Returns original value of @v
2192  */
2193 static inline s64
2194 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2195 {
2196         s64 c = atomic64_read(v);
2197
2198         do {
2199                 if (unlikely(c == u))
2200                         break;
2201         } while (!atomic64_try_cmpxchg(v, &c, c + a));
2202
2203         return c;
2204 }
2205 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
2206 #endif
2207
2208 #ifndef atomic64_add_unless
2209 /**
2210  * atomic64_add_unless - add unless the number is already a given value
2211  * @v: pointer of type atomic64_t
2212  * @a: the amount to add to v...
2213  * @u: ...unless v is equal to u.
2214  *
2215  * Atomically adds @a to @v, if @v was not already @u.
2216  * Returns true if the addition was done.
2217  */
2218 static inline bool
2219 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2220 {
2221         return atomic64_fetch_add_unless(v, a, u) != u;
2222 }
2223 #define atomic64_add_unless atomic64_add_unless
2224 #endif
2225
2226 #ifndef atomic64_inc_not_zero
2227 /**
2228  * atomic64_inc_not_zero - increment unless the number is zero
2229  * @v: pointer of type atomic64_t
2230  *
2231  * Atomically increments @v by 1, if @v is non-zero.
2232  * Returns true if the increment was done.
2233  */
2234 static inline bool
2235 atomic64_inc_not_zero(atomic64_t *v)
2236 {
2237         return atomic64_add_unless(v, 1, 0);
2238 }
2239 #define atomic64_inc_not_zero atomic64_inc_not_zero
2240 #endif
2241
2242 #ifndef atomic64_inc_unless_negative
2243 static inline bool
2244 atomic64_inc_unless_negative(atomic64_t *v)
2245 {
2246         s64 c = atomic64_read(v);
2247
2248         do {
2249                 if (unlikely(c < 0))
2250                         return false;
2251         } while (!atomic64_try_cmpxchg(v, &c, c + 1));
2252
2253         return true;
2254 }
2255 #define atomic64_inc_unless_negative atomic64_inc_unless_negative
2256 #endif
2257
2258 #ifndef atomic64_dec_unless_positive
2259 static inline bool
2260 atomic64_dec_unless_positive(atomic64_t *v)
2261 {
2262         s64 c = atomic64_read(v);
2263
2264         do {
2265                 if (unlikely(c > 0))
2266                         return false;
2267         } while (!atomic64_try_cmpxchg(v, &c, c - 1));
2268
2269         return true;
2270 }
2271 #define atomic64_dec_unless_positive atomic64_dec_unless_positive
2272 #endif
2273
2274 #ifndef atomic64_dec_if_positive
2275 static inline s64
2276 atomic64_dec_if_positive(atomic64_t *v)
2277 {
2278         s64 dec, c = atomic64_read(v);
2279
2280         do {
2281                 dec = c - 1;
2282                 if (unlikely(dec < 0))
2283                         break;
2284         } while (!atomic64_try_cmpxchg(v, &c, dec));
2285
2286         return dec;
2287 }
2288 #define atomic64_dec_if_positive atomic64_dec_if_positive
2289 #endif
2290
2291 #define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
2292 #define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
2293
2294 #endif /* _LINUX_ATOMIC_FALLBACK_H */
2295 // 25de4a2804d70f57e994fe3b419148658bb5378a