arch/powerpc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_ATOMIC_H_
   3 #define _ASM_POWERPC_ATOMIC_H_
   4
   5 /*
   6  * PowerPC atomic operations
   7  */
   8
   9 #ifdef __KERNEL__
  10 #include <linux/types.h>
  11 #include <asm/cmpxchg.h>
  12 #include <asm/barrier.h>
  13
  14 #define ATOMIC_INIT(i)          { (i) }
  15
  16 /*
  17  * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
  18  * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
  19  * on the platform without lwsync.
  20  */
  21 #define __atomic_op_acquire(op, args...)                                \
  22 ({                                                                      \
  23         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  24         __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory");    \
  25         __ret;                                                          \
  26 })
  27
  28 #define __atomic_op_release(op, args...)                                \
  29 ({                                                                      \
  30         __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory");    \
  31         op##_relaxed(args);                                             \
  32 })
  33
  34 static __inline__ int atomic_read(const atomic_t *v)
  35 {
  36         int t;
  37
  38         __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
  39
  40         return t;
  41 }
  42
  43 static __inline__ void atomic_set(atomic_t *v, int i)
  44 {
  45         __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
  46 }
  47
  48 #define ATOMIC_OP(op, asm_op)                                           \
  49 static __inline__ void atomic_##op(int a, atomic_t *v)                  \
  50 {                                                                       \
  51         int t;                                                          \
  52                                                                         \
  53         __asm__ __volatile__(                                           \
  54 "1:     lwarx   %0,0,%3         # atomic_" #op "\n"                     \
  55         #asm_op " %0,%2,%0\n"                                           \
  56         PPC405_ERR77(0,%3)                                              \
  57 "       stwcx.  %0,0,%3 \n"                                             \
  58 "       bne-    1b\n"                                                   \
  59         : "=&r" (t), "+m" (v->counter)                                  \
  60         : "r" (a), "r" (&v->counter)                                    \
  61         : "cc");                                                        \
  62 }                                                                       \
  63
  64 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op)                            \
  65 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v)      \
  66 {                                                                       \
  67         int t;                                                          \
  68                                                                         \
  69         __asm__ __volatile__(                                           \
  70 "1:     lwarx   %0,0,%3         # atomic_" #op "_return_relaxed\n"      \
  71         #asm_op " %0,%2,%0\n"                                           \
  72         PPC405_ERR77(0, %3)                                             \
  73 "       stwcx.  %0,0,%3\n"                                              \
  74 "       bne-    1b\n"                                                   \
  75         : "=&r" (t), "+m" (v->counter)                                  \
  76         : "r" (a), "r" (&v->counter)                                    \
  77         : "cc");                                                        \
  78                                                                         \
  79         return t;                                                       \
  80 }
  81
  82 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op)                             \
  83 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v)       \
  84 {                                                                       \
  85         int res, t;                                                     \
  86                                                                         \
  87         __asm__ __volatile__(                                           \
  88 "1:     lwarx   %0,0,%4         # atomic_fetch_" #op "_relaxed\n"       \
  89         #asm_op " %1,%3,%0\n"                                           \
  90         PPC405_ERR77(0, %4)                                             \
  91 "       stwcx.  %1,0,%4\n"                                              \
  92 "       bne-    1b\n"                                                   \
  93         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
  94         : "r" (a), "r" (&v->counter)                                    \
  95         : "cc");                                                        \
  96                                                                         \
  97         return res;                                                     \
  98 }
  99
 100 #define ATOMIC_OPS(op, asm_op)                                          \
 101         ATOMIC_OP(op, asm_op)                                           \
 102         ATOMIC_OP_RETURN_RELAXED(op, asm_op)                            \
 103         ATOMIC_FETCH_OP_RELAXED(op, asm_op)
 104
 105 ATOMIC_OPS(add, add)
 106 ATOMIC_OPS(sub, subf)
 107
 108 #define atomic_add_return_relaxed atomic_add_return_relaxed
 109 #define atomic_sub_return_relaxed atomic_sub_return_relaxed
 110
 111 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
 112 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
 113
 114 #undef ATOMIC_OPS
 115 #define ATOMIC_OPS(op, asm_op)                                          \
 116         ATOMIC_OP(op, asm_op)                                           \
 117         ATOMIC_FETCH_OP_RELAXED(op, asm_op)
 118
 119 ATOMIC_OPS(and, and)
 120 ATOMIC_OPS(or, or)
 121 ATOMIC_OPS(xor, xor)
 122
 123 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
 124 #define atomic_fetch_or_relaxed  atomic_fetch_or_relaxed
 125 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
 126
 127 #undef ATOMIC_OPS
 128 #undef ATOMIC_FETCH_OP_RELAXED
 129 #undef ATOMIC_OP_RETURN_RELAXED
 130 #undef ATOMIC_OP
 131
 132 #define atomic_add_negative(a, v)       (atomic_add_return((a), (v)) < 0)
 133
 134 static __inline__ void atomic_inc(atomic_t *v)
 135 {
 136         int t;
 137
 138         __asm__ __volatile__(
 139 "1:     lwarx   %0,0,%2         # atomic_inc\n\
 140         addic   %0,%0,1\n"
 141         PPC405_ERR77(0,%2)
 142 "       stwcx.  %0,0,%2 \n\
 143         bne-    1b"
 144         : "=&r" (t), "+m" (v->counter)
 145         : "r" (&v->counter)
 146         : "cc", "xer");
 147 }
 148
 149 static __inline__ int atomic_inc_return_relaxed(atomic_t *v)
 150 {
 151         int t;
 152
 153         __asm__ __volatile__(
 154 "1:     lwarx   %0,0,%2         # atomic_inc_return_relaxed\n"
 155 "       addic   %0,%0,1\n"
 156         PPC405_ERR77(0, %2)
 157 "       stwcx.  %0,0,%2\n"
 158 "       bne-    1b"
 159         : "=&r" (t), "+m" (v->counter)
 160         : "r" (&v->counter)
 161         : "cc", "xer");
 162
 163         return t;
 164 }
 165
 166 /*
 167  * atomic_inc_and_test - increment and test
 168  * @v: pointer of type atomic_t
 169  *
 170  * Atomically increments @v by 1
 171  * and returns true if the result is zero, or false for all
 172  * other cases.
 173  */
 174 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
 175
 176 static __inline__ void atomic_dec(atomic_t *v)
 177 {
 178         int t;
 179
 180         __asm__ __volatile__(
 181 "1:     lwarx   %0,0,%2         # atomic_dec\n\
 182         addic   %0,%0,-1\n"
 183         PPC405_ERR77(0,%2)\
 184 "       stwcx.  %0,0,%2\n\
 185         bne-    1b"
 186         : "=&r" (t), "+m" (v->counter)
 187         : "r" (&v->counter)
 188         : "cc", "xer");
 189 }
 190
 191 static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
 192 {
 193         int t;
 194
 195         __asm__ __volatile__(
 196 "1:     lwarx   %0,0,%2         # atomic_dec_return_relaxed\n"
 197 "       addic   %0,%0,-1\n"
 198         PPC405_ERR77(0, %2)
 199 "       stwcx.  %0,0,%2\n"
 200 "       bne-    1b"
 201         : "=&r" (t), "+m" (v->counter)
 202         : "r" (&v->counter)
 203         : "cc", "xer");
 204
 205         return t;
 206 }
 207
 208 #define atomic_inc_return_relaxed atomic_inc_return_relaxed
 209 #define atomic_dec_return_relaxed atomic_dec_return_relaxed
 210
 211 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 212 #define atomic_cmpxchg_relaxed(v, o, n) \
 213         cmpxchg_relaxed(&((v)->counter), (o), (n))
 214 #define atomic_cmpxchg_acquire(v, o, n) \
 215         cmpxchg_acquire(&((v)->counter), (o), (n))
 216
 217 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
 218 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
 219
 220 /**
 221  * __atomic_add_unless - add unless the number is a given value
 222  * @v: pointer of type atomic_t
 223  * @a: the amount to add to v...
 224  * @u: ...unless v is equal to u.
 225  *
 226  * Atomically adds @a to @v, so long as it was not @u.
 227  * Returns the old value of @v.
 228  */
 229 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
 230 {
 231         int t;
 232
 233         __asm__ __volatile__ (
 234         PPC_ATOMIC_ENTRY_BARRIER
 235 "1:     lwarx   %0,0,%1         # __atomic_add_unless\n\
 236         cmpw    0,%0,%3 \n\
 237         beq     2f \n\
 238         add     %0,%2,%0 \n"
 239         PPC405_ERR77(0,%2)
 240 "       stwcx.  %0,0,%1 \n\
 241         bne-    1b \n"
 242         PPC_ATOMIC_EXIT_BARRIER
 243 "       subf    %0,%2,%0 \n\
 244 2:"
 245         : "=&r" (t)
 246         : "r" (&v->counter), "r" (a), "r" (u)
 247         : "cc", "memory");
 248
 249         return t;
 250 }
 251
 252 /**
 253  * atomic_inc_not_zero - increment unless the number is zero
 254  * @v: pointer of type atomic_t
 255  *
 256  * Atomically increments @v by 1, so long as @v is non-zero.
 257  * Returns non-zero if @v was non-zero, and zero otherwise.
 258  */
 259 static __inline__ int atomic_inc_not_zero(atomic_t *v)
 260 {
 261         int t1, t2;
 262
 263         __asm__ __volatile__ (
 264         PPC_ATOMIC_ENTRY_BARRIER
 265 "1:     lwarx   %0,0,%2         # atomic_inc_not_zero\n\
 266         cmpwi   0,%0,0\n\
 267         beq-    2f\n\
 268         addic   %1,%0,1\n"
 269         PPC405_ERR77(0,%2)
 270 "       stwcx.  %1,0,%2\n\
 271         bne-    1b\n"
 272         PPC_ATOMIC_EXIT_BARRIER
 273         "\n\
 274 2:"
 275         : "=&r" (t1), "=&r" (t2)
 276         : "r" (&v->counter)
 277         : "cc", "xer", "memory");
 278
 279         return t1;
 280 }
 281 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v))
 282
 283 #define atomic_sub_and_test(a, v)       (atomic_sub_return((a), (v)) == 0)
 284 #define atomic_dec_and_test(v)          (atomic_dec_return((v)) == 0)
 285
 286 /*
 287  * Atomically test *v and decrement if it is greater than 0.
 288  * The function returns the old value of *v minus 1, even if
 289  * the atomic variable, v, was not decremented.
 290  */
 291 static __inline__ int atomic_dec_if_positive(atomic_t *v)
 292 {
 293         int t;
 294
 295         __asm__ __volatile__(
 296         PPC_ATOMIC_ENTRY_BARRIER
 297 "1:     lwarx   %0,0,%1         # atomic_dec_if_positive\n\
 298         cmpwi   %0,1\n\
 299         addi    %0,%0,-1\n\
 300         blt-    2f\n"
 301         PPC405_ERR77(0,%1)
 302 "       stwcx.  %0,0,%1\n\
 303         bne-    1b"
 304         PPC_ATOMIC_EXIT_BARRIER
 305         "\n\
 306 2:"     : "=&b" (t)
 307         : "r" (&v->counter)
 308         : "cc", "memory");
 309
 310         return t;
 311 }
 312 #define atomic_dec_if_positive atomic_dec_if_positive
 313
 314 #ifdef __powerpc64__
 315
 316 #define ATOMIC64_INIT(i)        { (i) }
 317
 318 static __inline__ long atomic64_read(const atomic64_t *v)
 319 {
 320         long t;
 321
 322         __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
 323
 324         return t;
 325 }
 326
 327 static __inline__ void atomic64_set(atomic64_t *v, long i)
 328 {
 329         __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
 330 }
 331
 332 #define ATOMIC64_OP(op, asm_op)                                         \
 333 static __inline__ void atomic64_##op(long a, atomic64_t *v)             \
 334 {                                                                       \
 335         long t;                                                         \
 336                                                                         \
 337         __asm__ __volatile__(                                           \
 338 "1:     ldarx   %0,0,%3         # atomic64_" #op "\n"                   \
 339         #asm_op " %0,%2,%0\n"                                           \
 340 "       stdcx.  %0,0,%3 \n"                                             \
 341 "       bne-    1b\n"                                                   \
 342         : "=&r" (t), "+m" (v->counter)                                  \
 343         : "r" (a), "r" (&v->counter)                                    \
 344         : "cc");                                                        \
 345 }
 346
 347 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 348 static inline long                                                      \
 349 atomic64_##op##_return_relaxed(long a, atomic64_t *v)                   \
 350 {                                                                       \
 351         long t;                                                         \
 352                                                                         \
 353         __asm__ __volatile__(                                           \
 354 "1:     ldarx   %0,0,%3         # atomic64_" #op "_return_relaxed\n"    \
 355         #asm_op " %0,%2,%0\n"                                           \
 356 "       stdcx.  %0,0,%3\n"                                              \
 357 "       bne-    1b\n"                                                   \
 358         : "=&r" (t), "+m" (v->counter)                                  \
 359         : "r" (a), "r" (&v->counter)                                    \
 360         : "cc");                                                        \
 361                                                                         \
 362         return t;                                                       \
 363 }
 364
 365 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op)                           \
 366 static inline long                                                      \
 367 atomic64_fetch_##op##_relaxed(long a, atomic64_t *v)                    \
 368 {                                                                       \
 369         long res, t;                                                    \
 370                                                                         \
 371         __asm__ __volatile__(                                           \
 372 "1:     ldarx   %0,0,%4         # atomic64_fetch_" #op "_relaxed\n"     \
 373         #asm_op " %1,%3,%0\n"                                           \
 374 "       stdcx.  %1,0,%4\n"                                              \
 375 "       bne-    1b\n"                                                   \
 376         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
 377         : "r" (a), "r" (&v->counter)                                    \
 378         : "cc");                                                        \
 379                                                                         \
 380         return res;                                                     \
 381 }
 382
 383 #define ATOMIC64_OPS(op, asm_op)                                        \
 384         ATOMIC64_OP(op, asm_op)                                         \
 385         ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 386         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 387
 388 ATOMIC64_OPS(add, add)
 389 ATOMIC64_OPS(sub, subf)
 390
 391 #define atomic64_add_return_relaxed atomic64_add_return_relaxed
 392 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
 393
 394 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
 395 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
 396
 397 #undef ATOMIC64_OPS
 398 #define ATOMIC64_OPS(op, asm_op)                                        \
 399         ATOMIC64_OP(op, asm_op)                                         \
 400         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 401
 402 ATOMIC64_OPS(and, and)
 403 ATOMIC64_OPS(or, or)
 404 ATOMIC64_OPS(xor, xor)
 405
 406 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
 407 #define atomic64_fetch_or_relaxed  atomic64_fetch_or_relaxed
 408 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
 409
 410 #undef ATOPIC64_OPS
 411 #undef ATOMIC64_FETCH_OP_RELAXED
 412 #undef ATOMIC64_OP_RETURN_RELAXED
 413 #undef ATOMIC64_OP
 414
 415 #define atomic64_add_negative(a, v)     (atomic64_add_return((a), (v)) < 0)
 416
 417 static __inline__ void atomic64_inc(atomic64_t *v)
 418 {
 419         long t;
 420
 421         __asm__ __volatile__(
 422 "1:     ldarx   %0,0,%2         # atomic64_inc\n\
 423         addic   %0,%0,1\n\
 424         stdcx.  %0,0,%2 \n\
 425         bne-    1b"
 426         : "=&r" (t), "+m" (v->counter)
 427         : "r" (&v->counter)
 428         : "cc", "xer");
 429 }
 430
 431 static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v)
 432 {
 433         long t;
 434
 435         __asm__ __volatile__(
 436 "1:     ldarx   %0,0,%2         # atomic64_inc_return_relaxed\n"
 437 "       addic   %0,%0,1\n"
 438 "       stdcx.  %0,0,%2\n"
 439 "       bne-    1b"
 440         : "=&r" (t), "+m" (v->counter)
 441         : "r" (&v->counter)
 442         : "cc", "xer");
 443
 444         return t;
 445 }
 446
 447 /*
 448  * atomic64_inc_and_test - increment and test
 449  * @v: pointer of type atomic64_t
 450  *
 451  * Atomically increments @v by 1
 452  * and returns true if the result is zero, or false for all
 453  * other cases.
 454  */
 455 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
 456
 457 static __inline__ void atomic64_dec(atomic64_t *v)
 458 {
 459         long t;
 460
 461         __asm__ __volatile__(
 462 "1:     ldarx   %0,0,%2         # atomic64_dec\n\
 463         addic   %0,%0,-1\n\
 464         stdcx.  %0,0,%2\n\
 465         bne-    1b"
 466         : "=&r" (t), "+m" (v->counter)
 467         : "r" (&v->counter)
 468         : "cc", "xer");
 469 }
 470
 471 static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v)
 472 {
 473         long t;
 474
 475         __asm__ __volatile__(
 476 "1:     ldarx   %0,0,%2         # atomic64_dec_return_relaxed\n"
 477 "       addic   %0,%0,-1\n"
 478 "       stdcx.  %0,0,%2\n"
 479 "       bne-    1b"
 480         : "=&r" (t), "+m" (v->counter)
 481         : "r" (&v->counter)
 482         : "cc", "xer");
 483
 484         return t;
 485 }
 486
 487 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
 488 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
 489
 490 #define atomic64_sub_and_test(a, v)     (atomic64_sub_return((a), (v)) == 0)
 491 #define atomic64_dec_and_test(v)        (atomic64_dec_return((v)) == 0)
 492
 493 /*
 494  * Atomically test *v and decrement if it is greater than 0.
 495  * The function returns the old value of *v minus 1.
 496  */
 497 static __inline__ long atomic64_dec_if_positive(atomic64_t *v)
 498 {
 499         long t;
 500
 501         __asm__ __volatile__(
 502         PPC_ATOMIC_ENTRY_BARRIER
 503 "1:     ldarx   %0,0,%1         # atomic64_dec_if_positive\n\
 504         addic.  %0,%0,-1\n\
 505         blt-    2f\n\
 506         stdcx.  %0,0,%1\n\
 507         bne-    1b"
 508         PPC_ATOMIC_EXIT_BARRIER
 509         "\n\
 510 2:"     : "=&r" (t)
 511         : "r" (&v->counter)
 512         : "cc", "xer", "memory");
 513
 514         return t;
 515 }
 516
 517 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 518 #define atomic64_cmpxchg_relaxed(v, o, n) \
 519         cmpxchg_relaxed(&((v)->counter), (o), (n))
 520 #define atomic64_cmpxchg_acquire(v, o, n) \
 521         cmpxchg_acquire(&((v)->counter), (o), (n))
 522
 523 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 524 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
 525
 526 /**
 527  * atomic64_add_unless - add unless the number is a given value
 528  * @v: pointer of type atomic64_t
 529  * @a: the amount to add to v...
 530  * @u: ...unless v is equal to u.
 531  *
 532  * Atomically adds @a to @v, so long as it was not @u.
 533  * Returns the old value of @v.
 534  */
 535 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
 536 {
 537         long t;
 538
 539         __asm__ __volatile__ (
 540         PPC_ATOMIC_ENTRY_BARRIER
 541 "1:     ldarx   %0,0,%1         # __atomic_add_unless\n\
 542         cmpd    0,%0,%3 \n\
 543         beq     2f \n\
 544         add     %0,%2,%0 \n"
 545 "       stdcx.  %0,0,%1 \n\
 546         bne-    1b \n"
 547         PPC_ATOMIC_EXIT_BARRIER
 548 "       subf    %0,%2,%0 \n\
 549 2:"
 550         : "=&r" (t)
 551         : "r" (&v->counter), "r" (a), "r" (u)
 552         : "cc", "memory");
 553
 554         return t != u;
 555 }
 556
 557 /**
 558  * atomic_inc64_not_zero - increment unless the number is zero
 559  * @v: pointer of type atomic64_t
 560  *
 561  * Atomically increments @v by 1, so long as @v is non-zero.
 562  * Returns non-zero if @v was non-zero, and zero otherwise.
 563  */
 564 static __inline__ int atomic64_inc_not_zero(atomic64_t *v)
 565 {
 566         long t1, t2;
 567
 568         __asm__ __volatile__ (
 569         PPC_ATOMIC_ENTRY_BARRIER
 570 "1:     ldarx   %0,0,%2         # atomic64_inc_not_zero\n\
 571         cmpdi   0,%0,0\n\
 572         beq-    2f\n\
 573         addic   %1,%0,1\n\
 574         stdcx.  %1,0,%2\n\
 575         bne-    1b\n"
 576         PPC_ATOMIC_EXIT_BARRIER
 577         "\n\
 578 2:"
 579         : "=&r" (t1), "=&r" (t2)
 580         : "r" (&v->counter)
 581         : "cc", "xer", "memory");
 582
 583         return t1 != 0;
 584 }
 585
 586 #endif /* __powerpc64__ */
 587
 588 #endif /* __KERNEL__ */
 589 #endif /* _ASM_POWERPC_ATOMIC_H_ */