arch/mips/include/asm/atomic.h

   1 /*
   2  * Atomic operations that C can't guarantee us.  Useful for
   3  * resource counting etc..
   4  *
   5  * But use these as seldom as possible since they are much more slower
   6  * than regular operations.
   7  *
   8  * This file is subject to the terms and conditions of the GNU General Public
   9  * License.  See the file "COPYING" in the main directory of this archive
  10  * for more details.
  11  *
  12  * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
  13  */
  14 #ifndef _ASM_ATOMIC_H
  15 #define _ASM_ATOMIC_H
  16
  17 #include <linux/irqflags.h>
  18 #include <linux/types.h>
  19 #include <asm/barrier.h>
  20 #include <asm/cpu-features.h>
  21 #include <asm/cmpxchg.h>
  22 #include <asm/war.h>
  23
  24 #define ATOMIC_INIT(i)    { (i) }
  25
  26 /*
  27  * atomic_read - read atomic variable
  28  * @v: pointer of type atomic_t
  29  *
  30  * Atomically reads the value of @v.
  31  */
  32 #define atomic_read(v)          ACCESS_ONCE((v)->counter)
  33
  34 /*
  35  * atomic_set - set atomic variable
  36  * @v: pointer of type atomic_t
  37  * @i: required value
  38  *
  39  * Atomically sets the value of @v to @i.
  40  */
  41 #define atomic_set(v, i)                ((v)->counter = (i))
  42
  43 #define ATOMIC_OP(op, c_op, asm_op)                                             \
  44 static __inline__ void atomic_##op(int i, atomic_t * v)                         \
  45 {                                                                               \
  46         if (kernel_uses_llsc && R10000_LLSC_WAR) {                              \
  47                 int temp;                                                       \
  48                                                                                 \
  49                 __asm__ __volatile__(                                           \
  50                 "       .set    arch=r4000                              \n"     \
  51                 "1:     ll      %0, %1          # atomic_" #op "        \n"     \
  52                 "       " #asm_op " %0, %2                              \n"     \
  53                 "       sc      %0, %1                                  \n"     \
  54                 "       beqzl   %0, 1b                                  \n"     \
  55                 "       .set    mips0                                   \n"     \
  56                 : "=&r" (temp), "+m" (v->counter)                               \
  57                 : "Ir" (i));                                                    \
  58         } else if (kernel_uses_llsc) {                                          \
  59                 int temp;                                                       \
  60                                                                                 \
  61                 do {                                                            \
  62                         __asm__ __volatile__(                                   \
  63                         "       .set    arch=r4000                      \n"     \
  64                         "       ll      %0, %1          # atomic_" #op "\n"     \
  65                         "       " #asm_op " %0, %2                      \n"     \
  66                         "       sc      %0, %1                          \n"     \
  67                         "       .set    mips0                           \n"     \
  68                         : "=&r" (temp), "+m" (v->counter)                       \
  69                         : "Ir" (i));                                            \
  70                 } while (unlikely(!temp));                                      \
  71         } else {                                                                \
  72                 unsigned long flags;                                            \
  73                                                                                 \
  74                 raw_local_irq_save(flags);                                      \
  75                 v->counter c_op i;                                              \
  76                 raw_local_irq_restore(flags);                                   \
  77         }                                                                       \
  78 }                                                                               \
  79
  80 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                                      \
  81 static __inline__ int atomic_##op##_return(int i, atomic_t * v)                 \
  82 {                                                                               \
  83         int result;                                                             \
  84                                                                                 \
  85         smp_mb__before_llsc();                                                  \
  86                                                                                 \
  87         if (kernel_uses_llsc && R10000_LLSC_WAR) {                              \
  88                 int temp;                                                       \
  89                                                                                 \
  90                 __asm__ __volatile__(                                           \
  91                 "       .set    arch=r4000                              \n"     \
  92                 "1:     ll      %1, %2          # atomic_" #op "_return \n"     \
  93                 "       " #asm_op " %0, %1, %3                          \n"     \
  94                 "       sc      %0, %2                                  \n"     \
  95                 "       beqzl   %0, 1b                                  \n"     \
  96                 "       " #asm_op " %0, %1, %3                          \n"     \
  97                 "       .set    mips0                                   \n"     \
  98                 : "=&r" (result), "=&r" (temp), "+m" (v->counter)               \
  99                 : "Ir" (i));                                                    \
 100         } else if (kernel_uses_llsc) {                                          \
 101                 int temp;                                                       \
 102                                                                                 \
 103                 do {                                                            \
 104                         __asm__ __volatile__(                                   \
 105                         "       .set    arch=r4000                      \n"     \
 106                         "       ll      %1, %2  # atomic_" #op "_return \n"     \
 107                         "       " #asm_op " %0, %1, %3                  \n"     \
 108                         "       sc      %0, %2                          \n"     \
 109                         "       .set    mips0                           \n"     \
 110                         : "=&r" (result), "=&r" (temp), "+m" (v->counter)       \
 111                         : "Ir" (i));                                            \
 112                 } while (unlikely(!result));                                    \
 113                                                                                 \
 114                 result = temp; result c_op i;                                   \
 115         } else {                                                                \
 116                 unsigned long flags;                                            \
 117                                                                                 \
 118                 raw_local_irq_save(flags);                                      \
 119                 result = v->counter;                                            \
 120                 result c_op i;                                                  \
 121                 v->counter = result;                                            \
 122                 raw_local_irq_restore(flags);                                   \
 123         }                                                                       \
 124                                                                                 \
 125         smp_llsc_mb();                                                          \
 126                                                                                 \
 127         return result;                                                          \
 128 }
 129
 130 #define ATOMIC_OPS(op, c_op, asm_op)                                            \
 131         ATOMIC_OP(op, c_op, asm_op)                                             \
 132         ATOMIC_OP_RETURN(op, c_op, asm_op)
 133
 134 ATOMIC_OPS(add, +=, addu)
 135 ATOMIC_OPS(sub, -=, subu)
 136
 137 #undef ATOMIC_OPS
 138 #undef ATOMIC_OP_RETURN
 139 #undef ATOMIC_OP
 140
 141 /*
 142  * atomic_sub_if_positive - conditionally subtract integer from atomic variable
 143  * @i: integer value to subtract
 144  * @v: pointer of type atomic_t
 145  *
 146  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 147  * The function returns the old value of @v minus @i.
 148  */
 149 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 150 {
 151         int result;
 152
 153         smp_mb__before_llsc();
 154
 155         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 156                 int temp;
 157
 158                 __asm__ __volatile__(
 159                 "       .set    arch=r4000                              \n"
 160                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
 161                 "       subu    %0, %1, %3                              \n"
 162                 "       bltz    %0, 1f                                  \n"
 163                 "       sc      %0, %2                                  \n"
 164                 "       .set    noreorder                               \n"
 165                 "       beqzl   %0, 1b                                  \n"
 166                 "        subu   %0, %1, %3                              \n"
 167                 "       .set    reorder                                 \n"
 168                 "1:                                                     \n"
 169                 "       .set    mips0                                   \n"
 170                 : "=&r" (result), "=&r" (temp), "+m" (v->counter)
 171                 : "Ir" (i), "m" (v->counter)
 172                 : "memory");
 173         } else if (kernel_uses_llsc) {
 174                 int temp;
 175
 176                 __asm__ __volatile__(
 177                 "       .set    arch=r4000                              \n"
 178                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
 179                 "       subu    %0, %1, %3                              \n"
 180                 "       bltz    %0, 1f                                  \n"
 181                 "       sc      %0, %2                                  \n"
 182                 "       .set    noreorder                               \n"
 183                 "       beqz    %0, 1b                                  \n"
 184                 "        subu   %0, %1, %3                              \n"
 185                 "       .set    reorder                                 \n"
 186                 "1:                                                     \n"
 187                 "       .set    mips0                                   \n"
 188                 : "=&r" (result), "=&r" (temp), "+m" (v->counter)
 189                 : "Ir" (i));
 190         } else {
 191                 unsigned long flags;
 192
 193                 raw_local_irq_save(flags);
 194                 result = v->counter;
 195                 result -= i;
 196                 if (result >= 0)
 197                         v->counter = result;
 198                 raw_local_irq_restore(flags);
 199         }
 200
 201         smp_llsc_mb();
 202
 203         return result;
 204 }
 205
 206 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 207 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
 208
 209 /**
 210  * __atomic_add_unless - add unless the number is a given value
 211  * @v: pointer of type atomic_t
 212  * @a: the amount to add to v...
 213  * @u: ...unless v is equal to u.
 214  *
 215  * Atomically adds @a to @v, so long as it was not @u.
 216  * Returns the old value of @v.
 217  */
 218 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
 219 {
 220         int c, old;
 221         c = atomic_read(v);
 222         for (;;) {
 223                 if (unlikely(c == (u)))
 224                         break;
 225                 old = atomic_cmpxchg((v), c, c + (a));
 226                 if (likely(old == c))
 227                         break;
 228                 c = old;
 229         }
 230         return c;
 231 }
 232
 233 #define atomic_dec_return(v) atomic_sub_return(1, (v))
 234 #define atomic_inc_return(v) atomic_add_return(1, (v))
 235
 236 /*
 237  * atomic_sub_and_test - subtract value from variable and test result
 238  * @i: integer value to subtract
 239  * @v: pointer of type atomic_t
 240  *
 241  * Atomically subtracts @i from @v and returns
 242  * true if the result is zero, or false for all
 243  * other cases.
 244  */
 245 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
 246
 247 /*
 248  * atomic_inc_and_test - increment and test
 249  * @v: pointer of type atomic_t
 250  *
 251  * Atomically increments @v by 1
 252  * and returns true if the result is zero, or false for all
 253  * other cases.
 254  */
 255 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
 256
 257 /*
 258  * atomic_dec_and_test - decrement by 1 and test
 259  * @v: pointer of type atomic_t
 260  *
 261  * Atomically decrements @v by 1 and
 262  * returns true if the result is 0, or false for all other
 263  * cases.
 264  */
 265 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
 266
 267 /*
 268  * atomic_dec_if_positive - decrement by 1 if old value positive
 269  * @v: pointer of type atomic_t
 270  */
 271 #define atomic_dec_if_positive(v)       atomic_sub_if_positive(1, v)
 272
 273 /*
 274  * atomic_inc - increment atomic variable
 275  * @v: pointer of type atomic_t
 276  *
 277  * Atomically increments @v by 1.
 278  */
 279 #define atomic_inc(v) atomic_add(1, (v))
 280
 281 /*
 282  * atomic_dec - decrement and test
 283  * @v: pointer of type atomic_t
 284  *
 285  * Atomically decrements @v by 1.
 286  */
 287 #define atomic_dec(v) atomic_sub(1, (v))
 288
 289 /*
 290  * atomic_add_negative - add and test if negative
 291  * @v: pointer of type atomic_t
 292  * @i: integer value to add
 293  *
 294  * Atomically adds @i to @v and returns true
 295  * if the result is negative, or false when
 296  * result is greater than or equal to zero.
 297  */
 298 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0)
 299
 300 #ifdef CONFIG_64BIT
 301
 302 #define ATOMIC64_INIT(i)    { (i) }
 303
 304 /*
 305  * atomic64_read - read atomic variable
 306  * @v: pointer of type atomic64_t
 307  *
 308  */
 309 #define atomic64_read(v)        ACCESS_ONCE((v)->counter)
 310
 311 /*
 312  * atomic64_set - set atomic variable
 313  * @v: pointer of type atomic64_t
 314  * @i: required value
 315  */
 316 #define atomic64_set(v, i)      ((v)->counter = (i))
 317
 318 #define ATOMIC64_OP(op, c_op, asm_op)                                           \
 319 static __inline__ void atomic64_##op(long i, atomic64_t * v)                    \
 320 {                                                                               \
 321         if (kernel_uses_llsc && R10000_LLSC_WAR) {                              \
 322                 long temp;                                                      \
 323                                                                                 \
 324                 __asm__ __volatile__(                                           \
 325                 "       .set    arch=r4000                              \n"     \
 326                 "1:     lld     %0, %1          # atomic64_" #op "      \n"     \
 327                 "       " #asm_op " %0, %2                              \n"     \
 328                 "       scd     %0, %1                                  \n"     \
 329                 "       beqzl   %0, 1b                                  \n"     \
 330                 "       .set    mips0                                   \n"     \
 331                 : "=&r" (temp), "+m" (v->counter)                               \
 332                 : "Ir" (i));                                                    \
 333         } else if (kernel_uses_llsc) {                                          \
 334                 long temp;                                                      \
 335                                                                                 \
 336                 do {                                                            \
 337                         __asm__ __volatile__(                                   \
 338                         "       .set    arch=r4000                      \n"     \
 339                         "       lld     %0, %1          # atomic64_" #op "\n"   \
 340                         "       " #asm_op " %0, %2                      \n"     \
 341                         "       scd     %0, %1                          \n"     \
 342                         "       .set    mips0                           \n"     \
 343                         : "=&r" (temp), "+m" (v->counter)                       \
 344                         : "Ir" (i));                                            \
 345                 } while (unlikely(!temp));                                      \
 346         } else {                                                                \
 347                 unsigned long flags;                                            \
 348                                                                                 \
 349                 raw_local_irq_save(flags);                                      \
 350                 v->counter c_op i;                                              \
 351                 raw_local_irq_restore(flags);                                   \
 352         }                                                                       \
 353 }                                                                               \
 354
 355 #define ATOMIC64_OP_RETURN(op, c_op, asm_op)                                    \
 356 static __inline__ long atomic64_##op##_return(long i, atomic64_t * v)           \
 357 {                                                                               \
 358         long result;                                                            \
 359                                                                                 \
 360         smp_mb__before_llsc();                                                  \
 361                                                                                 \
 362         if (kernel_uses_llsc && R10000_LLSC_WAR) {                              \
 363                 long temp;                                                      \
 364                                                                                 \
 365                 __asm__ __volatile__(                                           \
 366                 "       .set    arch=r4000                              \n"     \
 367                 "1:     lld     %1, %2          # atomic64_" #op "_return\n"    \
 368                 "       " #asm_op " %0, %1, %3                          \n"     \
 369                 "       scd     %0, %2                                  \n"     \
 370                 "       beqzl   %0, 1b                                  \n"     \
 371                 "       " #asm_op " %0, %1, %3                          \n"     \
 372                 "       .set    mips0                                   \n"     \
 373                 : "=&r" (result), "=&r" (temp), "+m" (v->counter)               \
 374                 : "Ir" (i));                                                    \
 375         } else if (kernel_uses_llsc) {                                          \
 376                 long temp;                                                      \
 377                                                                                 \
 378                 do {                                                            \
 379                         __asm__ __volatile__(                                   \
 380                         "       .set    arch=r4000                      \n"     \
 381                         "       lld     %1, %2  # atomic64_" #op "_return\n"    \
 382                         "       " #asm_op " %0, %1, %3                  \n"     \
 383                         "       scd     %0, %2                          \n"     \
 384                         "       .set    mips0                           \n"     \
 385                         : "=&r" (result), "=&r" (temp), "=m" (v->counter)       \
 386                         : "Ir" (i), "m" (v->counter)                            \
 387                         : "memory");                                            \
 388                 } while (unlikely(!result));                                    \
 389                                                                                 \
 390                 result = temp; result c_op i;                                   \
 391         } else {                                                                \
 392                 unsigned long flags;                                            \
 393                                                                                 \
 394                 raw_local_irq_save(flags);                                      \
 395                 result = v->counter;                                            \
 396                 result c_op i;                                                  \
 397                 v->counter = result;                                            \
 398                 raw_local_irq_restore(flags);                                   \
 399         }                                                                       \
 400                                                                                 \
 401         smp_llsc_mb();                                                          \
 402                                                                                 \
 403         return result;                                                          \
 404 }
 405
 406 #define ATOMIC64_OPS(op, c_op, asm_op)                                          \
 407         ATOMIC64_OP(op, c_op, asm_op)                                           \
 408         ATOMIC64_OP_RETURN(op, c_op, asm_op)
 409
 410 ATOMIC64_OPS(add, +=, daddu)
 411 ATOMIC64_OPS(sub, -=, dsubu)
 412
 413 #undef ATOMIC64_OPS
 414 #undef ATOMIC64_OP_RETURN
 415 #undef ATOMIC64_OP
 416
 417 /*
 418  * atomic64_sub_if_positive - conditionally subtract integer from atomic variable
 419  * @i: integer value to subtract
 420  * @v: pointer of type atomic64_t
 421  *
 422  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 423  * The function returns the old value of @v minus @i.
 424  */
 425 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
 426 {
 427         long result;
 428
 429         smp_mb__before_llsc();
 430
 431         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 432                 long temp;
 433
 434                 __asm__ __volatile__(
 435                 "       .set    arch=r4000                              \n"
 436                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
 437                 "       dsubu   %0, %1, %3                              \n"
 438                 "       bltz    %0, 1f                                  \n"
 439                 "       scd     %0, %2                                  \n"
 440                 "       .set    noreorder                               \n"
 441                 "       beqzl   %0, 1b                                  \n"
 442                 "        dsubu  %0, %1, %3                              \n"
 443                 "       .set    reorder                                 \n"
 444                 "1:                                                     \n"
 445                 "       .set    mips0                                   \n"
 446                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
 447                 : "Ir" (i), "m" (v->counter)
 448                 : "memory");
 449         } else if (kernel_uses_llsc) {
 450                 long temp;
 451
 452                 __asm__ __volatile__(
 453                 "       .set    arch=r4000                              \n"
 454                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
 455                 "       dsubu   %0, %1, %3                              \n"
 456                 "       bltz    %0, 1f                                  \n"
 457                 "       scd     %0, %2                                  \n"
 458                 "       .set    noreorder                               \n"
 459                 "       beqz    %0, 1b                                  \n"
 460                 "        dsubu  %0, %1, %3                              \n"
 461                 "       .set    reorder                                 \n"
 462                 "1:                                                     \n"
 463                 "       .set    mips0                                   \n"
 464                 : "=&r" (result), "=&r" (temp), "+m" (v->counter)
 465                 : "Ir" (i));
 466         } else {
 467                 unsigned long flags;
 468
 469                 raw_local_irq_save(flags);
 470                 result = v->counter;
 471                 result -= i;
 472                 if (result >= 0)
 473                         v->counter = result;
 474                 raw_local_irq_restore(flags);
 475         }
 476
 477         smp_llsc_mb();
 478
 479         return result;
 480 }
 481
 482 #define atomic64_cmpxchg(v, o, n) \
 483         ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
 484 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
 485
 486 /**
 487  * atomic64_add_unless - add unless the number is a given value
 488  * @v: pointer of type atomic64_t
 489  * @a: the amount to add to v...
 490  * @u: ...unless v is equal to u.
 491  *
 492  * Atomically adds @a to @v, so long as it was not @u.
 493  * Returns the old value of @v.
 494  */
 495 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
 496 {
 497         long c, old;
 498         c = atomic64_read(v);
 499         for (;;) {
 500                 if (unlikely(c == (u)))
 501                         break;
 502                 old = atomic64_cmpxchg((v), c, c + (a));
 503                 if (likely(old == c))
 504                         break;
 505                 c = old;
 506         }
 507         return c != (u);
 508 }
 509
 510 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
 511
 512 #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
 513 #define atomic64_inc_return(v) atomic64_add_return(1, (v))
 514
 515 /*
 516  * atomic64_sub_and_test - subtract value from variable and test result
 517  * @i: integer value to subtract
 518  * @v: pointer of type atomic64_t
 519  *
 520  * Atomically subtracts @i from @v and returns
 521  * true if the result is zero, or false for all
 522  * other cases.
 523  */
 524 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
 525
 526 /*
 527  * atomic64_inc_and_test - increment and test
 528  * @v: pointer of type atomic64_t
 529  *
 530  * Atomically increments @v by 1
 531  * and returns true if the result is zero, or false for all
 532  * other cases.
 533  */
 534 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
 535
 536 /*
 537  * atomic64_dec_and_test - decrement by 1 and test
 538  * @v: pointer of type atomic64_t
 539  *
 540  * Atomically decrements @v by 1 and
 541  * returns true if the result is 0, or false for all other
 542  * cases.
 543  */
 544 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
 545
 546 /*
 547  * atomic64_dec_if_positive - decrement by 1 if old value positive
 548  * @v: pointer of type atomic64_t
 549  */
 550 #define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(1, v)
 551
 552 /*
 553  * atomic64_inc - increment atomic variable
 554  * @v: pointer of type atomic64_t
 555  *
 556  * Atomically increments @v by 1.
 557  */
 558 #define atomic64_inc(v) atomic64_add(1, (v))
 559
 560 /*
 561  * atomic64_dec - decrement and test
 562  * @v: pointer of type atomic64_t
 563  *
 564  * Atomically decrements @v by 1.
 565  */
 566 #define atomic64_dec(v) atomic64_sub(1, (v))
 567
 568 /*
 569  * atomic64_add_negative - add and test if negative
 570  * @v: pointer of type atomic64_t
 571  * @i: integer value to add
 572  *
 573  * Atomically adds @i to @v and returns true
 574  * if the result is negative, or false when
 575  * result is greater than or equal to zero.
 576  */
 577 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0)
 578
 579 #endif /* CONFIG_64BIT */
 580
 581 #endif /* _ASM_ATOMIC_H */