arch/arc/include/asm/atomic.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef _ASM_ARC_ATOMIC_H
  10 #define _ASM_ARC_ATOMIC_H
  11
  12 #ifndef __ASSEMBLY__
  13
  14 #include <linux/types.h>
  15 #include <linux/compiler.h>
  16 #include <asm/cmpxchg.h>
  17 #include <asm/barrier.h>
  18 #include <asm/smp.h>
  19
  20 #define ATOMIC_INIT(i)  { (i) }
  21
  22 #ifndef CONFIG_ARC_PLAT_EZNPS
  23
  24 #define atomic_read(v)  READ_ONCE((v)->counter)
  25
  26 #ifdef CONFIG_ARC_HAS_LLSC
  27
  28 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  29
  30 #define ATOMIC_OP(op, c_op, asm_op)                                     \
  31 static inline void atomic_##op(int i, atomic_t *v)                      \
  32 {                                                                       \
  33         unsigned int val;                                               \
  34                                                                         \
  35         __asm__ __volatile__(                                           \
  36         "1:     llock   %[val], [%[ctr]]                \n"             \
  37         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  38         "       scond   %[val], [%[ctr]]                \n"             \
  39         "       bnz     1b                              \n"             \
  40         : [val] "=&r"   (val) /* Early clobber to prevent reg reuse */  \
  41         : [ctr] "r"     (&v->counter), /* Not "m": llock only supports reg direct addr mode */  \
  42           [i]   "ir"    (i)                                             \
  43         : "cc");                                                        \
  44 }                                                                       \
  45
  46 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
  47 static inline int atomic_##op##_return(int i, atomic_t *v)              \
  48 {                                                                       \
  49         unsigned int val;                                               \
  50                                                                         \
  51         /*                                                              \
  52          * Explicit full memory barrier needed before/after as          \
  53          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  54          */                                                             \
  55         smp_mb();                                                       \
  56                                                                         \
  57         __asm__ __volatile__(                                           \
  58         "1:     llock   %[val], [%[ctr]]                \n"             \
  59         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  60         "       scond   %[val], [%[ctr]]                \n"             \
  61         "       bnz     1b                              \n"             \
  62         : [val] "=&r"   (val)                                           \
  63         : [ctr] "r"     (&v->counter),                                  \
  64           [i]   "ir"    (i)                                             \
  65         : "cc");                                                        \
  66                                                                         \
  67         smp_mb();                                                       \
  68                                                                         \
  69         return val;                                                     \
  70 }
  71
  72 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
  73 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
  74 {                                                                       \
  75         unsigned int val, orig;                                         \
  76                                                                         \
  77         /*                                                              \
  78          * Explicit full memory barrier needed before/after as          \
  79          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  80          */                                                             \
  81         smp_mb();                                                       \
  82                                                                         \
  83         __asm__ __volatile__(                                           \
  84         "1:     llock   %[orig], [%[ctr]]               \n"             \
  85         "       " #asm_op " %[val], %[orig], %[i]       \n"             \
  86         "       scond   %[val], [%[ctr]]                \n"             \
  87         "                                               \n"             \
  88         : [val] "=&r"   (val),                                          \
  89           [orig] "=&r" (orig)                                           \
  90         : [ctr] "r"     (&v->counter),                                  \
  91           [i]   "ir"    (i)                                             \
  92         : "cc");                                                        \
  93                                                                         \
  94         smp_mb();                                                       \
  95                                                                         \
  96         return orig;                                                    \
  97 }
  98
  99 #else   /* !CONFIG_ARC_HAS_LLSC */
 100
 101 #ifndef CONFIG_SMP
 102
 103  /* violating atomic_xxx API locking protocol in UP for optimization sake */
 104 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
 105
 106 #else
 107
 108 static inline void atomic_set(atomic_t *v, int i)
 109 {
 110         /*
 111          * Independent of hardware support, all of the atomic_xxx() APIs need
 112          * to follow the same locking rules to make sure that a "hardware"
 113          * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
 114          * sequence
 115          *
 116          * Thus atomic_set() despite being 1 insn (and seemingly atomic)
 117          * requires the locking.
 118          */
 119         unsigned long flags;
 120
 121         atomic_ops_lock(flags);
 122         WRITE_ONCE(v->counter, i);
 123         atomic_ops_unlock(flags);
 124 }
 125
 126 #endif
 127
 128 /*
 129  * Non hardware assisted Atomic-R-M-W
 130  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 131  */
 132
 133 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 134 static inline void atomic_##op(int i, atomic_t *v)                      \
 135 {                                                                       \
 136         unsigned long flags;                                            \
 137                                                                         \
 138         atomic_ops_lock(flags);                                         \
 139         v->counter c_op i;                                              \
 140         atomic_ops_unlock(flags);                                       \
 141 }
 142
 143 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 144 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 145 {                                                                       \
 146         unsigned long flags;                                            \
 147         unsigned long temp;                                             \
 148                                                                         \
 149         /*                                                              \
 150          * spin lock/unlock provides the needed smp_mb() before/after   \
 151          */                                                             \
 152         atomic_ops_lock(flags);                                         \
 153         temp = v->counter;                                              \
 154         temp c_op i;                                                    \
 155         v->counter = temp;                                              \
 156         atomic_ops_unlock(flags);                                       \
 157                                                                         \
 158         return temp;                                                    \
 159 }
 160
 161 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 162 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 163 {                                                                       \
 164         unsigned long flags;                                            \
 165         unsigned long orig;                                             \
 166                                                                         \
 167         /*                                                              \
 168          * spin lock/unlock provides the needed smp_mb() before/after   \
 169          */                                                             \
 170         atomic_ops_lock(flags);                                         \
 171         orig = v->counter;                                              \
 172         v->counter c_op i;                                              \
 173         atomic_ops_unlock(flags);                                       \
 174                                                                         \
 175         return orig;                                                    \
 176 }
 177
 178 #endif /* !CONFIG_ARC_HAS_LLSC */
 179
 180 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 181         ATOMIC_OP(op, c_op, asm_op)                                     \
 182         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 183         ATOMIC_FETCH_OP(op, c_op, asm_op)
 184
 185 ATOMIC_OPS(add, +=, add)
 186 ATOMIC_OPS(sub, -=, sub)
 187
 188 #define atomic_andnot atomic_andnot
 189
 190 #undef ATOMIC_OPS
 191 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 192         ATOMIC_OP(op, c_op, asm_op)                                     \
 193         ATOMIC_FETCH_OP(op, c_op, asm_op)
 194
 195 ATOMIC_OPS(and, &=, and)
 196 ATOMIC_OPS(andnot, &= ~, bic)
 197 ATOMIC_OPS(or, |=, or)
 198 ATOMIC_OPS(xor, ^=, xor)
 199
 200 #else /* CONFIG_ARC_PLAT_EZNPS */
 201
 202 static inline int atomic_read(const atomic_t *v)
 203 {
 204         int temp;
 205
 206         __asm__ __volatile__(
 207         "       ld.di %0, [%1]"
 208         : "=r"(temp)
 209         : "r"(&v->counter)
 210         : "memory");
 211         return temp;
 212 }
 213
 214 static inline void atomic_set(atomic_t *v, int i)
 215 {
 216         __asm__ __volatile__(
 217         "       st.di %0,[%1]"
 218         :
 219         : "r"(i), "r"(&v->counter)
 220         : "memory");
 221 }
 222
 223 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 224 static inline void atomic_##op(int i, atomic_t *v)                      \
 225 {                                                                       \
 226         __asm__ __volatile__(                                           \
 227         "       mov r2, %0\n"                                           \
 228         "       mov r3, %1\n"                                           \
 229         "       .word %2\n"                                             \
 230         :                                                               \
 231         : "r"(i), "r"(&v->counter), "i"(asm_op)                         \
 232         : "r2", "r3", "memory");                                        \
 233 }                                                                       \
 234
 235 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 236 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 237 {                                                                       \
 238         unsigned int temp = i;                                          \
 239                                                                         \
 240         /* Explicit full memory barrier needed before/after */          \
 241         smp_mb();                                                       \
 242                                                                         \
 243         __asm__ __volatile__(                                           \
 244         "       mov r2, %0\n"                                           \
 245         "       mov r3, %1\n"                                           \
 246         "       .word %2\n"                                             \
 247         "       mov %0, r2"                                             \
 248         : "+r"(temp)                                                    \
 249         : "r"(&v->counter), "i"(asm_op)                                 \
 250         : "r2", "r3", "memory");                                        \
 251                                                                         \
 252         smp_mb();                                                       \
 253                                                                         \
 254         temp c_op i;                                                    \
 255                                                                         \
 256         return temp;                                                    \
 257 }
 258
 259 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 260 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 261 {                                                                       \
 262         unsigned int temp = i;                                          \
 263                                                                         \
 264         /* Explicit full memory barrier needed before/after */          \
 265         smp_mb();                                                       \
 266                                                                         \
 267         __asm__ __volatile__(                                           \
 268         "       mov r2, %0\n"                                           \
 269         "       mov r3, %1\n"                                           \
 270         "       .word %2\n"                                             \
 271         "       mov %0, r2"                                             \
 272         : "+r"(temp)                                                    \
 273         : "r"(&v->counter), "i"(asm_op)                                 \
 274         : "r2", "r3", "memory");                                        \
 275                                                                         \
 276         smp_mb();                                                       \
 277                                                                         \
 278         return temp;                                                    \
 279 }
 280
 281 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 282         ATOMIC_OP(op, c_op, asm_op)                                     \
 283         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 284         ATOMIC_FETCH_OP(op, c_op, asm_op)
 285
 286 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
 287 #define atomic_sub(i, v) atomic_add(-(i), (v))
 288 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
 289 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
 290
 291 #undef ATOMIC_OPS
 292 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 293         ATOMIC_OP(op, c_op, asm_op)                                     \
 294         ATOMIC_FETCH_OP(op, c_op, asm_op)
 295
 296 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
 297 #define atomic_andnot(mask, v) atomic_and(~(mask), (v))
 298 #define atomic_fetch_andnot(mask, v) atomic_fetch_and(~(mask), (v))
 299 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
 300 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
 301
 302 #endif /* CONFIG_ARC_PLAT_EZNPS */
 303
 304 #undef ATOMIC_OPS
 305 #undef ATOMIC_FETCH_OP
 306 #undef ATOMIC_OP_RETURN
 307 #undef ATOMIC_OP
 308
 309 /**
 310  * __atomic_add_unless - add unless the number is a given value
 311  * @v: pointer of type atomic_t
 312  * @a: the amount to add to v...
 313  * @u: ...unless v is equal to u.
 314  *
 315  * Atomically adds @a to @v, so long as it was not @u.
 316  * Returns the old value of @v
 317  */
 318 #define __atomic_add_unless(v, a, u)                                    \
 319 ({                                                                      \
 320         int c, old;                                                     \
 321                                                                         \
 322         /*                                                              \
 323          * Explicit full memory barrier needed before/after as          \
 324          * LLOCK/SCOND thmeselves don't provide any such semantics      \
 325          */                                                             \
 326         smp_mb();                                                       \
 327                                                                         \
 328         c = atomic_read(v);                                             \
 329         while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\
 330                 c = old;                                                \
 331                                                                         \
 332         smp_mb();                                                       \
 333                                                                         \
 334         c;                                                              \
 335 })
 336
 337 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 338
 339 #define atomic_inc(v)                   atomic_add(1, v)
 340 #define atomic_dec(v)                   atomic_sub(1, v)
 341
 342 #define atomic_inc_and_test(v)          (atomic_add_return(1, v) == 0)
 343 #define atomic_dec_and_test(v)          (atomic_sub_return(1, v) == 0)
 344 #define atomic_inc_return(v)            atomic_add_return(1, (v))
 345 #define atomic_dec_return(v)            atomic_sub_return(1, (v))
 346 #define atomic_sub_and_test(i, v)       (atomic_sub_return(i, v) == 0)
 347
 348 #define atomic_add_negative(i, v)       (atomic_add_return(i, v) < 0)
 349
 350
 351 #ifdef CONFIG_GENERIC_ATOMIC64
 352
 353 #include <asm-generic/atomic64.h>
 354
 355 #else   /* Kconfig ensures this is only enabled with needed h/w assist */
 356
 357 /*
 358  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 359  *  - The address HAS to be 64-bit aligned
 360  *  - There are 2 semantics involved here:
 361  *    = exclusive implies no interim update between load/store to same addr
 362  *    = both words are observed/updated together: this is guaranteed even
 363  *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
 364  *      is NOT required to use LLOCKD+SCONDD, STD suffices
 365  */
 366
 367 typedef struct {
 368         aligned_u64 counter;
 369 } atomic64_t;
 370
 371 #define ATOMIC64_INIT(a) { (a) }
 372
 373 static inline long long atomic64_read(const atomic64_t *v)
 374 {
 375         unsigned long long val;
 376
 377         __asm__ __volatile__(
 378         "       ldd   %0, [%1]  \n"
 379         : "=r"(val)
 380         : "r"(&v->counter));
 381
 382         return val;
 383 }
 384
 385 static inline void atomic64_set(atomic64_t *v, long long a)
 386 {
 387         /*
 388          * This could have been a simple assignment in "C" but would need
 389          * explicit volatile. Otherwise gcc optimizers could elide the store
 390          * which borked atomic64 self-test
 391          * In the inline asm version, memory clobber needed for exact same
 392          * reason, to tell gcc about the store.
 393          *
 394          * This however is not needed for sibling atomic64_add() etc since both
 395          * load/store are explicitly done in inline asm. As long as API is used
 396          * for each access, gcc has no way to optimize away any load/store
 397          */
 398         __asm__ __volatile__(
 399         "       std   %0, [%1]  \n"
 400         :
 401         : "r"(a), "r"(&v->counter)
 402         : "memory");
 403 }
 404
 405 #define ATOMIC64_OP(op, op1, op2)                                       \
 406 static inline void atomic64_##op(long long a, atomic64_t *v)            \
 407 {                                                                       \
 408         unsigned long long val;                                         \
 409                                                                         \
 410         __asm__ __volatile__(                                           \
 411         "1:                             \n"                             \
 412         "       llockd  %0, [%1]        \n"                             \
 413         "       " #op1 " %L0, %L0, %L2  \n"                             \
 414         "       " #op2 " %H0, %H0, %H2  \n"                             \
 415         "       scondd   %0, [%1]       \n"                             \
 416         "       bnz     1b              \n"                             \
 417         : "=&r"(val)                                                    \
 418         : "r"(&v->counter), "ir"(a)                                     \
 419         : "cc");                                                \
 420 }                                                                       \
 421
 422 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
 423 static inline long long atomic64_##op##_return(long long a, atomic64_t *v)      \
 424 {                                                                       \
 425         unsigned long long val;                                         \
 426                                                                         \
 427         smp_mb();                                                       \
 428                                                                         \
 429         __asm__ __volatile__(                                           \
 430         "1:                             \n"                             \
 431         "       llockd   %0, [%1]       \n"                             \
 432         "       " #op1 " %L0, %L0, %L2  \n"                             \
 433         "       " #op2 " %H0, %H0, %H2  \n"                             \
 434         "       scondd   %0, [%1]       \n"                             \
 435         "       bnz     1b              \n"                             \
 436         : [val] "=&r"(val)                                              \
 437         : "r"(&v->counter), "ir"(a)                                     \
 438         : "cc");        /* memory clobber comes from smp_mb() */        \
 439                                                                         \
 440         smp_mb();                                                       \
 441                                                                         \
 442         return val;                                                     \
 443 }
 444
 445 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
 446 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
 447 {                                                                       \
 448         unsigned long long val, orig;                                   \
 449                                                                         \
 450         smp_mb();                                                       \
 451                                                                         \
 452         __asm__ __volatile__(                                           \
 453         "1:                             \n"                             \
 454         "       llockd   %0, [%2]       \n"                             \
 455         "       " #op1 " %L1, %L0, %L3  \n"                             \
 456         "       " #op2 " %H1, %H0, %H3  \n"                             \
 457         "       scondd   %1, [%2]       \n"                             \
 458         "       bnz     1b              \n"                             \
 459         : "=&r"(orig), "=&r"(val)                                       \
 460         : "r"(&v->counter), "ir"(a)                                     \
 461         : "cc");        /* memory clobber comes from smp_mb() */        \
 462                                                                         \
 463         smp_mb();                                                       \
 464                                                                         \
 465         return orig;                                                    \
 466 }
 467
 468 #define ATOMIC64_OPS(op, op1, op2)                                      \
 469         ATOMIC64_OP(op, op1, op2)                                       \
 470         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 471         ATOMIC64_FETCH_OP(op, op1, op2)
 472
 473 #define atomic64_andnot atomic64_andnot
 474
 475 ATOMIC64_OPS(add, add.f, adc)
 476 ATOMIC64_OPS(sub, sub.f, sbc)
 477 ATOMIC64_OPS(and, and, and)
 478 ATOMIC64_OPS(andnot, bic, bic)
 479 ATOMIC64_OPS(or, or, or)
 480 ATOMIC64_OPS(xor, xor, xor)
 481
 482 #undef ATOMIC64_OPS
 483 #undef ATOMIC64_FETCH_OP
 484 #undef ATOMIC64_OP_RETURN
 485 #undef ATOMIC64_OP
 486
 487 static inline long long
 488 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
 489 {
 490         long long prev;
 491
 492         smp_mb();
 493
 494         __asm__ __volatile__(
 495         "1:     llockd  %0, [%1]        \n"
 496         "       brne    %L0, %L2, 2f    \n"
 497         "       brne    %H0, %H2, 2f    \n"
 498         "       scondd  %3, [%1]        \n"
 499         "       bnz     1b              \n"
 500         "2:                             \n"
 501         : "=&r"(prev)
 502         : "r"(ptr), "ir"(expected), "r"(new)
 503         : "cc");        /* memory clobber comes from smp_mb() */
 504
 505         smp_mb();
 506
 507         return prev;
 508 }
 509
 510 static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
 511 {
 512         long long prev;
 513
 514         smp_mb();
 515
 516         __asm__ __volatile__(
 517         "1:     llockd  %0, [%1]        \n"
 518         "       scondd  %2, [%1]        \n"
 519         "       bnz     1b              \n"
 520         "2:                             \n"
 521         : "=&r"(prev)
 522         : "r"(ptr), "r"(new)
 523         : "cc");        /* memory clobber comes from smp_mb() */
 524
 525         smp_mb();
 526
 527         return prev;
 528 }
 529
 530 /**
 531  * atomic64_dec_if_positive - decrement by 1 if old value positive
 532  * @v: pointer of type atomic64_t
 533  *
 534  * The function returns the old value of *v minus 1, even if
 535  * the atomic variable, v, was not decremented.
 536  */
 537
 538 static inline long long atomic64_dec_if_positive(atomic64_t *v)
 539 {
 540         long long val;
 541
 542         smp_mb();
 543
 544         __asm__ __volatile__(
 545         "1:     llockd  %0, [%1]        \n"
 546         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 547         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 548         "       brlt    %H0, 0, 2f      \n"
 549         "       scondd  %0, [%1]        \n"
 550         "       bnz     1b              \n"
 551         "2:                             \n"
 552         : "=&r"(val)
 553         : "r"(&v->counter)
 554         : "cc");        /* memory clobber comes from smp_mb() */
 555
 556         smp_mb();
 557
 558         return val;
 559 }
 560
 561 /**
 562  * atomic64_add_unless - add unless the number is a given value
 563  * @v: pointer of type atomic64_t
 564  * @a: the amount to add to v...
 565  * @u: ...unless v is equal to u.
 566  *
 567  * if (v != u) { v += a; ret = 1} else {ret = 0}
 568  * Returns 1 iff @v was not @u (i.e. if add actually happened)
 569  */
 570 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 571 {
 572         long long val;
 573         int op_done;
 574
 575         smp_mb();
 576
 577         __asm__ __volatile__(
 578         "1:     llockd  %0, [%2]        \n"
 579         "       mov     %1, 1           \n"
 580         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 581         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 582         "       mov     %1, 0           \n"
 583         "2:                             \n"
 584         "       add.f   %L0, %L0, %L3   \n"
 585         "       adc     %H0, %H0, %H3   \n"
 586         "       scondd  %0, [%2]        \n"
 587         "       bnz     1b              \n"
 588         "3:                             \n"
 589         : "=&r"(val), "=&r" (op_done)
 590         : "r"(&v->counter), "r"(a), "r"(u)
 591         : "cc");        /* memory clobber comes from smp_mb() */
 592
 593         smp_mb();
 594
 595         return op_done;
 596 }
 597
 598 #define atomic64_add_negative(a, v)     (atomic64_add_return((a), (v)) < 0)
 599 #define atomic64_inc(v)                 atomic64_add(1LL, (v))
 600 #define atomic64_inc_return(v)          atomic64_add_return(1LL, (v))
 601 #define atomic64_inc_and_test(v)        (atomic64_inc_return(v) == 0)
 602 #define atomic64_sub_and_test(a, v)     (atomic64_sub_return((a), (v)) == 0)
 603 #define atomic64_dec(v)                 atomic64_sub(1LL, (v))
 604 #define atomic64_dec_return(v)          atomic64_sub_return(1LL, (v))
 605 #define atomic64_dec_and_test(v)        (atomic64_dec_return((v)) == 0)
 606 #define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1LL, 0LL)
 607
 608 #endif  /* !CONFIG_GENERIC_ATOMIC64 */
 609
 610 #endif  /* !__ASSEMBLY__ */
 611
 612 #endif