arch/arm64/include/asm/atomic_lse.h

   1 /*
   2  * Based on arch/arm/include/asm/atomic.h
   3  *
   4  * Copyright (C) 1996 Russell King.
   5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   6  * Copyright (C) 2012 ARM Ltd.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #ifndef __ASM_ATOMIC_LSE_H
  22 #define __ASM_ATOMIC_LSE_H
  23
  24 #ifndef __ARM64_IN_ATOMIC_IMPL
  25 #error "please don't include this file directly"
  26 #endif
  27
  28 #define __LL_SC_ATOMIC(op)      __LL_SC_CALL(atomic_##op)
  29 #define ATOMIC_OP(op, asm_op)                                           \
  30 static inline void atomic_##op(int i, atomic_t *v)                      \
  31 {                                                                       \
  32         register int w0 asm ("w0") = i;                                 \
  33         register atomic_t *x1 asm ("x1") = v;                           \
  34                                                                         \
  35         asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op),          \
  36 "       " #asm_op "     %w[i], %[v]\n")                                 \
  37         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
  38         : "r" (x1)                                                      \
  39         : __LL_SC_CLOBBERS);                                            \
  40 }
  41
  42 ATOMIC_OP(andnot, stclr)
  43 ATOMIC_OP(or, stset)
  44 ATOMIC_OP(xor, steor)
  45 ATOMIC_OP(add, stadd)
  46
  47 #undef ATOMIC_OP
  48
  49 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)                    \
  50 static inline int atomic_fetch_##op##name(int i, atomic_t *v)           \
  51 {                                                                       \
  52         register int w0 asm ("w0") = i;                                 \
  53         register atomic_t *x1 asm ("x1") = v;                           \
  54                                                                         \
  55         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
  56         /* LL/SC */                                                     \
  57         __LL_SC_ATOMIC(fetch_##op##name),                               \
  58         /* LSE atomics */                                               \
  59 "       " #asm_op #mb " %w[i], %w[i], %[v]")                            \
  60         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
  61         : "r" (x1)                                                      \
  62         : __LL_SC_CLOBBERS, ##cl);                                      \
  63                                                                         \
  64         return w0;                                                      \
  65 }
  66
  67 #define ATOMIC_FETCH_OPS(op, asm_op)                                    \
  68         ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)                       \
  69         ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")             \
  70         ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")             \
  71         ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
  72
  73 ATOMIC_FETCH_OPS(andnot, ldclr)
  74 ATOMIC_FETCH_OPS(or, ldset)
  75 ATOMIC_FETCH_OPS(xor, ldeor)
  76 ATOMIC_FETCH_OPS(add, ldadd)
  77
  78 #undef ATOMIC_FETCH_OP
  79 #undef ATOMIC_FETCH_OPS
  80
  81 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...)                           \
  82 static inline int atomic_add_return##name(int i, atomic_t *v)           \
  83 {                                                                       \
  84         register int w0 asm ("w0") = i;                                 \
  85         register atomic_t *x1 asm ("x1") = v;                           \
  86                                                                         \
  87         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
  88         /* LL/SC */                                                     \
  89         __LL_SC_ATOMIC(add_return##name)                                \
  90         __nops(1),                                                      \
  91         /* LSE atomics */                                               \
  92         "       ldadd" #mb "    %w[i], w30, %[v]\n"                     \
  93         "       add     %w[i], %w[i], w30")                             \
  94         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
  95         : "r" (x1)                                                      \
  96         : __LL_SC_CLOBBERS, ##cl);                                      \
  97                                                                         \
  98         return w0;                                                      \
  99 }
 100
 101 ATOMIC_OP_ADD_RETURN(_relaxed,   )
 102 ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
 103 ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
 104 ATOMIC_OP_ADD_RETURN(        , al, "memory")
 105
 106 #undef ATOMIC_OP_ADD_RETURN
 107
 108 static inline void atomic_and(int i, atomic_t *v)
 109 {
 110         register int w0 asm ("w0") = i;
 111         register atomic_t *x1 asm ("x1") = v;
 112
 113         asm volatile(ARM64_LSE_ATOMIC_INSN(
 114         /* LL/SC */
 115         __LL_SC_ATOMIC(and)
 116         __nops(1),
 117         /* LSE atomics */
 118         "       mvn     %w[i], %w[i]\n"
 119         "       stclr   %w[i], %[v]")
 120         : [i] "+r" (w0), [v] "+Q" (v->counter)
 121         : "r" (x1)
 122         : __LL_SC_CLOBBERS);
 123 }
 124
 125 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)                            \
 126 static inline int atomic_fetch_and##name(int i, atomic_t *v)            \
 127 {                                                                       \
 128         register int w0 asm ("w0") = i;                                 \
 129         register atomic_t *x1 asm ("x1") = v;                           \
 130                                                                         \
 131         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 132         /* LL/SC */                                                     \
 133         __LL_SC_ATOMIC(fetch_and##name)                                 \
 134         __nops(1),                                                      \
 135         /* LSE atomics */                                               \
 136         "       mvn     %w[i], %w[i]\n"                                 \
 137         "       ldclr" #mb "    %w[i], %w[i], %[v]")                    \
 138         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
 139         : "r" (x1)                                                      \
 140         : __LL_SC_CLOBBERS, ##cl);                                      \
 141                                                                         \
 142         return w0;                                                      \
 143 }
 144
 145 ATOMIC_FETCH_OP_AND(_relaxed,   )
 146 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
 147 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
 148 ATOMIC_FETCH_OP_AND(        , al, "memory")
 149
 150 #undef ATOMIC_FETCH_OP_AND
 151
 152 static inline void atomic_sub(int i, atomic_t *v)
 153 {
 154         register int w0 asm ("w0") = i;
 155         register atomic_t *x1 asm ("x1") = v;
 156
 157         asm volatile(ARM64_LSE_ATOMIC_INSN(
 158         /* LL/SC */
 159         __LL_SC_ATOMIC(sub)
 160         __nops(1),
 161         /* LSE atomics */
 162         "       neg     %w[i], %w[i]\n"
 163         "       stadd   %w[i], %[v]")
 164         : [i] "+r" (w0), [v] "+Q" (v->counter)
 165         : "r" (x1)
 166         : __LL_SC_CLOBBERS);
 167 }
 168
 169 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...)                           \
 170 static inline int atomic_sub_return##name(int i, atomic_t *v)           \
 171 {                                                                       \
 172         register int w0 asm ("w0") = i;                                 \
 173         register atomic_t *x1 asm ("x1") = v;                           \
 174                                                                         \
 175         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 176         /* LL/SC */                                                     \
 177         __LL_SC_ATOMIC(sub_return##name)                                \
 178         __nops(2),                                                      \
 179         /* LSE atomics */                                               \
 180         "       neg     %w[i], %w[i]\n"                                 \
 181         "       ldadd" #mb "    %w[i], w30, %[v]\n"                     \
 182         "       add     %w[i], %w[i], w30")                             \
 183         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
 184         : "r" (x1)                                                      \
 185         : __LL_SC_CLOBBERS , ##cl);                                     \
 186                                                                         \
 187         return w0;                                                      \
 188 }
 189
 190 ATOMIC_OP_SUB_RETURN(_relaxed,   )
 191 ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
 192 ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
 193 ATOMIC_OP_SUB_RETURN(        , al, "memory")
 194
 195 #undef ATOMIC_OP_SUB_RETURN
 196
 197 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...)                            \
 198 static inline int atomic_fetch_sub##name(int i, atomic_t *v)            \
 199 {                                                                       \
 200         register int w0 asm ("w0") = i;                                 \
 201         register atomic_t *x1 asm ("x1") = v;                           \
 202                                                                         \
 203         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 204         /* LL/SC */                                                     \
 205         __LL_SC_ATOMIC(fetch_sub##name)                                 \
 206         __nops(1),                                                      \
 207         /* LSE atomics */                                               \
 208         "       neg     %w[i], %w[i]\n"                                 \
 209         "       ldadd" #mb "    %w[i], %w[i], %[v]")                    \
 210         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
 211         : "r" (x1)                                                      \
 212         : __LL_SC_CLOBBERS, ##cl);                                      \
 213                                                                         \
 214         return w0;                                                      \
 215 }
 216
 217 ATOMIC_FETCH_OP_SUB(_relaxed,   )
 218 ATOMIC_FETCH_OP_SUB(_acquire,  a, "memory")
 219 ATOMIC_FETCH_OP_SUB(_release,  l, "memory")
 220 ATOMIC_FETCH_OP_SUB(        , al, "memory")
 221
 222 #undef ATOMIC_FETCH_OP_SUB
 223 #undef __LL_SC_ATOMIC
 224
 225 #define __LL_SC_ATOMIC64(op)    __LL_SC_CALL(atomic64_##op)
 226 #define ATOMIC64_OP(op, asm_op)                                         \
 227 static inline void atomic64_##op(long i, atomic64_t *v)                 \
 228 {                                                                       \
 229         register long x0 asm ("x0") = i;                                \
 230         register atomic64_t *x1 asm ("x1") = v;                         \
 231                                                                         \
 232         asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op),        \
 233 "       " #asm_op "     %[i], %[v]\n")                                  \
 234         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 235         : "r" (x1)                                                      \
 236         : __LL_SC_CLOBBERS);                                            \
 237 }
 238
 239 ATOMIC64_OP(andnot, stclr)
 240 ATOMIC64_OP(or, stset)
 241 ATOMIC64_OP(xor, steor)
 242 ATOMIC64_OP(add, stadd)
 243
 244 #undef ATOMIC64_OP
 245
 246 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)                  \
 247 static inline long atomic64_fetch_##op##name(long i, atomic64_t *v)     \
 248 {                                                                       \
 249         register long x0 asm ("x0") = i;                                \
 250         register atomic64_t *x1 asm ("x1") = v;                         \
 251                                                                         \
 252         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 253         /* LL/SC */                                                     \
 254         __LL_SC_ATOMIC64(fetch_##op##name),                             \
 255         /* LSE atomics */                                               \
 256 "       " #asm_op #mb " %[i], %[i], %[v]")                              \
 257         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 258         : "r" (x1)                                                      \
 259         : __LL_SC_CLOBBERS, ##cl);                                      \
 260                                                                         \
 261         return x0;                                                      \
 262 }
 263
 264 #define ATOMIC64_FETCH_OPS(op, asm_op)                                  \
 265         ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)                     \
 266         ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")           \
 267         ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")           \
 268         ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
 269
 270 ATOMIC64_FETCH_OPS(andnot, ldclr)
 271 ATOMIC64_FETCH_OPS(or, ldset)
 272 ATOMIC64_FETCH_OPS(xor, ldeor)
 273 ATOMIC64_FETCH_OPS(add, ldadd)
 274
 275 #undef ATOMIC64_FETCH_OP
 276 #undef ATOMIC64_FETCH_OPS
 277
 278 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)                         \
 279 static inline long atomic64_add_return##name(long i, atomic64_t *v)     \
 280 {                                                                       \
 281         register long x0 asm ("x0") = i;                                \
 282         register atomic64_t *x1 asm ("x1") = v;                         \
 283                                                                         \
 284         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 285         /* LL/SC */                                                     \
 286         __LL_SC_ATOMIC64(add_return##name)                              \
 287         __nops(1),                                                      \
 288         /* LSE atomics */                                               \
 289         "       ldadd" #mb "    %[i], x30, %[v]\n"                      \
 290         "       add     %[i], %[i], x30")                               \
 291         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 292         : "r" (x1)                                                      \
 293         : __LL_SC_CLOBBERS, ##cl);                                      \
 294                                                                         \
 295         return x0;                                                      \
 296 }
 297
 298 ATOMIC64_OP_ADD_RETURN(_relaxed,   )
 299 ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
 300 ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
 301 ATOMIC64_OP_ADD_RETURN(        , al, "memory")
 302
 303 #undef ATOMIC64_OP_ADD_RETURN
 304
 305 static inline void atomic64_and(long i, atomic64_t *v)
 306 {
 307         register long x0 asm ("x0") = i;
 308         register atomic64_t *x1 asm ("x1") = v;
 309
 310         asm volatile(ARM64_LSE_ATOMIC_INSN(
 311         /* LL/SC */
 312         __LL_SC_ATOMIC64(and)
 313         __nops(1),
 314         /* LSE atomics */
 315         "       mvn     %[i], %[i]\n"
 316         "       stclr   %[i], %[v]")
 317         : [i] "+r" (x0), [v] "+Q" (v->counter)
 318         : "r" (x1)
 319         : __LL_SC_CLOBBERS);
 320 }
 321
 322 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)                          \
 323 static inline long atomic64_fetch_and##name(long i, atomic64_t *v)      \
 324 {                                                                       \
 325         register long x0 asm ("x0") = i;                                \
 326         register atomic64_t *x1 asm ("x1") = v;                         \
 327                                                                         \
 328         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 329         /* LL/SC */                                                     \
 330         __LL_SC_ATOMIC64(fetch_and##name)                               \
 331         __nops(1),                                                      \
 332         /* LSE atomics */                                               \
 333         "       mvn     %[i], %[i]\n"                                   \
 334         "       ldclr" #mb "    %[i], %[i], %[v]")                      \
 335         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 336         : "r" (x1)                                                      \
 337         : __LL_SC_CLOBBERS, ##cl);                                      \
 338                                                                         \
 339         return x0;                                                      \
 340 }
 341
 342 ATOMIC64_FETCH_OP_AND(_relaxed,   )
 343 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
 344 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
 345 ATOMIC64_FETCH_OP_AND(        , al, "memory")
 346
 347 #undef ATOMIC64_FETCH_OP_AND
 348
 349 static inline void atomic64_sub(long i, atomic64_t *v)
 350 {
 351         register long x0 asm ("x0") = i;
 352         register atomic64_t *x1 asm ("x1") = v;
 353
 354         asm volatile(ARM64_LSE_ATOMIC_INSN(
 355         /* LL/SC */
 356         __LL_SC_ATOMIC64(sub)
 357         __nops(1),
 358         /* LSE atomics */
 359         "       neg     %[i], %[i]\n"
 360         "       stadd   %[i], %[v]")
 361         : [i] "+r" (x0), [v] "+Q" (v->counter)
 362         : "r" (x1)
 363         : __LL_SC_CLOBBERS);
 364 }
 365
 366 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)                         \
 367 static inline long atomic64_sub_return##name(long i, atomic64_t *v)     \
 368 {                                                                       \
 369         register long x0 asm ("x0") = i;                                \
 370         register atomic64_t *x1 asm ("x1") = v;                         \
 371                                                                         \
 372         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 373         /* LL/SC */                                                     \
 374         __LL_SC_ATOMIC64(sub_return##name)                              \
 375         __nops(2),                                                      \
 376         /* LSE atomics */                                               \
 377         "       neg     %[i], %[i]\n"                                   \
 378         "       ldadd" #mb "    %[i], x30, %[v]\n"                      \
 379         "       add     %[i], %[i], x30")                               \
 380         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 381         : "r" (x1)                                                      \
 382         : __LL_SC_CLOBBERS, ##cl);                                      \
 383                                                                         \
 384         return x0;                                                      \
 385 }
 386
 387 ATOMIC64_OP_SUB_RETURN(_relaxed,   )
 388 ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
 389 ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
 390 ATOMIC64_OP_SUB_RETURN(        , al, "memory")
 391
 392 #undef ATOMIC64_OP_SUB_RETURN
 393
 394 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...)                          \
 395 static inline long atomic64_fetch_sub##name(long i, atomic64_t *v)      \
 396 {                                                                       \
 397         register long x0 asm ("x0") = i;                                \
 398         register atomic64_t *x1 asm ("x1") = v;                         \
 399                                                                         \
 400         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 401         /* LL/SC */                                                     \
 402         __LL_SC_ATOMIC64(fetch_sub##name)                               \
 403         __nops(1),                                                      \
 404         /* LSE atomics */                                               \
 405         "       neg     %[i], %[i]\n"                                   \
 406         "       ldadd" #mb "    %[i], %[i], %[v]")                      \
 407         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 408         : "r" (x1)                                                      \
 409         : __LL_SC_CLOBBERS, ##cl);                                      \
 410                                                                         \
 411         return x0;                                                      \
 412 }
 413
 414 ATOMIC64_FETCH_OP_SUB(_relaxed,   )
 415 ATOMIC64_FETCH_OP_SUB(_acquire,  a, "memory")
 416 ATOMIC64_FETCH_OP_SUB(_release,  l, "memory")
 417 ATOMIC64_FETCH_OP_SUB(        , al, "memory")
 418
 419 #undef ATOMIC64_FETCH_OP_SUB
 420
 421 static inline long atomic64_dec_if_positive(atomic64_t *v)
 422 {
 423         register long x0 asm ("x0") = (long)v;
 424
 425         asm volatile(ARM64_LSE_ATOMIC_INSN(
 426         /* LL/SC */
 427         __LL_SC_ATOMIC64(dec_if_positive)
 428         __nops(6),
 429         /* LSE atomics */
 430         "1:     ldr     x30, %[v]\n"
 431         "       subs    %[ret], x30, #1\n"
 432         "       b.lt    2f\n"
 433         "       casal   x30, %[ret], %[v]\n"
 434         "       sub     x30, x30, #1\n"
 435         "       sub     x30, x30, %[ret]\n"
 436         "       cbnz    x30, 1b\n"
 437         "2:")
 438         : [ret] "+r" (x0), [v] "+Q" (v->counter)
 439         :
 440         : __LL_SC_CLOBBERS, "cc", "memory");
 441
 442         return x0;
 443 }
 444
 445 #undef __LL_SC_ATOMIC64
 446
 447 #define __LL_SC_CMPXCHG(op)     __LL_SC_CALL(__cmpxchg_case_##op)
 448
 449 #define __CMPXCHG_CASE(w, sz, name, mb, cl...)                          \
 450 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr,   \
 451                                                   unsigned long old,    \
 452                                                   unsigned long new)    \
 453 {                                                                       \
 454         register unsigned long x0 asm ("x0") = (unsigned long)ptr;      \
 455         register unsigned long x1 asm ("x1") = old;                     \
 456         register unsigned long x2 asm ("x2") = new;                     \
 457                                                                         \
 458         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 459         /* LL/SC */                                                     \
 460         __LL_SC_CMPXCHG(name)                                           \
 461         __nops(2),                                                      \
 462         /* LSE atomics */                                               \
 463         "       mov     " #w "30, %" #w "[old]\n"                       \
 464         "       cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n"         \
 465         "       mov     %" #w "[ret], " #w "30")                        \
 466         : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr)             \
 467         : [old] "r" (x1), [new] "r" (x2)                                \
 468         : __LL_SC_CLOBBERS, ##cl);                                      \
 469                                                                         \
 470         return x0;                                                      \
 471 }
 472
 473 __CMPXCHG_CASE(w, b,     1,   )
 474 __CMPXCHG_CASE(w, h,     2,   )
 475 __CMPXCHG_CASE(w,  ,     4,   )
 476 __CMPXCHG_CASE(x,  ,     8,   )
 477 __CMPXCHG_CASE(w, b, acq_1,  a, "memory")
 478 __CMPXCHG_CASE(w, h, acq_2,  a, "memory")
 479 __CMPXCHG_CASE(w,  , acq_4,  a, "memory")
 480 __CMPXCHG_CASE(x,  , acq_8,  a, "memory")
 481 __CMPXCHG_CASE(w, b, rel_1,  l, "memory")
 482 __CMPXCHG_CASE(w, h, rel_2,  l, "memory")
 483 __CMPXCHG_CASE(w,  , rel_4,  l, "memory")
 484 __CMPXCHG_CASE(x,  , rel_8,  l, "memory")
 485 __CMPXCHG_CASE(w, b,  mb_1, al, "memory")
 486 __CMPXCHG_CASE(w, h,  mb_2, al, "memory")
 487 __CMPXCHG_CASE(w,  ,  mb_4, al, "memory")
 488 __CMPXCHG_CASE(x,  ,  mb_8, al, "memory")
 489
 490 #undef __LL_SC_CMPXCHG
 491 #undef __CMPXCHG_CASE
 492
 493 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
 494
 495 #define __CMPXCHG_DBL(name, mb, cl...)                                  \
 496 static inline long __cmpxchg_double##name(unsigned long old1,           \
 497                                          unsigned long old2,            \
 498                                          unsigned long new1,            \
 499                                          unsigned long new2,            \
 500                                          volatile void *ptr)            \
 501 {                                                                       \
 502         unsigned long oldval1 = old1;                                   \
 503         unsigned long oldval2 = old2;                                   \
 504         register unsigned long x0 asm ("x0") = old1;                    \
 505         register unsigned long x1 asm ("x1") = old2;                    \
 506         register unsigned long x2 asm ("x2") = new1;                    \
 507         register unsigned long x3 asm ("x3") = new2;                    \
 508         register unsigned long x4 asm ("x4") = (unsigned long)ptr;      \
 509                                                                         \
 510         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 511         /* LL/SC */                                                     \
 512         __LL_SC_CMPXCHG_DBL(name)                                       \
 513         __nops(3),                                                      \
 514         /* LSE atomics */                                               \
 515         "       casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
 516         "       eor     %[old1], %[old1], %[oldval1]\n"                 \
 517         "       eor     %[old2], %[old2], %[oldval2]\n"                 \
 518         "       orr     %[old1], %[old1], %[old2]")                     \
 519         : [old1] "+r" (x0), [old2] "+r" (x1),                           \
 520           [v] "+Q" (*(unsigned long *)ptr)                              \
 521         : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),             \
 522           [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)              \
 523         : __LL_SC_CLOBBERS, ##cl);                                      \
 524                                                                         \
 525         return x0;                                                      \
 526 }
 527
 528 __CMPXCHG_DBL(   ,   )
 529 __CMPXCHG_DBL(_mb, al, "memory")
 530
 531 #undef __LL_SC_CMPXCHG_DBL
 532 #undef __CMPXCHG_DBL
 533
 534 #endif  /* __ASM_ATOMIC_LSE_H */