arch/arm64/include/asm/atomic_lse.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/atomic.h
   4  *
   5  * Copyright (C) 1996 Russell King.
   6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   7  * Copyright (C) 2012 ARM Ltd.
   8  */
   9
  10 #ifndef __ASM_ATOMIC_LSE_H
  11 #define __ASM_ATOMIC_LSE_H
  12
  13 #define ATOMIC_OP(op, asm_op)                                           \
  14 static inline void __lse_atomic_##op(int i, atomic_t *v)                        \
  15 {                                                                       \
  16         asm volatile(                                                   \
  17         __LSE_PREAMBLE                                                  \
  18 "       " #asm_op "     %w[i], %[v]\n"                                  \
  19         : [i] "+r" (i), [v] "+Q" (v->counter)                           \
  20         : "r" (v));                                                     \
  21 }
  22
  23 ATOMIC_OP(andnot, stclr)
  24 ATOMIC_OP(or, stset)
  25 ATOMIC_OP(xor, steor)
  26 ATOMIC_OP(add, stadd)
  27
  28 #undef ATOMIC_OP
  29
  30 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)                    \
  31 static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v)     \
  32 {                                                                       \
  33         asm volatile(                                                   \
  34         __LSE_PREAMBLE                                                  \
  35 "       " #asm_op #mb " %w[i], %w[i], %[v]"                             \
  36         : [i] "+r" (i), [v] "+Q" (v->counter)                           \
  37         : "r" (v)                                                       \
  38         : cl);                                                          \
  39                                                                         \
  40         return i;                                                       \
  41 }
  42
  43 #define ATOMIC_FETCH_OPS(op, asm_op)                                    \
  44         ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)                       \
  45         ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")             \
  46         ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")             \
  47         ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
  48
  49 ATOMIC_FETCH_OPS(andnot, ldclr)
  50 ATOMIC_FETCH_OPS(or, ldset)
  51 ATOMIC_FETCH_OPS(xor, ldeor)
  52 ATOMIC_FETCH_OPS(add, ldadd)
  53
  54 #undef ATOMIC_FETCH_OP
  55 #undef ATOMIC_FETCH_OPS
  56
  57 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...)                           \
  58 static inline int __lse_atomic_add_return##name(int i, atomic_t *v)     \
  59 {                                                                       \
  60         u32 tmp;                                                        \
  61                                                                         \
  62         asm volatile(                                                   \
  63         __LSE_PREAMBLE                                                  \
  64         "       ldadd" #mb "    %w[i], %w[tmp], %[v]\n"                 \
  65         "       add     %w[i], %w[i], %w[tmp]"                          \
  66         : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)        \
  67         : "r" (v)                                                       \
  68         : cl);                                                          \
  69                                                                         \
  70         return i;                                                       \
  71 }
  72
  73 ATOMIC_OP_ADD_RETURN(_relaxed,   )
  74 ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
  75 ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
  76 ATOMIC_OP_ADD_RETURN(        , al, "memory")
  77
  78 #undef ATOMIC_OP_ADD_RETURN
  79
  80 static inline void __lse_atomic_and(int i, atomic_t *v)
  81 {
  82         asm volatile(
  83         __LSE_PREAMBLE
  84         "       mvn     %w[i], %w[i]\n"
  85         "       stclr   %w[i], %[v]"
  86         : [i] "+&r" (i), [v] "+Q" (v->counter)
  87         : "r" (v));
  88 }
  89
  90 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)                            \
  91 static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v)      \
  92 {                                                                       \
  93         asm volatile(                                                   \
  94         __LSE_PREAMBLE                                                  \
  95         "       mvn     %w[i], %w[i]\n"                                 \
  96         "       ldclr" #mb "    %w[i], %w[i], %[v]"                     \
  97         : [i] "+&r" (i), [v] "+Q" (v->counter)                          \
  98         : "r" (v)                                                       \
  99         : cl);                                                          \
 100                                                                         \
 101         return i;                                                       \
 102 }
 103
 104 ATOMIC_FETCH_OP_AND(_relaxed,   )
 105 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
 106 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
 107 ATOMIC_FETCH_OP_AND(        , al, "memory")
 108
 109 #undef ATOMIC_FETCH_OP_AND
 110
 111 static inline void __lse_atomic_sub(int i, atomic_t *v)
 112 {
 113         asm volatile(
 114         __LSE_PREAMBLE
 115         "       neg     %w[i], %w[i]\n"
 116         "       stadd   %w[i], %[v]"
 117         : [i] "+&r" (i), [v] "+Q" (v->counter)
 118         : "r" (v));
 119 }
 120
 121 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...)                           \
 122 static inline int __lse_atomic_sub_return##name(int i, atomic_t *v)     \
 123 {                                                                       \
 124         u32 tmp;                                                        \
 125                                                                         \
 126         asm volatile(                                                   \
 127         __LSE_PREAMBLE                                                  \
 128         "       neg     %w[i], %w[i]\n"                                 \
 129         "       ldadd" #mb "    %w[i], %w[tmp], %[v]\n"                 \
 130         "       add     %w[i], %w[i], %w[tmp]"                          \
 131         : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)       \
 132         : "r" (v)                                                       \
 133         : cl);                                                  \
 134                                                                         \
 135         return i;                                                       \
 136 }
 137
 138 ATOMIC_OP_SUB_RETURN(_relaxed,   )
 139 ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
 140 ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
 141 ATOMIC_OP_SUB_RETURN(        , al, "memory")
 142
 143 #undef ATOMIC_OP_SUB_RETURN
 144
 145 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...)                            \
 146 static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v)      \
 147 {                                                                       \
 148         asm volatile(                                                   \
 149         __LSE_PREAMBLE                                                  \
 150         "       neg     %w[i], %w[i]\n"                                 \
 151         "       ldadd" #mb "    %w[i], %w[i], %[v]"                     \
 152         : [i] "+&r" (i), [v] "+Q" (v->counter)                          \
 153         : "r" (v)                                                       \
 154         : cl);                                                          \
 155                                                                         \
 156         return i;                                                       \
 157 }
 158
 159 ATOMIC_FETCH_OP_SUB(_relaxed,   )
 160 ATOMIC_FETCH_OP_SUB(_acquire,  a, "memory")
 161 ATOMIC_FETCH_OP_SUB(_release,  l, "memory")
 162 ATOMIC_FETCH_OP_SUB(        , al, "memory")
 163
 164 #undef ATOMIC_FETCH_OP_SUB
 165
 166 #define ATOMIC64_OP(op, asm_op)                                         \
 167 static inline void __lse_atomic64_##op(s64 i, atomic64_t *v)            \
 168 {                                                                       \
 169         asm volatile(                                                   \
 170         __LSE_PREAMBLE                                                  \
 171 "       " #asm_op "     %[i], %[v]\n"                                   \
 172         : [i] "+r" (i), [v] "+Q" (v->counter)                           \
 173         : "r" (v));                                                     \
 174 }
 175
 176 ATOMIC64_OP(andnot, stclr)
 177 ATOMIC64_OP(or, stset)
 178 ATOMIC64_OP(xor, steor)
 179 ATOMIC64_OP(add, stadd)
 180
 181 #undef ATOMIC64_OP
 182
 183 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)                  \
 184 static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\
 185 {                                                                       \
 186         asm volatile(                                                   \
 187         __LSE_PREAMBLE                                                  \
 188 "       " #asm_op #mb " %[i], %[i], %[v]"                               \
 189         : [i] "+r" (i), [v] "+Q" (v->counter)                           \
 190         : "r" (v)                                                       \
 191         : cl);                                                          \
 192                                                                         \
 193         return i;                                                       \
 194 }
 195
 196 #define ATOMIC64_FETCH_OPS(op, asm_op)                                  \
 197         ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)                     \
 198         ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")           \
 199         ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")           \
 200         ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
 201
 202 ATOMIC64_FETCH_OPS(andnot, ldclr)
 203 ATOMIC64_FETCH_OPS(or, ldset)
 204 ATOMIC64_FETCH_OPS(xor, ldeor)
 205 ATOMIC64_FETCH_OPS(add, ldadd)
 206
 207 #undef ATOMIC64_FETCH_OP
 208 #undef ATOMIC64_FETCH_OPS
 209
 210 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)                         \
 211 static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\
 212 {                                                                       \
 213         unsigned long tmp;                                              \
 214                                                                         \
 215         asm volatile(                                                   \
 216         __LSE_PREAMBLE                                                  \
 217         "       ldadd" #mb "    %[i], %x[tmp], %[v]\n"                  \
 218         "       add     %[i], %[i], %x[tmp]"                            \
 219         : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)        \
 220         : "r" (v)                                                       \
 221         : cl);                                                          \
 222                                                                         \
 223         return i;                                                       \
 224 }
 225
 226 ATOMIC64_OP_ADD_RETURN(_relaxed,   )
 227 ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
 228 ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
 229 ATOMIC64_OP_ADD_RETURN(        , al, "memory")
 230
 231 #undef ATOMIC64_OP_ADD_RETURN
 232
 233 static inline void __lse_atomic64_and(s64 i, atomic64_t *v)
 234 {
 235         asm volatile(
 236         __LSE_PREAMBLE
 237         "       mvn     %[i], %[i]\n"
 238         "       stclr   %[i], %[v]"
 239         : [i] "+&r" (i), [v] "+Q" (v->counter)
 240         : "r" (v));
 241 }
 242
 243 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)                          \
 244 static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v) \
 245 {                                                                       \
 246         asm volatile(                                                   \
 247         __LSE_PREAMBLE                                                  \
 248         "       mvn     %[i], %[i]\n"                                   \
 249         "       ldclr" #mb "    %[i], %[i], %[v]"                       \
 250         : [i] "+&r" (i), [v] "+Q" (v->counter)                          \
 251         : "r" (v)                                                       \
 252         : cl);                                                          \
 253                                                                         \
 254         return i;                                                       \
 255 }
 256
 257 ATOMIC64_FETCH_OP_AND(_relaxed,   )
 258 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
 259 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
 260 ATOMIC64_FETCH_OP_AND(        , al, "memory")
 261
 262 #undef ATOMIC64_FETCH_OP_AND
 263
 264 static inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
 265 {
 266         asm volatile(
 267         __LSE_PREAMBLE
 268         "       neg     %[i], %[i]\n"
 269         "       stadd   %[i], %[v]"
 270         : [i] "+&r" (i), [v] "+Q" (v->counter)
 271         : "r" (v));
 272 }
 273
 274 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)                         \
 275 static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)        \
 276 {                                                                       \
 277         unsigned long tmp;                                              \
 278                                                                         \
 279         asm volatile(                                                   \
 280         __LSE_PREAMBLE                                                  \
 281         "       neg     %[i], %[i]\n"                                   \
 282         "       ldadd" #mb "    %[i], %x[tmp], %[v]\n"                  \
 283         "       add     %[i], %[i], %x[tmp]"                            \
 284         : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)       \
 285         : "r" (v)                                                       \
 286         : cl);                                                          \
 287                                                                         \
 288         return i;                                                       \
 289 }
 290
 291 ATOMIC64_OP_SUB_RETURN(_relaxed,   )
 292 ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
 293 ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
 294 ATOMIC64_OP_SUB_RETURN(        , al, "memory")
 295
 296 #undef ATOMIC64_OP_SUB_RETURN
 297
 298 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...)                          \
 299 static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \
 300 {                                                                       \
 301         asm volatile(                                                   \
 302         __LSE_PREAMBLE                                                  \
 303         "       neg     %[i], %[i]\n"                                   \
 304         "       ldadd" #mb "    %[i], %[i], %[v]"                       \
 305         : [i] "+&r" (i), [v] "+Q" (v->counter)                          \
 306         : "r" (v)                                                       \
 307         : cl);                                                          \
 308                                                                         \
 309         return i;                                                       \
 310 }
 311
 312 ATOMIC64_FETCH_OP_SUB(_relaxed,   )
 313 ATOMIC64_FETCH_OP_SUB(_acquire,  a, "memory")
 314 ATOMIC64_FETCH_OP_SUB(_release,  l, "memory")
 315 ATOMIC64_FETCH_OP_SUB(        , al, "memory")
 316
 317 #undef ATOMIC64_FETCH_OP_SUB
 318
 319 static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
 320 {
 321         unsigned long tmp;
 322
 323         asm volatile(
 324         __LSE_PREAMBLE
 325         "1:     ldr     %x[tmp], %[v]\n"
 326         "       subs    %[ret], %x[tmp], #1\n"
 327         "       b.lt    2f\n"
 328         "       casal   %x[tmp], %[ret], %[v]\n"
 329         "       sub     %x[tmp], %x[tmp], #1\n"
 330         "       sub     %x[tmp], %x[tmp], %[ret]\n"
 331         "       cbnz    %x[tmp], 1b\n"
 332         "2:"
 333         : [ret] "+&r" (v), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)
 334         :
 335         : "cc", "memory");
 336
 337         return (long)v;
 338 }
 339
 340 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...)                     \
 341 static __always_inline u##sz                                            \
 342 __lse__cmpxchg_case_##name##sz(volatile void *ptr,                      \
 343                                               u##sz old,                \
 344                                               u##sz new)                \
 345 {                                                                       \
 346         register unsigned long x0 asm ("x0") = (unsigned long)ptr;      \
 347         register u##sz x1 asm ("x1") = old;                             \
 348         register u##sz x2 asm ("x2") = new;                             \
 349         unsigned long tmp;                                              \
 350                                                                         \
 351         asm volatile(                                                   \
 352         __LSE_PREAMBLE                                                  \
 353         "       mov     %" #w "[tmp], %" #w "[old]\n"                   \
 354         "       cas" #mb #sfx "\t%" #w "[tmp], %" #w "[new], %[v]\n"    \
 355         "       mov     %" #w "[ret], %" #w "[tmp]"                     \
 356         : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr),            \
 357           [tmp] "=&r" (tmp)                                             \
 358         : [old] "r" (x1), [new] "r" (x2)                                \
 359         : cl);                                                          \
 360                                                                         \
 361         return x0;                                                      \
 362 }
 363
 364 __CMPXCHG_CASE(w, b,     ,  8,   )
 365 __CMPXCHG_CASE(w, h,     , 16,   )
 366 __CMPXCHG_CASE(w,  ,     , 32,   )
 367 __CMPXCHG_CASE(x,  ,     , 64,   )
 368 __CMPXCHG_CASE(w, b, acq_,  8,  a, "memory")
 369 __CMPXCHG_CASE(w, h, acq_, 16,  a, "memory")
 370 __CMPXCHG_CASE(w,  , acq_, 32,  a, "memory")
 371 __CMPXCHG_CASE(x,  , acq_, 64,  a, "memory")
 372 __CMPXCHG_CASE(w, b, rel_,  8,  l, "memory")
 373 __CMPXCHG_CASE(w, h, rel_, 16,  l, "memory")
 374 __CMPXCHG_CASE(w,  , rel_, 32,  l, "memory")
 375 __CMPXCHG_CASE(x,  , rel_, 64,  l, "memory")
 376 __CMPXCHG_CASE(w, b,  mb_,  8, al, "memory")
 377 __CMPXCHG_CASE(w, h,  mb_, 16, al, "memory")
 378 __CMPXCHG_CASE(w,  ,  mb_, 32, al, "memory")
 379 __CMPXCHG_CASE(x,  ,  mb_, 64, al, "memory")
 380
 381 #undef __CMPXCHG_CASE
 382
 383 #define __CMPXCHG_DBL(name, mb, cl...)                                  \
 384 static __always_inline long                                             \
 385 __lse__cmpxchg_double##name(unsigned long old1,                         \
 386                                          unsigned long old2,            \
 387                                          unsigned long new1,            \
 388                                          unsigned long new2,            \
 389                                          volatile void *ptr)            \
 390 {                                                                       \
 391         unsigned long oldval1 = old1;                                   \
 392         unsigned long oldval2 = old2;                                   \
 393         register unsigned long x0 asm ("x0") = old1;                    \
 394         register unsigned long x1 asm ("x1") = old2;                    \
 395         register unsigned long x2 asm ("x2") = new1;                    \
 396         register unsigned long x3 asm ("x3") = new2;                    \
 397         register unsigned long x4 asm ("x4") = (unsigned long)ptr;      \
 398                                                                         \
 399         asm volatile(                                                   \
 400         __LSE_PREAMBLE                                                  \
 401         "       casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
 402         "       eor     %[old1], %[old1], %[oldval1]\n"                 \
 403         "       eor     %[old2], %[old2], %[oldval2]\n"                 \
 404         "       orr     %[old1], %[old1], %[old2]"                      \
 405         : [old1] "+&r" (x0), [old2] "+&r" (x1),                         \
 406           [v] "+Q" (*(unsigned long *)ptr)                              \
 407         : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),             \
 408           [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)              \
 409         : cl);                                                          \
 410                                                                         \
 411         return x0;                                                      \
 412 }
 413
 414 __CMPXCHG_DBL(   ,   )
 415 __CMPXCHG_DBL(_mb, al, "memory")
 416
 417 #undef __CMPXCHG_DBL
 418
 419 #endif  /* __ASM_ATOMIC_LSE_H */