arch/riscv/include/asm/atomic.h

   1 /*
   2  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   3  * Copyright (C) 2012 Regents of the University of California
   4  * Copyright (C) 2017 SiFive
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public Licence
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the Licence, or (at your option) any later version.
  10  */
  11
  12 #ifndef _ASM_RISCV_ATOMIC_H
  13 #define _ASM_RISCV_ATOMIC_H
  14
  15 #ifdef CONFIG_GENERIC_ATOMIC64
  16 # include <asm-generic/atomic64.h>
  17 #else
  18 # if (__riscv_xlen < 64)
  19 #  error "64-bit atomics require XLEN to be at least 64"
  20 # endif
  21 #endif
  22
  23 #include <asm/cmpxchg.h>
  24 #include <asm/barrier.h>
  25
  26 #define ATOMIC_INIT(i)  { (i) }
  27
  28 #define __atomic_op_acquire(op, args...)                                \
  29 ({                                                                      \
  30         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  31         __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory");    \
  32         __ret;                                                          \
  33 })
  34
  35 #define __atomic_op_release(op, args...)                                \
  36 ({                                                                      \
  37         __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");    \
  38         op##_relaxed(args);                                             \
  39 })
  40
  41 static __always_inline int atomic_read(const atomic_t *v)
  42 {
  43         return READ_ONCE(v->counter);
  44 }
  45 static __always_inline void atomic_set(atomic_t *v, int i)
  46 {
  47         WRITE_ONCE(v->counter, i);
  48 }
  49
  50 #ifndef CONFIG_GENERIC_ATOMIC64
  51 #define ATOMIC64_INIT(i) { (i) }
  52 static __always_inline long atomic64_read(const atomic64_t *v)
  53 {
  54         return READ_ONCE(v->counter);
  55 }
  56 static __always_inline void atomic64_set(atomic64_t *v, long i)
  57 {
  58         WRITE_ONCE(v->counter, i);
  59 }
  60 #endif
  61
  62 /*
  63  * First, the atomic ops that have no ordering constraints and therefor don't
  64  * have the AQ or RL bits set.  These don't return anything, so there's only
  65  * one version to worry about.
  66  */
  67 #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix)              \
  68 static __always_inline                                                  \
  69 void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v)             \
  70 {                                                                       \
  71         __asm__ __volatile__ (                                          \
  72                 "       amo" #asm_op "." #asm_type " zero, %1, %0"      \
  73                 : "+A" (v->counter)                                     \
  74                 : "r" (I)                                               \
  75                 : "memory");                                            \
  76 }                                                                       \
  77
  78 #ifdef CONFIG_GENERIC_ATOMIC64
  79 #define ATOMIC_OPS(op, asm_op, I)                                       \
  80         ATOMIC_OP (op, asm_op, I, w,  int,   )
  81 #else
  82 #define ATOMIC_OPS(op, asm_op, I)                                       \
  83         ATOMIC_OP (op, asm_op, I, w,  int,   )                          \
  84         ATOMIC_OP (op, asm_op, I, d, long, 64)
  85 #endif
  86
  87 ATOMIC_OPS(add, add,  i)
  88 ATOMIC_OPS(sub, add, -i)
  89 ATOMIC_OPS(and, and,  i)
  90 ATOMIC_OPS( or,  or,  i)
  91 ATOMIC_OPS(xor, xor,  i)
  92
  93 #undef ATOMIC_OP
  94 #undef ATOMIC_OPS
  95
  96 /*
  97  * Atomic ops that have ordered, relaxed, acquire, and release variants.
  98  * There's two flavors of these: the arithmatic ops have both fetch and return
  99  * versions, while the logical ops only have fetch versions.
 100  */
 101 #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix)        \
 102 static __always_inline                                                  \
 103 c_type atomic##prefix##_fetch_##op##_relaxed(c_type i,                  \
 104                                              atomic##prefix##_t *v)     \
 105 {                                                                       \
 106         register c_type ret;                                            \
 107         __asm__ __volatile__ (                                          \
 108                 "       amo" #asm_op "." #asm_type " %1, %2, %0"        \
 109                 : "+A" (v->counter), "=r" (ret)                         \
 110                 : "r" (I)                                               \
 111                 : "memory");                                            \
 112         return ret;                                                     \
 113 }                                                                       \
 114 static __always_inline                                                  \
 115 c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)     \
 116 {                                                                       \
 117         register c_type ret;                                            \
 118         __asm__ __volatile__ (                                          \
 119                 "       amo" #asm_op "." #asm_type ".aqrl  %1, %2, %0"  \
 120                 : "+A" (v->counter), "=r" (ret)                         \
 121                 : "r" (I)                                               \
 122                 : "memory");                                            \
 123         return ret;                                                     \
 124 }
 125
 126 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
 127 static __always_inline                                                  \
 128 c_type atomic##prefix##_##op##_return_relaxed(c_type i,                 \
 129                                               atomic##prefix##_t *v)    \
 130 {                                                                       \
 131         return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I;      \
 132 }                                                                       \
 133 static __always_inline                                                  \
 134 c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v)  \
 135 {                                                                       \
 136         return atomic##prefix##_fetch_##op(i, v) c_op I;                \
 137 }
 138
 139 #ifdef CONFIG_GENERIC_ATOMIC64
 140 #define ATOMIC_OPS(op, asm_op, c_op, I)                                 \
 141         ATOMIC_FETCH_OP( op, asm_op,       I, w,  int,   )              \
 142         ATOMIC_OP_RETURN(op, asm_op, c_op, I, w,  int,   )
 143 #else
 144 #define ATOMIC_OPS(op, asm_op, c_op, I)                                 \
 145         ATOMIC_FETCH_OP( op, asm_op,       I, w,  int,   )              \
 146         ATOMIC_OP_RETURN(op, asm_op, c_op, I, w,  int,   )              \
 147         ATOMIC_FETCH_OP( op, asm_op,       I, d, long, 64)              \
 148         ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, long, 64)
 149 #endif
 150
 151 ATOMIC_OPS(add, add, +,  i)
 152 ATOMIC_OPS(sub, add, +, -i)
 153
 154 #define atomic_add_return_relaxed       atomic_add_return_relaxed
 155 #define atomic_sub_return_relaxed       atomic_sub_return_relaxed
 156 #define atomic_add_return               atomic_add_return
 157 #define atomic_sub_return               atomic_sub_return
 158
 159 #define atomic_fetch_add_relaxed        atomic_fetch_add_relaxed
 160 #define atomic_fetch_sub_relaxed        atomic_fetch_sub_relaxed
 161 #define atomic_fetch_add                atomic_fetch_add
 162 #define atomic_fetch_sub                atomic_fetch_sub
 163
 164 #ifndef CONFIG_GENERIC_ATOMIC64
 165 #define atomic64_add_return_relaxed     atomic64_add_return_relaxed
 166 #define atomic64_sub_return_relaxed     atomic64_sub_return_relaxed
 167 #define atomic64_add_return             atomic64_add_return
 168 #define atomic64_sub_return             atomic64_sub_return
 169
 170 #define atomic64_fetch_add_relaxed      atomic64_fetch_add_relaxed
 171 #define atomic64_fetch_sub_relaxed      atomic64_fetch_sub_relaxed
 172 #define atomic64_fetch_add              atomic64_fetch_add
 173 #define atomic64_fetch_sub              atomic64_fetch_sub
 174 #endif
 175
 176 #undef ATOMIC_OPS
 177
 178 #ifdef CONFIG_GENERIC_ATOMIC64
 179 #define ATOMIC_OPS(op, asm_op, I)                                       \
 180         ATOMIC_FETCH_OP(op, asm_op, I, w,  int,   )
 181 #else
 182 #define ATOMIC_OPS(op, asm_op, I)                                       \
 183         ATOMIC_FETCH_OP(op, asm_op, I, w,  int,   )                     \
 184         ATOMIC_FETCH_OP(op, asm_op, I, d, long, 64)
 185 #endif
 186
 187 ATOMIC_OPS(and, and, i)
 188 ATOMIC_OPS( or,  or, i)
 189 ATOMIC_OPS(xor, xor, i)
 190
 191 #define atomic_fetch_and_relaxed        atomic_fetch_and_relaxed
 192 #define atomic_fetch_or_relaxed         atomic_fetch_or_relaxed
 193 #define atomic_fetch_xor_relaxed        atomic_fetch_xor_relaxed
 194 #define atomic_fetch_and                atomic_fetch_and
 195 #define atomic_fetch_or                 atomic_fetch_or
 196 #define atomic_fetch_xor                atomic_fetch_xor
 197
 198 #ifndef CONFIG_GENERIC_ATOMIC64
 199 #define atomic64_fetch_and_relaxed      atomic64_fetch_and_relaxed
 200 #define atomic64_fetch_or_relaxed       atomic64_fetch_or_relaxed
 201 #define atomic64_fetch_xor_relaxed      atomic64_fetch_xor_relaxed
 202 #define atomic64_fetch_and              atomic64_fetch_and
 203 #define atomic64_fetch_or               atomic64_fetch_or
 204 #define atomic64_fetch_xor              atomic64_fetch_xor
 205 #endif
 206
 207 #undef ATOMIC_OPS
 208
 209 #undef ATOMIC_FETCH_OP
 210 #undef ATOMIC_OP_RETURN
 211
 212 /*
 213  * The extra atomic operations that are constructed from one of the core
 214  * AMO-based operations above (aside from sub, which is easier to fit above).
 215  * These are required to perform a full barrier, but they're OK this way
 216  * because atomic_*_return is also required to perform a full barrier.
 217  *
 218  */
 219 #define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix)              \
 220 static __always_inline                                                  \
 221 bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v)             \
 222 {                                                                       \
 223         return atomic##prefix##_##func_op##_return(i, v) comp_op I;     \
 224 }
 225
 226 #ifdef CONFIG_GENERIC_ATOMIC64
 227 #define ATOMIC_OPS(op, func_op, comp_op, I)                             \
 228         ATOMIC_OP(op, func_op, comp_op, I,  int,   )
 229 #else
 230 #define ATOMIC_OPS(op, func_op, comp_op, I)                             \
 231         ATOMIC_OP(op, func_op, comp_op, I,  int,   )                    \
 232         ATOMIC_OP(op, func_op, comp_op, I, long, 64)
 233 #endif
 234
 235 ATOMIC_OPS(add_and_test, add, ==, 0)
 236 ATOMIC_OPS(sub_and_test, sub, ==, 0)
 237 ATOMIC_OPS(add_negative, add,  <, 0)
 238
 239 #undef ATOMIC_OP
 240 #undef ATOMIC_OPS
 241
 242 #define ATOMIC_OP(op, func_op, I, c_type, prefix)                       \
 243 static __always_inline                                                  \
 244 void atomic##prefix##_##op(atomic##prefix##_t *v)                       \
 245 {                                                                       \
 246         atomic##prefix##_##func_op(I, v);                               \
 247 }
 248
 249 #define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix)                 \
 250 static __always_inline                                                  \
 251 c_type atomic##prefix##_fetch_##op##_relaxed(atomic##prefix##_t *v)     \
 252 {                                                                       \
 253         return atomic##prefix##_fetch_##func_op##_relaxed(I, v);        \
 254 }                                                                       \
 255 static __always_inline                                                  \
 256 c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v)               \
 257 {                                                                       \
 258         return atomic##prefix##_fetch_##func_op(I, v);                  \
 259 }
 260
 261 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix)           \
 262 static __always_inline                                                  \
 263 c_type atomic##prefix##_##op##_return_relaxed(atomic##prefix##_t *v)    \
 264 {                                                                       \
 265         return atomic##prefix##_fetch_##op##_relaxed(v) c_op I;         \
 266 }                                                                       \
 267 static __always_inline                                                  \
 268 c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v)            \
 269 {                                                                       \
 270         return atomic##prefix##_fetch_##op(v) c_op I;                   \
 271 }
 272
 273 #ifdef CONFIG_GENERIC_ATOMIC64
 274 #define ATOMIC_OPS(op, asm_op, c_op, I)                                 \
 275         ATOMIC_OP(       op, asm_op,       I,  int,   )                 \
 276         ATOMIC_FETCH_OP( op, asm_op,       I,  int,   )                 \
 277         ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )
 278 #else
 279 #define ATOMIC_OPS(op, asm_op, c_op, I)                                 \
 280         ATOMIC_OP(       op, asm_op,       I,  int,   )                 \
 281         ATOMIC_FETCH_OP( op, asm_op,       I,  int,   )                 \
 282         ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )                 \
 283         ATOMIC_OP(       op, asm_op,       I, long, 64)                 \
 284         ATOMIC_FETCH_OP( op, asm_op,       I, long, 64)                 \
 285         ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)
 286 #endif
 287
 288 ATOMIC_OPS(inc, add, +,  1)
 289 ATOMIC_OPS(dec, add, +, -1)
 290
 291 #define atomic_inc_return_relaxed       atomic_inc_return_relaxed
 292 #define atomic_dec_return_relaxed       atomic_dec_return_relaxed
 293 #define atomic_inc_return               atomic_inc_return
 294 #define atomic_dec_return               atomic_dec_return
 295
 296 #define atomic_fetch_inc_relaxed        atomic_fetch_inc_relaxed
 297 #define atomic_fetch_dec_relaxed        atomic_fetch_dec_relaxed
 298 #define atomic_fetch_inc                atomic_fetch_inc
 299 #define atomic_fetch_dec                atomic_fetch_dec
 300
 301 #ifndef CONFIG_GENERIC_ATOMIC64
 302 #define atomic64_inc_return_relaxed     atomic64_inc_return_relaxed
 303 #define atomic64_dec_return_relaxed     atomic64_dec_return_relaxed
 304 #define atomic64_inc_return             atomic64_inc_return
 305 #define atomic64_dec_return             atomic64_dec_return
 306
 307 #define atomic64_fetch_inc_relaxed      atomic64_fetch_inc_relaxed
 308 #define atomic64_fetch_dec_relaxed      atomic64_fetch_dec_relaxed
 309 #define atomic64_fetch_inc              atomic64_fetch_inc
 310 #define atomic64_fetch_dec              atomic64_fetch_dec
 311 #endif
 312
 313 #undef ATOMIC_OPS
 314 #undef ATOMIC_OP
 315 #undef ATOMIC_FETCH_OP
 316 #undef ATOMIC_OP_RETURN
 317
 318 #define ATOMIC_OP(op, func_op, comp_op, I, prefix)                      \
 319 static __always_inline                                                  \
 320 bool atomic##prefix##_##op(atomic##prefix##_t *v)                       \
 321 {                                                                       \
 322         return atomic##prefix##_##func_op##_return(v) comp_op I;        \
 323 }
 324
 325 ATOMIC_OP(inc_and_test, inc, ==, 0,   )
 326 ATOMIC_OP(dec_and_test, dec, ==, 0,   )
 327 #ifndef CONFIG_GENERIC_ATOMIC64
 328 ATOMIC_OP(inc_and_test, inc, ==, 0, 64)
 329 ATOMIC_OP(dec_and_test, dec, ==, 0, 64)
 330 #endif
 331
 332 #undef ATOMIC_OP
 333
 334 /* This is required to provide a full barrier on success. */
 335 static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
 336 {
 337        int prev, rc;
 338
 339         __asm__ __volatile__ (
 340                 "0:     lr.w     %[p],  %[c]\n"
 341                 "       beq      %[p],  %[u], 1f\n"
 342                 "       add      %[rc], %[p], %[a]\n"
 343                 "       sc.w.rl  %[rc], %[rc], %[c]\n"
 344                 "       bnez     %[rc], 0b\n"
 345                 "       fence    rw, rw\n"
 346                 "1:\n"
 347                 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 348                 : [a]"r" (a), [u]"r" (u)
 349                 : "memory");
 350         return prev;
 351 }
 352
 353 #ifndef CONFIG_GENERIC_ATOMIC64
 354 static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u)
 355 {
 356        long prev, rc;
 357
 358         __asm__ __volatile__ (
 359                 "0:     lr.d     %[p],  %[c]\n"
 360                 "       beq      %[p],  %[u], 1f\n"
 361                 "       add      %[rc], %[p], %[a]\n"
 362                 "       sc.d.rl  %[rc], %[rc], %[c]\n"
 363                 "       bnez     %[rc], 0b\n"
 364                 "       fence    rw, rw\n"
 365                 "1:\n"
 366                 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 367                 : [a]"r" (a), [u]"r" (u)
 368                 : "memory");
 369         return prev;
 370 }
 371
 372 static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)
 373 {
 374         return __atomic64_add_unless(v, a, u) != u;
 375 }
 376 #endif
 377
 378 /*
 379  * The extra atomic operations that are constructed from one of the core
 380  * LR/SC-based operations above.
 381  */
 382 static __always_inline int atomic_inc_not_zero(atomic_t *v)
 383 {
 384         return __atomic_add_unless(v, 1, 0);
 385 }
 386
 387 #ifndef CONFIG_GENERIC_ATOMIC64
 388 static __always_inline long atomic64_inc_not_zero(atomic64_t *v)
 389 {
 390         return atomic64_add_unless(v, 1, 0);
 391 }
 392 #endif
 393
 394 /*
 395  * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
 396  * {cmp,}xchg and the operations that return, so they need a full barrier.
 397  */
 398 #define ATOMIC_OP(c_t, prefix, size)                                    \
 399 static __always_inline                                                  \
 400 c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n)         \
 401 {                                                                       \
 402         return __xchg_relaxed(&(v->counter), n, size);                  \
 403 }                                                                       \
 404 static __always_inline                                                  \
 405 c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n)         \
 406 {                                                                       \
 407         return __xchg_acquire(&(v->counter), n, size);                  \
 408 }                                                                       \
 409 static __always_inline                                                  \
 410 c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n)         \
 411 {                                                                       \
 412         return __xchg_release(&(v->counter), n, size);                  \
 413 }                                                                       \
 414 static __always_inline                                                  \
 415 c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n)                 \
 416 {                                                                       \
 417         return __xchg(&(v->counter), n, size);                          \
 418 }                                                                       \
 419 static __always_inline                                                  \
 420 c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v,             \
 421                                      c_t o, c_t n)                      \
 422 {                                                                       \
 423         return __cmpxchg_relaxed(&(v->counter), o, n, size);            \
 424 }                                                                       \
 425 static __always_inline                                                  \
 426 c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v,             \
 427                                      c_t o, c_t n)                      \
 428 {                                                                       \
 429         return __cmpxchg_acquire(&(v->counter), o, n, size);            \
 430 }                                                                       \
 431 static __always_inline                                                  \
 432 c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v,             \
 433                                      c_t o, c_t n)                      \
 434 {                                                                       \
 435         return __cmpxchg_release(&(v->counter), o, n, size);            \
 436 }                                                                       \
 437 static __always_inline                                                  \
 438 c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n)       \
 439 {                                                                       \
 440         return __cmpxchg(&(v->counter), o, n, size);                    \
 441 }
 442
 443 #ifdef CONFIG_GENERIC_ATOMIC64
 444 #define ATOMIC_OPS()                                                    \
 445         ATOMIC_OP( int,   , 4)
 446 #else
 447 #define ATOMIC_OPS()                                                    \
 448         ATOMIC_OP( int,   , 4)                                          \
 449         ATOMIC_OP(long, 64, 8)
 450 #endif
 451
 452 ATOMIC_OPS()
 453
 454 #undef ATOMIC_OPS
 455 #undef ATOMIC_OP
 456
 457 static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
 458 {
 459        int prev, rc;
 460
 461         __asm__ __volatile__ (
 462                 "0:     lr.w     %[p],  %[c]\n"
 463                 "       sub      %[rc], %[p], %[o]\n"
 464                 "       bltz     %[rc], 1f\n"
 465                 "       sc.w.rl  %[rc], %[rc], %[c]\n"
 466                 "       bnez     %[rc], 0b\n"
 467                 "       fence    rw, rw\n"
 468                 "1:\n"
 469                 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 470                 : [o]"r" (offset)
 471                 : "memory");
 472         return prev - offset;
 473 }
 474
 475 #define atomic_dec_if_positive(v)       atomic_sub_if_positive(v, 1)
 476
 477 #ifndef CONFIG_GENERIC_ATOMIC64
 478 static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)
 479 {
 480        long prev, rc;
 481
 482         __asm__ __volatile__ (
 483                 "0:     lr.d     %[p],  %[c]\n"
 484                 "       sub      %[rc], %[p], %[o]\n"
 485                 "       bltz     %[rc], 1f\n"
 486                 "       sc.d.rl  %[rc], %[rc], %[c]\n"
 487                 "       bnez     %[rc], 0b\n"
 488                 "       fence    rw, rw\n"
 489                 "1:\n"
 490                 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 491                 : [o]"r" (offset)
 492                 : "memory");
 493         return prev - offset;
 494 }
 495
 496 #define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(v, 1)
 497 #endif
 498
 499 #endif /* _ASM_RISCV_ATOMIC_H */