2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #ifndef _ASM_ARC_ATOMIC_H
10 #define _ASM_ARC_ATOMIC_H
14 #include <linux/types.h>
15 #include <linux/compiler.h>
16 #include <asm/cmpxchg.h>
17 #include <asm/barrier.h>
20 #define ATOMIC_INIT(i) { (i) }
22 #ifndef CONFIG_ARC_PLAT_EZNPS
24 #define atomic_read(v) READ_ONCE((v)->counter)
26 #ifdef CONFIG_ARC_HAS_LLSC
28 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
30 #define ATOMIC_OP(op, c_op, asm_op) \
31 static inline void atomic_##op(int i, atomic_t *v) \
35 __asm__ __volatile__( \
36 "1: llock %[val], [%[ctr]] \n" \
37 " " #asm_op " %[val], %[val], %[i] \n" \
38 " scond %[val], [%[ctr]] \n" \
40 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \
41 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \
46 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
47 static inline int atomic_##op##_return(int i, atomic_t *v) \
52 * Explicit full memory barrier needed before/after as \
53 * LLOCK/SCOND thmeselves don't provide any such semantics \
57 __asm__ __volatile__( \
58 "1: llock %[val], [%[ctr]] \n" \
59 " " #asm_op " %[val], %[val], %[i] \n" \
60 " scond %[val], [%[ctr]] \n" \
63 : [ctr] "r" (&v->counter), \
72 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
73 static inline int atomic_fetch_##op(int i, atomic_t *v) \
75 unsigned int val, orig; \
78 * Explicit full memory barrier needed before/after as \
79 * LLOCK/SCOND thmeselves don't provide any such semantics \
83 __asm__ __volatile__( \
84 "1: llock %[orig], [%[ctr]] \n" \
85 " " #asm_op " %[val], %[orig], %[i] \n" \
86 " scond %[val], [%[ctr]] \n" \
88 : [val] "=&r" (val), \
90 : [ctr] "r" (&v->counter), \
99 #else /* !CONFIG_ARC_HAS_LLSC */
103 /* violating atomic_xxx API locking protocol in UP for optimization sake */
104 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
108 static inline void atomic_set(atomic_t *v, int i)
111 * Independent of hardware support, all of the atomic_xxx() APIs need
112 * to follow the same locking rules to make sure that a "hardware"
113 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
116 * Thus atomic_set() despite being 1 insn (and seemingly atomic)
117 * requires the locking.
121 atomic_ops_lock(flags);
122 WRITE_ONCE(v->counter, i);
123 atomic_ops_unlock(flags);
126 #define atomic_set_release(v, i) atomic_set((v), (i))
131 * Non hardware assisted Atomic-R-M-W
132 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
135 #define ATOMIC_OP(op, c_op, asm_op) \
136 static inline void atomic_##op(int i, atomic_t *v) \
138 unsigned long flags; \
140 atomic_ops_lock(flags); \
142 atomic_ops_unlock(flags); \
145 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
146 static inline int atomic_##op##_return(int i, atomic_t *v) \
148 unsigned long flags; \
149 unsigned long temp; \
152 * spin lock/unlock provides the needed smp_mb() before/after \
154 atomic_ops_lock(flags); \
158 atomic_ops_unlock(flags); \
163 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
164 static inline int atomic_fetch_##op(int i, atomic_t *v) \
166 unsigned long flags; \
167 unsigned long orig; \
170 * spin lock/unlock provides the needed smp_mb() before/after \
172 atomic_ops_lock(flags); \
175 atomic_ops_unlock(flags); \
180 #endif /* !CONFIG_ARC_HAS_LLSC */
182 #define ATOMIC_OPS(op, c_op, asm_op) \
183 ATOMIC_OP(op, c_op, asm_op) \
184 ATOMIC_OP_RETURN(op, c_op, asm_op) \
185 ATOMIC_FETCH_OP(op, c_op, asm_op)
187 ATOMIC_OPS(add, +=, add)
188 ATOMIC_OPS(sub, -=, sub)
190 #define atomic_andnot atomic_andnot
193 #define ATOMIC_OPS(op, c_op, asm_op) \
194 ATOMIC_OP(op, c_op, asm_op) \
195 ATOMIC_FETCH_OP(op, c_op, asm_op)
197 ATOMIC_OPS(and, &=, and)
198 ATOMIC_OPS(andnot, &= ~, bic)
199 ATOMIC_OPS(or, |=, or)
200 ATOMIC_OPS(xor, ^=, xor)
202 #else /* CONFIG_ARC_PLAT_EZNPS */
204 static inline int atomic_read(const atomic_t *v)
208 __asm__ __volatile__(
216 static inline void atomic_set(atomic_t *v, int i)
218 __asm__ __volatile__(
221 : "r"(i), "r"(&v->counter)
225 #define ATOMIC_OP(op, c_op, asm_op) \
226 static inline void atomic_##op(int i, atomic_t *v) \
228 __asm__ __volatile__( \
233 : "r"(i), "r"(&v->counter), "i"(asm_op) \
234 : "r2", "r3", "memory"); \
237 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
238 static inline int atomic_##op##_return(int i, atomic_t *v) \
240 unsigned int temp = i; \
242 /* Explicit full memory barrier needed before/after */ \
245 __asm__ __volatile__( \
251 : "r"(&v->counter), "i"(asm_op) \
252 : "r2", "r3", "memory"); \
261 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
262 static inline int atomic_fetch_##op(int i, atomic_t *v) \
264 unsigned int temp = i; \
266 /* Explicit full memory barrier needed before/after */ \
269 __asm__ __volatile__( \
275 : "r"(&v->counter), "i"(asm_op) \
276 : "r2", "r3", "memory"); \
283 #define ATOMIC_OPS(op, c_op, asm_op) \
284 ATOMIC_OP(op, c_op, asm_op) \
285 ATOMIC_OP_RETURN(op, c_op, asm_op) \
286 ATOMIC_FETCH_OP(op, c_op, asm_op)
288 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
289 #define atomic_sub(i, v) atomic_add(-(i), (v))
290 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
291 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
294 #define ATOMIC_OPS(op, c_op, asm_op) \
295 ATOMIC_OP(op, c_op, asm_op) \
296 ATOMIC_FETCH_OP(op, c_op, asm_op)
298 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
299 #define atomic_andnot(mask, v) atomic_and(~(mask), (v))
300 #define atomic_fetch_andnot(mask, v) atomic_fetch_and(~(mask), (v))
301 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
302 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
304 #endif /* CONFIG_ARC_PLAT_EZNPS */
307 #undef ATOMIC_FETCH_OP
308 #undef ATOMIC_OP_RETURN
311 #define atomic_inc(v) atomic_add(1, v)
312 #define atomic_dec(v) atomic_sub(1, v)
314 #define atomic_inc_return(v) atomic_add_return(1, (v))
315 #define atomic_dec_return(v) atomic_sub_return(1, (v))
317 #ifdef CONFIG_GENERIC_ATOMIC64
319 #include <asm-generic/atomic64.h>
321 #else /* Kconfig ensures this is only enabled with needed h/w assist */
324 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
325 * - The address HAS to be 64-bit aligned
326 * - There are 2 semantics involved here:
327 * = exclusive implies no interim update between load/store to same addr
328 * = both words are observed/updated together: this is guaranteed even
329 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
330 * is NOT required to use LLOCKD+SCONDD, STD suffices
337 #define ATOMIC64_INIT(a) { (a) }
339 static inline long long atomic64_read(const atomic64_t *v)
341 unsigned long long val;
343 __asm__ __volatile__(
351 static inline void atomic64_set(atomic64_t *v, long long a)
354 * This could have been a simple assignment in "C" but would need
355 * explicit volatile. Otherwise gcc optimizers could elide the store
356 * which borked atomic64 self-test
357 * In the inline asm version, memory clobber needed for exact same
358 * reason, to tell gcc about the store.
360 * This however is not needed for sibling atomic64_add() etc since both
361 * load/store are explicitly done in inline asm. As long as API is used
362 * for each access, gcc has no way to optimize away any load/store
364 __asm__ __volatile__(
367 : "r"(a), "r"(&v->counter)
371 #define ATOMIC64_OP(op, op1, op2) \
372 static inline void atomic64_##op(long long a, atomic64_t *v) \
374 unsigned long long val; \
376 __asm__ __volatile__( \
378 " llockd %0, [%1] \n" \
379 " " #op1 " %L0, %L0, %L2 \n" \
380 " " #op2 " %H0, %H0, %H2 \n" \
381 " scondd %0, [%1] \n" \
384 : "r"(&v->counter), "ir"(a) \
388 #define ATOMIC64_OP_RETURN(op, op1, op2) \
389 static inline long long atomic64_##op##_return(long long a, atomic64_t *v) \
391 unsigned long long val; \
395 __asm__ __volatile__( \
397 " llockd %0, [%1] \n" \
398 " " #op1 " %L0, %L0, %L2 \n" \
399 " " #op2 " %H0, %H0, %H2 \n" \
400 " scondd %0, [%1] \n" \
403 : "r"(&v->counter), "ir"(a) \
404 : "cc"); /* memory clobber comes from smp_mb() */ \
411 #define ATOMIC64_FETCH_OP(op, op1, op2) \
412 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
414 unsigned long long val, orig; \
418 __asm__ __volatile__( \
420 " llockd %0, [%2] \n" \
421 " " #op1 " %L1, %L0, %L3 \n" \
422 " " #op2 " %H1, %H0, %H3 \n" \
423 " scondd %1, [%2] \n" \
425 : "=&r"(orig), "=&r"(val) \
426 : "r"(&v->counter), "ir"(a) \
427 : "cc"); /* memory clobber comes from smp_mb() */ \
434 #define ATOMIC64_OPS(op, op1, op2) \
435 ATOMIC64_OP(op, op1, op2) \
436 ATOMIC64_OP_RETURN(op, op1, op2) \
437 ATOMIC64_FETCH_OP(op, op1, op2)
439 #define atomic64_andnot atomic64_andnot
441 ATOMIC64_OPS(add, add.f, adc)
442 ATOMIC64_OPS(sub, sub.f, sbc)
443 ATOMIC64_OPS(and, and, and)
444 ATOMIC64_OPS(andnot, bic, bic)
445 ATOMIC64_OPS(or, or, or)
446 ATOMIC64_OPS(xor, xor, xor)
449 #undef ATOMIC64_FETCH_OP
450 #undef ATOMIC64_OP_RETURN
453 static inline long long
454 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
460 __asm__ __volatile__(
461 "1: llockd %0, [%1] \n"
462 " brne %L0, %L2, 2f \n"
463 " brne %H0, %H2, 2f \n"
464 " scondd %3, [%1] \n"
468 : "r"(ptr), "ir"(expected), "r"(new)
469 : "cc"); /* memory clobber comes from smp_mb() */
476 static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
482 __asm__ __volatile__(
483 "1: llockd %0, [%1] \n"
484 " scondd %2, [%1] \n"
489 : "cc"); /* memory clobber comes from smp_mb() */
497 * atomic64_dec_if_positive - decrement by 1 if old value positive
498 * @v: pointer of type atomic64_t
500 * The function returns the old value of *v minus 1, even if
501 * the atomic variable, v, was not decremented.
504 static inline long long atomic64_dec_if_positive(atomic64_t *v)
510 __asm__ __volatile__(
511 "1: llockd %0, [%1] \n"
512 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n"
513 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n"
514 " brlt %H0, 0, 2f \n"
515 " scondd %0, [%1] \n"
520 : "cc"); /* memory clobber comes from smp_mb() */
528 * atomic64_fetch_add_unless - add unless the number is a given value
529 * @v: pointer of type atomic64_t
530 * @a: the amount to add to v...
531 * @u: ...unless v is equal to u.
533 * Atomically adds @a to @v, if it was not @u.
534 * Returns the old value of @v
536 static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a,
543 __asm__ __volatile__(
544 "1: llockd %0, [%2] \n"
545 " brne %L0, %L4, 2f # continue to add since v != u \n"
546 " breq.d %H0, %H4, 3f # return since v == u \n"
548 " add.f %L1, %L0, %L3 \n"
549 " adc %H1, %H0, %H3 \n"
550 " scondd %1, [%2] \n"
553 : "=&r"(old), "=&r" (temp)
554 : "r"(&v->counter), "r"(a), "r"(u)
555 : "cc"); /* memory clobber comes from smp_mb() */
561 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
563 #define atomic64_inc(v) atomic64_add(1LL, (v))
564 #define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
565 #define atomic64_dec(v) atomic64_sub(1LL, (v))
566 #define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
568 #endif /* !CONFIG_GENERIC_ATOMIC64 */
570 #endif /* !__ASSEMBLY__ */