2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
8 #ifndef __ASM_CMPXCHG_H
9 #define __ASM_CMPXCHG_H
11 #include <linux/bug.h>
12 #include <linux/irqflags.h>
13 #include <asm/compiler.h>
17 * Using a branch-likely instruction to check the result of an sc instruction
18 * works around a bug present in R10000 CPUs prior to revision 3.0 that could
19 * cause ll-sc sequences to execute non-atomically.
22 # define __scbeqz "beqzl"
24 # define __scbeqz "beqz"
28 * These functions doesn't exist, so if they are called you'll either:
30 * - Get an error at compile-time due to __compiletime_error, if supported by
35 * - Get an error at link-time due to the call to the missing function.
37 extern unsigned long __cmpxchg_called_with_bad_pointer(void)
38 __compiletime_error("Bad argument size for cmpxchg");
39 extern unsigned long __xchg_called_with_bad_pointer(void)
40 __compiletime_error("Bad argument size for xchg");
42 #define __xchg_asm(ld, st, m, val) \
44 __typeof(*(m)) __ret; \
46 if (kernel_uses_llsc) { \
47 __asm__ __volatile__( \
51 " .set " MIPS_ISA_ARCH_LEVEL " \n" \
52 "1: " ld " %0, %2 # __xchg_asm \n" \
55 " .set " MIPS_ISA_ARCH_LEVEL " \n" \
57 "\t" __scbeqz " $1, 1b \n" \
59 : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
60 : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \
63 unsigned long __flags; \
65 raw_local_irq_save(__flags); \
68 raw_local_irq_restore(__flags); \
74 extern unsigned long __xchg_small(volatile void *ptr, unsigned long val,
77 static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
83 return __xchg_small(ptr, x, size);
86 return __xchg_asm("ll", "sc", (volatile u32 *)ptr, x);
89 if (!IS_ENABLED(CONFIG_64BIT))
90 return __xchg_called_with_bad_pointer();
92 return __xchg_asm("lld", "scd", (volatile u64 *)ptr, x);
95 return __xchg_called_with_bad_pointer();
99 #define xchg(ptr, x) \
101 __typeof__(*(ptr)) __res; \
103 smp_mb__before_llsc(); \
105 __res = (__typeof__(*(ptr))) \
106 __xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
113 #define __cmpxchg_asm(ld, st, m, old, new) \
115 __typeof(*(m)) __ret; \
117 if (kernel_uses_llsc) { \
118 __asm__ __volatile__( \
122 " .set "MIPS_ISA_ARCH_LEVEL" \n" \
123 "1: " ld " %0, %2 # __cmpxchg_asm \n" \
124 " bne %0, %z3, 2f \n" \
127 " .set "MIPS_ISA_ARCH_LEVEL" \n" \
128 " " st " $1, %1 \n" \
129 "\t" __scbeqz " $1, 1b \n" \
132 : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
133 : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \
136 unsigned long __flags; \
138 raw_local_irq_save(__flags); \
142 raw_local_irq_restore(__flags); \
148 extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old,
149 unsigned long new, unsigned int size);
151 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
152 unsigned long new, unsigned int size)
157 return __cmpxchg_small(ptr, old, new, size);
160 return __cmpxchg_asm("ll", "sc", (volatile u32 *)ptr,
164 /* lld/scd are only available for MIPS64 */
165 if (!IS_ENABLED(CONFIG_64BIT))
166 return __cmpxchg_called_with_bad_pointer();
168 return __cmpxchg_asm("lld", "scd", (volatile u64 *)ptr,
172 return __cmpxchg_called_with_bad_pointer();
176 #define cmpxchg_local(ptr, old, new) \
177 ((__typeof__(*(ptr))) \
179 (unsigned long)(__typeof__(*(ptr)))(old), \
180 (unsigned long)(__typeof__(*(ptr)))(new), \
183 #define cmpxchg(ptr, old, new) \
185 __typeof__(*(ptr)) __res; \
187 smp_mb__before_llsc(); \
188 __res = cmpxchg_local((ptr), (old), (new)); \
195 #define cmpxchg64_local(ptr, o, n) \
197 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
198 cmpxchg_local((ptr), (o), (n)); \
201 #define cmpxchg64(ptr, o, n) \
203 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
204 cmpxchg((ptr), (o), (n)); \
207 #include <asm-generic/cmpxchg-local.h>
208 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
210 #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n))
216 #endif /* __ASM_CMPXCHG_H */