Merge branch 'locking/atomics' into locking/core, to pick up WIP commits
[sfrench/cifs-2.6.git] / arch / arm64 / include / asm / atomic_lse.h
1 /*
2  * Based on arch/arm/include/asm/atomic.h
3  *
4  * Copyright (C) 1996 Russell King.
5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
6  * Copyright (C) 2012 ARM Ltd.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #ifndef __ASM_ATOMIC_LSE_H
22 #define __ASM_ATOMIC_LSE_H
23
24 #ifndef __ARM64_IN_ATOMIC_IMPL
25 #error "please don't include this file directly"
26 #endif
27
28 #define __LL_SC_ATOMIC(op)      __LL_SC_CALL(arch_atomic_##op)
29 #define ATOMIC_OP(op, asm_op)                                           \
30 static inline void arch_atomic_##op(int i, atomic_t *v)                 \
31 {                                                                       \
32         register int w0 asm ("w0") = i;                                 \
33         register atomic_t *x1 asm ("x1") = v;                           \
34                                                                         \
35         asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op),          \
36 "       " #asm_op "     %w[i], %[v]\n")                                 \
37         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
38         : "r" (x1)                                                      \
39         : __LL_SC_CLOBBERS);                                            \
40 }
41
42 ATOMIC_OP(andnot, stclr)
43 ATOMIC_OP(or, stset)
44 ATOMIC_OP(xor, steor)
45 ATOMIC_OP(add, stadd)
46
47 #undef ATOMIC_OP
48
49 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)                    \
50 static inline int arch_atomic_fetch_##op##name(int i, atomic_t *v)      \
51 {                                                                       \
52         register int w0 asm ("w0") = i;                                 \
53         register atomic_t *x1 asm ("x1") = v;                           \
54                                                                         \
55         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
56         /* LL/SC */                                                     \
57         __LL_SC_ATOMIC(fetch_##op##name),                               \
58         /* LSE atomics */                                               \
59 "       " #asm_op #mb " %w[i], %w[i], %[v]")                            \
60         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
61         : "r" (x1)                                                      \
62         : __LL_SC_CLOBBERS, ##cl);                                      \
63                                                                         \
64         return w0;                                                      \
65 }
66
67 #define ATOMIC_FETCH_OPS(op, asm_op)                                    \
68         ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)                       \
69         ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")             \
70         ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")             \
71         ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
72
73 ATOMIC_FETCH_OPS(andnot, ldclr)
74 ATOMIC_FETCH_OPS(or, ldset)
75 ATOMIC_FETCH_OPS(xor, ldeor)
76 ATOMIC_FETCH_OPS(add, ldadd)
77
78 #undef ATOMIC_FETCH_OP
79 #undef ATOMIC_FETCH_OPS
80
81 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...)                           \
82 static inline int arch_atomic_add_return##name(int i, atomic_t *v)      \
83 {                                                                       \
84         register int w0 asm ("w0") = i;                                 \
85         register atomic_t *x1 asm ("x1") = v;                           \
86                                                                         \
87         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
88         /* LL/SC */                                                     \
89         __LL_SC_ATOMIC(add_return##name)                                \
90         __nops(1),                                                      \
91         /* LSE atomics */                                               \
92         "       ldadd" #mb "    %w[i], w30, %[v]\n"                     \
93         "       add     %w[i], %w[i], w30")                             \
94         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
95         : "r" (x1)                                                      \
96         : __LL_SC_CLOBBERS, ##cl);                                      \
97                                                                         \
98         return w0;                                                      \
99 }
100
101 ATOMIC_OP_ADD_RETURN(_relaxed,   )
102 ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
103 ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
104 ATOMIC_OP_ADD_RETURN(        , al, "memory")
105
106 #undef ATOMIC_OP_ADD_RETURN
107
108 static inline void arch_atomic_and(int i, atomic_t *v)
109 {
110         register int w0 asm ("w0") = i;
111         register atomic_t *x1 asm ("x1") = v;
112
113         asm volatile(ARM64_LSE_ATOMIC_INSN(
114         /* LL/SC */
115         __LL_SC_ATOMIC(and)
116         __nops(1),
117         /* LSE atomics */
118         "       mvn     %w[i], %w[i]\n"
119         "       stclr   %w[i], %[v]")
120         : [i] "+&r" (w0), [v] "+Q" (v->counter)
121         : "r" (x1)
122         : __LL_SC_CLOBBERS);
123 }
124
125 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)                            \
126 static inline int arch_atomic_fetch_and##name(int i, atomic_t *v)       \
127 {                                                                       \
128         register int w0 asm ("w0") = i;                                 \
129         register atomic_t *x1 asm ("x1") = v;                           \
130                                                                         \
131         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
132         /* LL/SC */                                                     \
133         __LL_SC_ATOMIC(fetch_and##name)                                 \
134         __nops(1),                                                      \
135         /* LSE atomics */                                               \
136         "       mvn     %w[i], %w[i]\n"                                 \
137         "       ldclr" #mb "    %w[i], %w[i], %[v]")                    \
138         : [i] "+&r" (w0), [v] "+Q" (v->counter)                         \
139         : "r" (x1)                                                      \
140         : __LL_SC_CLOBBERS, ##cl);                                      \
141                                                                         \
142         return w0;                                                      \
143 }
144
145 ATOMIC_FETCH_OP_AND(_relaxed,   )
146 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
147 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
148 ATOMIC_FETCH_OP_AND(        , al, "memory")
149
150 #undef ATOMIC_FETCH_OP_AND
151
152 static inline void arch_atomic_sub(int i, atomic_t *v)
153 {
154         register int w0 asm ("w0") = i;
155         register atomic_t *x1 asm ("x1") = v;
156
157         asm volatile(ARM64_LSE_ATOMIC_INSN(
158         /* LL/SC */
159         __LL_SC_ATOMIC(sub)
160         __nops(1),
161         /* LSE atomics */
162         "       neg     %w[i], %w[i]\n"
163         "       stadd   %w[i], %[v]")
164         : [i] "+&r" (w0), [v] "+Q" (v->counter)
165         : "r" (x1)
166         : __LL_SC_CLOBBERS);
167 }
168
169 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...)                           \
170 static inline int arch_atomic_sub_return##name(int i, atomic_t *v)      \
171 {                                                                       \
172         register int w0 asm ("w0") = i;                                 \
173         register atomic_t *x1 asm ("x1") = v;                           \
174                                                                         \
175         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
176         /* LL/SC */                                                     \
177         __LL_SC_ATOMIC(sub_return##name)                                \
178         __nops(2),                                                      \
179         /* LSE atomics */                                               \
180         "       neg     %w[i], %w[i]\n"                                 \
181         "       ldadd" #mb "    %w[i], w30, %[v]\n"                     \
182         "       add     %w[i], %w[i], w30")                             \
183         : [i] "+&r" (w0), [v] "+Q" (v->counter)                         \
184         : "r" (x1)                                                      \
185         : __LL_SC_CLOBBERS , ##cl);                                     \
186                                                                         \
187         return w0;                                                      \
188 }
189
190 ATOMIC_OP_SUB_RETURN(_relaxed,   )
191 ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
192 ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
193 ATOMIC_OP_SUB_RETURN(        , al, "memory")
194
195 #undef ATOMIC_OP_SUB_RETURN
196
197 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...)                            \
198 static inline int arch_atomic_fetch_sub##name(int i, atomic_t *v)       \
199 {                                                                       \
200         register int w0 asm ("w0") = i;                                 \
201         register atomic_t *x1 asm ("x1") = v;                           \
202                                                                         \
203         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
204         /* LL/SC */                                                     \
205         __LL_SC_ATOMIC(fetch_sub##name)                                 \
206         __nops(1),                                                      \
207         /* LSE atomics */                                               \
208         "       neg     %w[i], %w[i]\n"                                 \
209         "       ldadd" #mb "    %w[i], %w[i], %[v]")                    \
210         : [i] "+&r" (w0), [v] "+Q" (v->counter)                         \
211         : "r" (x1)                                                      \
212         : __LL_SC_CLOBBERS, ##cl);                                      \
213                                                                         \
214         return w0;                                                      \
215 }
216
217 ATOMIC_FETCH_OP_SUB(_relaxed,   )
218 ATOMIC_FETCH_OP_SUB(_acquire,  a, "memory")
219 ATOMIC_FETCH_OP_SUB(_release,  l, "memory")
220 ATOMIC_FETCH_OP_SUB(        , al, "memory")
221
222 #undef ATOMIC_FETCH_OP_SUB
223 #undef __LL_SC_ATOMIC
224
225 #define __LL_SC_ATOMIC64(op)    __LL_SC_CALL(arch_atomic64_##op)
226 #define ATOMIC64_OP(op, asm_op)                                         \
227 static inline void arch_atomic64_##op(long i, atomic64_t *v)            \
228 {                                                                       \
229         register long x0 asm ("x0") = i;                                \
230         register atomic64_t *x1 asm ("x1") = v;                         \
231                                                                         \
232         asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op),        \
233 "       " #asm_op "     %[i], %[v]\n")                                  \
234         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
235         : "r" (x1)                                                      \
236         : __LL_SC_CLOBBERS);                                            \
237 }
238
239 ATOMIC64_OP(andnot, stclr)
240 ATOMIC64_OP(or, stset)
241 ATOMIC64_OP(xor, steor)
242 ATOMIC64_OP(add, stadd)
243
244 #undef ATOMIC64_OP
245
246 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)                  \
247 static inline long arch_atomic64_fetch_##op##name(long i, atomic64_t *v)\
248 {                                                                       \
249         register long x0 asm ("x0") = i;                                \
250         register atomic64_t *x1 asm ("x1") = v;                         \
251                                                                         \
252         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
253         /* LL/SC */                                                     \
254         __LL_SC_ATOMIC64(fetch_##op##name),                             \
255         /* LSE atomics */                                               \
256 "       " #asm_op #mb " %[i], %[i], %[v]")                              \
257         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
258         : "r" (x1)                                                      \
259         : __LL_SC_CLOBBERS, ##cl);                                      \
260                                                                         \
261         return x0;                                                      \
262 }
263
264 #define ATOMIC64_FETCH_OPS(op, asm_op)                                  \
265         ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)                     \
266         ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")           \
267         ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")           \
268         ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
269
270 ATOMIC64_FETCH_OPS(andnot, ldclr)
271 ATOMIC64_FETCH_OPS(or, ldset)
272 ATOMIC64_FETCH_OPS(xor, ldeor)
273 ATOMIC64_FETCH_OPS(add, ldadd)
274
275 #undef ATOMIC64_FETCH_OP
276 #undef ATOMIC64_FETCH_OPS
277
278 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)                         \
279 static inline long arch_atomic64_add_return##name(long i, atomic64_t *v)\
280 {                                                                       \
281         register long x0 asm ("x0") = i;                                \
282         register atomic64_t *x1 asm ("x1") = v;                         \
283                                                                         \
284         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
285         /* LL/SC */                                                     \
286         __LL_SC_ATOMIC64(add_return##name)                              \
287         __nops(1),                                                      \
288         /* LSE atomics */                                               \
289         "       ldadd" #mb "    %[i], x30, %[v]\n"                      \
290         "       add     %[i], %[i], x30")                               \
291         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
292         : "r" (x1)                                                      \
293         : __LL_SC_CLOBBERS, ##cl);                                      \
294                                                                         \
295         return x0;                                                      \
296 }
297
298 ATOMIC64_OP_ADD_RETURN(_relaxed,   )
299 ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
300 ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
301 ATOMIC64_OP_ADD_RETURN(        , al, "memory")
302
303 #undef ATOMIC64_OP_ADD_RETURN
304
305 static inline void arch_atomic64_and(long i, atomic64_t *v)
306 {
307         register long x0 asm ("x0") = i;
308         register atomic64_t *x1 asm ("x1") = v;
309
310         asm volatile(ARM64_LSE_ATOMIC_INSN(
311         /* LL/SC */
312         __LL_SC_ATOMIC64(and)
313         __nops(1),
314         /* LSE atomics */
315         "       mvn     %[i], %[i]\n"
316         "       stclr   %[i], %[v]")
317         : [i] "+&r" (x0), [v] "+Q" (v->counter)
318         : "r" (x1)
319         : __LL_SC_CLOBBERS);
320 }
321
322 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)                          \
323 static inline long arch_atomic64_fetch_and##name(long i, atomic64_t *v) \
324 {                                                                       \
325         register long x0 asm ("x0") = i;                                \
326         register atomic64_t *x1 asm ("x1") = v;                         \
327                                                                         \
328         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
329         /* LL/SC */                                                     \
330         __LL_SC_ATOMIC64(fetch_and##name)                               \
331         __nops(1),                                                      \
332         /* LSE atomics */                                               \
333         "       mvn     %[i], %[i]\n"                                   \
334         "       ldclr" #mb "    %[i], %[i], %[v]")                      \
335         : [i] "+&r" (x0), [v] "+Q" (v->counter)                         \
336         : "r" (x1)                                                      \
337         : __LL_SC_CLOBBERS, ##cl);                                      \
338                                                                         \
339         return x0;                                                      \
340 }
341
342 ATOMIC64_FETCH_OP_AND(_relaxed,   )
343 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
344 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
345 ATOMIC64_FETCH_OP_AND(        , al, "memory")
346
347 #undef ATOMIC64_FETCH_OP_AND
348
349 static inline void arch_atomic64_sub(long i, atomic64_t *v)
350 {
351         register long x0 asm ("x0") = i;
352         register atomic64_t *x1 asm ("x1") = v;
353
354         asm volatile(ARM64_LSE_ATOMIC_INSN(
355         /* LL/SC */
356         __LL_SC_ATOMIC64(sub)
357         __nops(1),
358         /* LSE atomics */
359         "       neg     %[i], %[i]\n"
360         "       stadd   %[i], %[v]")
361         : [i] "+&r" (x0), [v] "+Q" (v->counter)
362         : "r" (x1)
363         : __LL_SC_CLOBBERS);
364 }
365
366 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)                         \
367 static inline long arch_atomic64_sub_return##name(long i, atomic64_t *v)\
368 {                                                                       \
369         register long x0 asm ("x0") = i;                                \
370         register atomic64_t *x1 asm ("x1") = v;                         \
371                                                                         \
372         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
373         /* LL/SC */                                                     \
374         __LL_SC_ATOMIC64(sub_return##name)                              \
375         __nops(2),                                                      \
376         /* LSE atomics */                                               \
377         "       neg     %[i], %[i]\n"                                   \
378         "       ldadd" #mb "    %[i], x30, %[v]\n"                      \
379         "       add     %[i], %[i], x30")                               \
380         : [i] "+&r" (x0), [v] "+Q" (v->counter)                         \
381         : "r" (x1)                                                      \
382         : __LL_SC_CLOBBERS, ##cl);                                      \
383                                                                         \
384         return x0;                                                      \
385 }
386
387 ATOMIC64_OP_SUB_RETURN(_relaxed,   )
388 ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
389 ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
390 ATOMIC64_OP_SUB_RETURN(        , al, "memory")
391
392 #undef ATOMIC64_OP_SUB_RETURN
393
394 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...)                          \
395 static inline long arch_atomic64_fetch_sub##name(long i, atomic64_t *v) \
396 {                                                                       \
397         register long x0 asm ("x0") = i;                                \
398         register atomic64_t *x1 asm ("x1") = v;                         \
399                                                                         \
400         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
401         /* LL/SC */                                                     \
402         __LL_SC_ATOMIC64(fetch_sub##name)                               \
403         __nops(1),                                                      \
404         /* LSE atomics */                                               \
405         "       neg     %[i], %[i]\n"                                   \
406         "       ldadd" #mb "    %[i], %[i], %[v]")                      \
407         : [i] "+&r" (x0), [v] "+Q" (v->counter)                         \
408         : "r" (x1)                                                      \
409         : __LL_SC_CLOBBERS, ##cl);                                      \
410                                                                         \
411         return x0;                                                      \
412 }
413
414 ATOMIC64_FETCH_OP_SUB(_relaxed,   )
415 ATOMIC64_FETCH_OP_SUB(_acquire,  a, "memory")
416 ATOMIC64_FETCH_OP_SUB(_release,  l, "memory")
417 ATOMIC64_FETCH_OP_SUB(        , al, "memory")
418
419 #undef ATOMIC64_FETCH_OP_SUB
420
421 static inline long arch_atomic64_dec_if_positive(atomic64_t *v)
422 {
423         register long x0 asm ("x0") = (long)v;
424
425         asm volatile(ARM64_LSE_ATOMIC_INSN(
426         /* LL/SC */
427         __LL_SC_ATOMIC64(dec_if_positive)
428         __nops(6),
429         /* LSE atomics */
430         "1:     ldr     x30, %[v]\n"
431         "       subs    %[ret], x30, #1\n"
432         "       b.lt    2f\n"
433         "       casal   x30, %[ret], %[v]\n"
434         "       sub     x30, x30, #1\n"
435         "       sub     x30, x30, %[ret]\n"
436         "       cbnz    x30, 1b\n"
437         "2:")
438         : [ret] "+&r" (x0), [v] "+Q" (v->counter)
439         :
440         : __LL_SC_CLOBBERS, "cc", "memory");
441
442         return x0;
443 }
444
445 #undef __LL_SC_ATOMIC64
446
447 #define __LL_SC_CMPXCHG(op)     __LL_SC_CALL(__cmpxchg_case_##op)
448
449 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...)                     \
450 static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr,       \
451                                               u##sz old,                \
452                                               u##sz new)                \
453 {                                                                       \
454         register unsigned long x0 asm ("x0") = (unsigned long)ptr;      \
455         register u##sz x1 asm ("x1") = old;                             \
456         register u##sz x2 asm ("x2") = new;                             \
457                                                                         \
458         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
459         /* LL/SC */                                                     \
460         __LL_SC_CMPXCHG(name##sz)                                       \
461         __nops(2),                                                      \
462         /* LSE atomics */                                               \
463         "       mov     " #w "30, %" #w "[old]\n"                       \
464         "       cas" #mb #sfx "\t" #w "30, %" #w "[new], %[v]\n"        \
465         "       mov     %" #w "[ret], " #w "30")                        \
466         : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr)             \
467         : [old] "r" (x1), [new] "r" (x2)                                \
468         : __LL_SC_CLOBBERS, ##cl);                                      \
469                                                                         \
470         return x0;                                                      \
471 }
472
473 __CMPXCHG_CASE(w, b,     ,  8,   )
474 __CMPXCHG_CASE(w, h,     , 16,   )
475 __CMPXCHG_CASE(w,  ,     , 32,   )
476 __CMPXCHG_CASE(x,  ,     , 64,   )
477 __CMPXCHG_CASE(w, b, acq_,  8,  a, "memory")
478 __CMPXCHG_CASE(w, h, acq_, 16,  a, "memory")
479 __CMPXCHG_CASE(w,  , acq_, 32,  a, "memory")
480 __CMPXCHG_CASE(x,  , acq_, 64,  a, "memory")
481 __CMPXCHG_CASE(w, b, rel_,  8,  l, "memory")
482 __CMPXCHG_CASE(w, h, rel_, 16,  l, "memory")
483 __CMPXCHG_CASE(w,  , rel_, 32,  l, "memory")
484 __CMPXCHG_CASE(x,  , rel_, 64,  l, "memory")
485 __CMPXCHG_CASE(w, b,  mb_,  8, al, "memory")
486 __CMPXCHG_CASE(w, h,  mb_, 16, al, "memory")
487 __CMPXCHG_CASE(w,  ,  mb_, 32, al, "memory")
488 __CMPXCHG_CASE(x,  ,  mb_, 64, al, "memory")
489
490 #undef __LL_SC_CMPXCHG
491 #undef __CMPXCHG_CASE
492
493 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
494
495 #define __CMPXCHG_DBL(name, mb, cl...)                                  \
496 static inline long __cmpxchg_double##name(unsigned long old1,           \
497                                          unsigned long old2,            \
498                                          unsigned long new1,            \
499                                          unsigned long new2,            \
500                                          volatile void *ptr)            \
501 {                                                                       \
502         unsigned long oldval1 = old1;                                   \
503         unsigned long oldval2 = old2;                                   \
504         register unsigned long x0 asm ("x0") = old1;                    \
505         register unsigned long x1 asm ("x1") = old2;                    \
506         register unsigned long x2 asm ("x2") = new1;                    \
507         register unsigned long x3 asm ("x3") = new2;                    \
508         register unsigned long x4 asm ("x4") = (unsigned long)ptr;      \
509                                                                         \
510         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
511         /* LL/SC */                                                     \
512         __LL_SC_CMPXCHG_DBL(name)                                       \
513         __nops(3),                                                      \
514         /* LSE atomics */                                               \
515         "       casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
516         "       eor     %[old1], %[old1], %[oldval1]\n"                 \
517         "       eor     %[old2], %[old2], %[oldval2]\n"                 \
518         "       orr     %[old1], %[old1], %[old2]")                     \
519         : [old1] "+&r" (x0), [old2] "+&r" (x1),                         \
520           [v] "+Q" (*(unsigned long *)ptr)                              \
521         : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),             \
522           [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)              \
523         : __LL_SC_CLOBBERS, ##cl);                                      \
524                                                                         \
525         return x0;                                                      \
526 }
527
528 __CMPXCHG_DBL(   ,   )
529 __CMPXCHG_DBL(_mb, al, "memory")
530
531 #undef __LL_SC_CMPXCHG_DBL
532 #undef __CMPXCHG_DBL
533
534 #endif  /* __ASM_ATOMIC_LSE_H */