x86: fix 32-bit case of __get_user_asm_u64()
[sfrench/cifs-2.6.git] / arch / x86 / include / asm / uaccess.h
1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
3 /*
4  * User space memory access functions
5  */
6 #include <linux/compiler.h>
7 #include <linux/kasan-checks.h>
8 #include <linux/string.h>
9 #include <asm/asm.h>
10 #include <asm/page.h>
11 #include <asm/smap.h>
12 #include <asm/extable.h>
13
14 /*
15  * The fs value determines whether argument validity checking should be
16  * performed or not.  If get_fs() == USER_DS, checking is performed, with
17  * get_fs() == KERNEL_DS, checking is bypassed.
18  *
19  * For historical reasons, these macros are grossly misnamed.
20  */
21
22 #define MAKE_MM_SEG(s)  ((mm_segment_t) { (s) })
23
24 #define KERNEL_DS       MAKE_MM_SEG(-1UL)
25 #define USER_DS         MAKE_MM_SEG(TASK_SIZE_MAX)
26
27 #define get_ds()        (KERNEL_DS)
28 #define get_fs()        (current->thread.addr_limit)
29 #define set_fs(x)       (current->thread.addr_limit = (x))
30
31 #define segment_eq(a, b)        ((a).seg == (b).seg)
32
33 #define user_addr_max() (current->thread.addr_limit.seg)
34 #define __addr_ok(addr)         \
35         ((unsigned long __force)(addr) < user_addr_max())
36
37 /*
38  * Test whether a block of memory is a valid user space address.
39  * Returns 0 if the range is valid, nonzero otherwise.
40  */
41 static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
42 {
43         /*
44          * If we have used "sizeof()" for the size,
45          * we know it won't overflow the limit (but
46          * it might overflow the 'addr', so it's
47          * important to subtract the size from the
48          * limit, not add it to the address).
49          */
50         if (__builtin_constant_p(size))
51                 return unlikely(addr > limit - size);
52
53         /* Arbitrary sizes? Be careful about overflow */
54         addr += size;
55         if (unlikely(addr < size))
56                 return true;
57         return unlikely(addr > limit);
58 }
59
60 #define __range_not_ok(addr, size, limit)                               \
61 ({                                                                      \
62         __chk_user_ptr(addr);                                           \
63         __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
64 })
65
66 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
67 # define WARN_ON_IN_IRQ()       WARN_ON_ONCE(!in_task())
68 #else
69 # define WARN_ON_IN_IRQ()
70 #endif
71
72 /**
73  * access_ok: - Checks if a user space pointer is valid
74  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
75  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
76  *        to write to a block, it is always safe to read from it.
77  * @addr: User space pointer to start of block to check
78  * @size: Size of block to check
79  *
80  * Context: User context only. This function may sleep if pagefaults are
81  *          enabled.
82  *
83  * Checks if a pointer to a block of memory in user space is valid.
84  *
85  * Returns true (nonzero) if the memory block may be valid, false (zero)
86  * if it is definitely invalid.
87  *
88  * Note that, depending on architecture, this function probably just
89  * checks that the pointer is in the user space range - after calling
90  * this function, memory access functions may still return -EFAULT.
91  */
92 #define access_ok(type, addr, size)                                     \
93 ({                                                                      \
94         WARN_ON_IN_IRQ();                                               \
95         likely(!__range_not_ok(addr, size, user_addr_max()));           \
96 })
97
98 /*
99  * These are the main single-value transfer routines.  They automatically
100  * use the right size if we just have the right pointer type.
101  *
102  * This gets kind of ugly. We want to return _two_ values in "get_user()"
103  * and yet we don't want to do any pointers, because that is too much
104  * of a performance impact. Thus we have a few rather ugly macros here,
105  * and hide all the ugliness from the user.
106  *
107  * The "__xxx" versions of the user access functions are versions that
108  * do not verify the address space, that must have been done previously
109  * with a separate "access_ok()" call (this is used when we do multiple
110  * accesses to the same area of user memory).
111  */
112
113 extern int __get_user_1(void);
114 extern int __get_user_2(void);
115 extern int __get_user_4(void);
116 extern int __get_user_8(void);
117 extern int __get_user_bad(void);
118
119 #define __uaccess_begin() stac()
120 #define __uaccess_end()   clac()
121
122 /*
123  * This is a type: either unsigned long, if the argument fits into
124  * that type, or otherwise unsigned long long.
125  */
126 #define __inttype(x) \
127 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
128
129 /**
130  * get_user: - Get a simple variable from user space.
131  * @x:   Variable to store result.
132  * @ptr: Source address, in user space.
133  *
134  * Context: User context only. This function may sleep if pagefaults are
135  *          enabled.
136  *
137  * This macro copies a single simple variable from user space to kernel
138  * space.  It supports simple types like char and int, but not larger
139  * data types like structures or arrays.
140  *
141  * @ptr must have pointer-to-simple-variable type, and the result of
142  * dereferencing @ptr must be assignable to @x without a cast.
143  *
144  * Returns zero on success, or -EFAULT on error.
145  * On error, the variable @x is set to zero.
146  */
147 /*
148  * Careful: we have to cast the result to the type of the pointer
149  * for sign reasons.
150  *
151  * The use of _ASM_DX as the register specifier is a bit of a
152  * simplification, as gcc only cares about it as the starting point
153  * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
154  * (%ecx being the next register in gcc's x86 register sequence), and
155  * %rdx on 64 bits.
156  *
157  * Clang/LLVM cares about the size of the register, but still wants
158  * the base register for something that ends up being a pair.
159  */
160 #define get_user(x, ptr)                                                \
161 ({                                                                      \
162         int __ret_gu;                                                   \
163         register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);            \
164         register void *__sp asm(_ASM_SP);                               \
165         __chk_user_ptr(ptr);                                            \
166         might_fault();                                                  \
167         asm volatile("call __get_user_%P4"                              \
168                      : "=a" (__ret_gu), "=r" (__val_gu), "+r" (__sp)    \
169                      : "0" (ptr), "i" (sizeof(*(ptr))));                \
170         (x) = (__force __typeof__(*(ptr))) __val_gu;                    \
171         __builtin_expect(__ret_gu, 0);                                  \
172 })
173
174 #define __put_user_x(size, x, ptr, __ret_pu)                    \
175         asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
176                      : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
177
178
179
180 #ifdef CONFIG_X86_32
181 #define __put_user_asm_u64(x, addr, err, errret)                        \
182         asm volatile("\n"                                               \
183                      "1:        movl %%eax,0(%2)\n"                     \
184                      "2:        movl %%edx,4(%2)\n"                     \
185                      "3:"                                               \
186                      ".section .fixup,\"ax\"\n"                         \
187                      "4:        movl %3,%0\n"                           \
188                      "  jmp 3b\n"                                       \
189                      ".previous\n"                                      \
190                      _ASM_EXTABLE(1b, 4b)                               \
191                      _ASM_EXTABLE(2b, 4b)                               \
192                      : "=r" (err)                                       \
193                      : "A" (x), "r" (addr), "i" (errret), "0" (err))
194
195 #define __put_user_asm_ex_u64(x, addr)                                  \
196         asm volatile("\n"                                               \
197                      "1:        movl %%eax,0(%1)\n"                     \
198                      "2:        movl %%edx,4(%1)\n"                     \
199                      "3:"                                               \
200                      _ASM_EXTABLE_EX(1b, 2b)                            \
201                      _ASM_EXTABLE_EX(2b, 3b)                            \
202                      : : "A" (x), "r" (addr))
203
204 #define __put_user_x8(x, ptr, __ret_pu)                         \
205         asm volatile("call __put_user_8" : "=a" (__ret_pu)      \
206                      : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
207 #else
208 #define __put_user_asm_u64(x, ptr, retval, errret) \
209         __put_user_asm(x, ptr, retval, "q", "", "er", errret)
210 #define __put_user_asm_ex_u64(x, addr)  \
211         __put_user_asm_ex(x, addr, "q", "", "er")
212 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
213 #endif
214
215 extern void __put_user_bad(void);
216
217 /*
218  * Strange magic calling convention: pointer in %ecx,
219  * value in %eax(:%edx), return value in %eax. clobbers %rbx
220  */
221 extern void __put_user_1(void);
222 extern void __put_user_2(void);
223 extern void __put_user_4(void);
224 extern void __put_user_8(void);
225
226 /**
227  * put_user: - Write a simple value into user space.
228  * @x:   Value to copy to user space.
229  * @ptr: Destination address, in user space.
230  *
231  * Context: User context only. This function may sleep if pagefaults are
232  *          enabled.
233  *
234  * This macro copies a single simple value from kernel space to user
235  * space.  It supports simple types like char and int, but not larger
236  * data types like structures or arrays.
237  *
238  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
239  * to the result of dereferencing @ptr.
240  *
241  * Returns zero on success, or -EFAULT on error.
242  */
243 #define put_user(x, ptr)                                        \
244 ({                                                              \
245         int __ret_pu;                                           \
246         __typeof__(*(ptr)) __pu_val;                            \
247         __chk_user_ptr(ptr);                                    \
248         might_fault();                                          \
249         __pu_val = x;                                           \
250         switch (sizeof(*(ptr))) {                               \
251         case 1:                                                 \
252                 __put_user_x(1, __pu_val, ptr, __ret_pu);       \
253                 break;                                          \
254         case 2:                                                 \
255                 __put_user_x(2, __pu_val, ptr, __ret_pu);       \
256                 break;                                          \
257         case 4:                                                 \
258                 __put_user_x(4, __pu_val, ptr, __ret_pu);       \
259                 break;                                          \
260         case 8:                                                 \
261                 __put_user_x8(__pu_val, ptr, __ret_pu);         \
262                 break;                                          \
263         default:                                                \
264                 __put_user_x(X, __pu_val, ptr, __ret_pu);       \
265                 break;                                          \
266         }                                                       \
267         __builtin_expect(__ret_pu, 0);                          \
268 })
269
270 #define __put_user_size(x, ptr, size, retval, errret)                   \
271 do {                                                                    \
272         retval = 0;                                                     \
273         __chk_user_ptr(ptr);                                            \
274         switch (size) {                                                 \
275         case 1:                                                         \
276                 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
277                 break;                                                  \
278         case 2:                                                         \
279                 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
280                 break;                                                  \
281         case 4:                                                         \
282                 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
283                 break;                                                  \
284         case 8:                                                         \
285                 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval,  \
286                                    errret);                             \
287                 break;                                                  \
288         default:                                                        \
289                 __put_user_bad();                                       \
290         }                                                               \
291 } while (0)
292
293 /*
294  * This doesn't do __uaccess_begin/end - the exception handling
295  * around it must do that.
296  */
297 #define __put_user_size_ex(x, ptr, size)                                \
298 do {                                                                    \
299         __chk_user_ptr(ptr);                                            \
300         switch (size) {                                                 \
301         case 1:                                                         \
302                 __put_user_asm_ex(x, ptr, "b", "b", "iq");              \
303                 break;                                                  \
304         case 2:                                                         \
305                 __put_user_asm_ex(x, ptr, "w", "w", "ir");              \
306                 break;                                                  \
307         case 4:                                                         \
308                 __put_user_asm_ex(x, ptr, "l", "k", "ir");              \
309                 break;                                                  \
310         case 8:                                                         \
311                 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr);      \
312                 break;                                                  \
313         default:                                                        \
314                 __put_user_bad();                                       \
315         }                                                               \
316 } while (0)
317
318 #ifdef CONFIG_X86_32
319 #define __get_user_asm_u64(x, ptr, retval, errret)                      \
320 ({                                                                      \
321         __typeof__(ptr) __ptr = (ptr);                                  \
322         asm volatile("\n"                                       \
323                      "1:        movl %2,%%eax\n"                        \
324                      "2:        movl %3,%%edx\n"                        \
325                      "3:\n"                             \
326                      ".section .fixup,\"ax\"\n"                         \
327                      "4:        mov %4,%0\n"                            \
328                      "  xorl %%eax,%%eax\n"                             \
329                      "  xorl %%edx,%%edx\n"                             \
330                      "  jmp 3b\n"                                       \
331                      ".previous\n"                                      \
332                      _ASM_EXTABLE(1b, 4b)                               \
333                      _ASM_EXTABLE(2b, 4b)                               \
334                      : "=r" (retval), "=&A"(x)                          \
335                      : "m" (__m(__ptr)), "m" __m(((u32 *)(__ptr)) + 1), \
336                        "i" (errret), "0" (retval));                     \
337 })
338
339 #define __get_user_asm_ex_u64(x, ptr)                   (x) = __get_user_bad()
340 #else
341 #define __get_user_asm_u64(x, ptr, retval, errret) \
342          __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
343 #define __get_user_asm_ex_u64(x, ptr) \
344          __get_user_asm_ex(x, ptr, "q", "", "=r")
345 #endif
346
347 #define __get_user_size(x, ptr, size, retval, errret)                   \
348 do {                                                                    \
349         retval = 0;                                                     \
350         __chk_user_ptr(ptr);                                            \
351         switch (size) {                                                 \
352         case 1:                                                         \
353                 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
354                 break;                                                  \
355         case 2:                                                         \
356                 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
357                 break;                                                  \
358         case 4:                                                         \
359                 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
360                 break;                                                  \
361         case 8:                                                         \
362                 __get_user_asm_u64(x, ptr, retval, errret);             \
363                 break;                                                  \
364         default:                                                        \
365                 (x) = __get_user_bad();                                 \
366         }                                                               \
367 } while (0)
368
369 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)       \
370         asm volatile("\n"                                               \
371                      "1:        mov"itype" %2,%"rtype"1\n"              \
372                      "2:\n"                                             \
373                      ".section .fixup,\"ax\"\n"                         \
374                      "3:        mov %3,%0\n"                            \
375                      "  xor"itype" %"rtype"1,%"rtype"1\n"               \
376                      "  jmp 2b\n"                                       \
377                      ".previous\n"                                      \
378                      _ASM_EXTABLE(1b, 3b)                               \
379                      : "=r" (err), ltype(x)                             \
380                      : "m" (__m(addr)), "i" (errret), "0" (err))
381
382 #define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret)        \
383         asm volatile("\n"                                               \
384                      "1:        mov"itype" %2,%"rtype"1\n"              \
385                      "2:\n"                                             \
386                      ".section .fixup,\"ax\"\n"                         \
387                      "3:        mov %3,%0\n"                            \
388                      "  jmp 2b\n"                                       \
389                      ".previous\n"                                      \
390                      _ASM_EXTABLE(1b, 3b)                               \
391                      : "=r" (err), ltype(x)                             \
392                      : "m" (__m(addr)), "i" (errret), "0" (err))
393
394 /*
395  * This doesn't do __uaccess_begin/end - the exception handling
396  * around it must do that.
397  */
398 #define __get_user_size_ex(x, ptr, size)                                \
399 do {                                                                    \
400         __chk_user_ptr(ptr);                                            \
401         switch (size) {                                                 \
402         case 1:                                                         \
403                 __get_user_asm_ex(x, ptr, "b", "b", "=q");              \
404                 break;                                                  \
405         case 2:                                                         \
406                 __get_user_asm_ex(x, ptr, "w", "w", "=r");              \
407                 break;                                                  \
408         case 4:                                                         \
409                 __get_user_asm_ex(x, ptr, "l", "k", "=r");              \
410                 break;                                                  \
411         case 8:                                                         \
412                 __get_user_asm_ex_u64(x, ptr);                          \
413                 break;                                                  \
414         default:                                                        \
415                 (x) = __get_user_bad();                                 \
416         }                                                               \
417 } while (0)
418
419 #define __get_user_asm_ex(x, addr, itype, rtype, ltype)                 \
420         asm volatile("1:        mov"itype" %1,%"rtype"0\n"              \
421                      "2:\n"                                             \
422                      ".section .fixup,\"ax\"\n"                         \
423                      "3:xor"itype" %"rtype"0,%"rtype"0\n"               \
424                      "  jmp 2b\n"                                       \
425                      ".previous\n"                                      \
426                      _ASM_EXTABLE_EX(1b, 3b)                            \
427                      : ltype(x) : "m" (__m(addr)))
428
429 #define __put_user_nocheck(x, ptr, size)                        \
430 ({                                                              \
431         int __pu_err;                                           \
432         __uaccess_begin();                                      \
433         __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
434         __uaccess_end();                                        \
435         __builtin_expect(__pu_err, 0);                          \
436 })
437
438 #define __get_user_nocheck(x, ptr, size)                                \
439 ({                                                                      \
440         int __gu_err;                                                   \
441         __inttype(*(ptr)) __gu_val;                                     \
442         __uaccess_begin();                                              \
443         __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);    \
444         __uaccess_end();                                                \
445         (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
446         __builtin_expect(__gu_err, 0);                                  \
447 })
448
449 /* FIXME: this hack is definitely wrong -AK */
450 struct __large_struct { unsigned long buf[100]; };
451 #define __m(x) (*(struct __large_struct __user *)(x))
452
453 /*
454  * Tell gcc we read from memory instead of writing: this is because
455  * we do not write to any memory gcc knows about, so there are no
456  * aliasing issues.
457  */
458 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)       \
459         asm volatile("\n"                                               \
460                      "1:        mov"itype" %"rtype"1,%2\n"              \
461                      "2:\n"                                             \
462                      ".section .fixup,\"ax\"\n"                         \
463                      "3:        mov %3,%0\n"                            \
464                      "  jmp 2b\n"                                       \
465                      ".previous\n"                                      \
466                      _ASM_EXTABLE(1b, 3b)                               \
467                      : "=r"(err)                                        \
468                      : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
469
470 #define __put_user_asm_ex(x, addr, itype, rtype, ltype)                 \
471         asm volatile("1:        mov"itype" %"rtype"0,%1\n"              \
472                      "2:\n"                                             \
473                      _ASM_EXTABLE_EX(1b, 2b)                            \
474                      : : ltype(x), "m" (__m(addr)))
475
476 /*
477  * uaccess_try and catch
478  */
479 #define uaccess_try     do {                                            \
480         current->thread.uaccess_err = 0;                                \
481         __uaccess_begin();                                              \
482         barrier();
483
484 #define uaccess_catch(err)                                              \
485         __uaccess_end();                                                \
486         (err) |= (current->thread.uaccess_err ? -EFAULT : 0);           \
487 } while (0)
488
489 /**
490  * __get_user: - Get a simple variable from user space, with less checking.
491  * @x:   Variable to store result.
492  * @ptr: Source address, in user space.
493  *
494  * Context: User context only. This function may sleep if pagefaults are
495  *          enabled.
496  *
497  * This macro copies a single simple variable from user space to kernel
498  * space.  It supports simple types like char and int, but not larger
499  * data types like structures or arrays.
500  *
501  * @ptr must have pointer-to-simple-variable type, and the result of
502  * dereferencing @ptr must be assignable to @x without a cast.
503  *
504  * Caller must check the pointer with access_ok() before calling this
505  * function.
506  *
507  * Returns zero on success, or -EFAULT on error.
508  * On error, the variable @x is set to zero.
509  */
510
511 #define __get_user(x, ptr)                                              \
512         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
513
514 /**
515  * __put_user: - Write a simple value into user space, with less checking.
516  * @x:   Value to copy to user space.
517  * @ptr: Destination address, in user space.
518  *
519  * Context: User context only. This function may sleep if pagefaults are
520  *          enabled.
521  *
522  * This macro copies a single simple value from kernel space to user
523  * space.  It supports simple types like char and int, but not larger
524  * data types like structures or arrays.
525  *
526  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
527  * to the result of dereferencing @ptr.
528  *
529  * Caller must check the pointer with access_ok() before calling this
530  * function.
531  *
532  * Returns zero on success, or -EFAULT on error.
533  */
534
535 #define __put_user(x, ptr)                                              \
536         __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
537
538 #define __get_user_unaligned __get_user
539 #define __put_user_unaligned __put_user
540
541 /*
542  * {get|put}_user_try and catch
543  *
544  * get_user_try {
545  *      get_user_ex(...);
546  * } get_user_catch(err)
547  */
548 #define get_user_try            uaccess_try
549 #define get_user_catch(err)     uaccess_catch(err)
550
551 #define get_user_ex(x, ptr)     do {                                    \
552         unsigned long __gue_val;                                        \
553         __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr))));       \
554         (x) = (__force __typeof__(*(ptr)))__gue_val;                    \
555 } while (0)
556
557 #define put_user_try            uaccess_try
558 #define put_user_catch(err)     uaccess_catch(err)
559
560 #define put_user_ex(x, ptr)                                             \
561         __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
562
563 extern unsigned long
564 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
565 extern __must_check long
566 strncpy_from_user(char *dst, const char __user *src, long count);
567
568 extern __must_check long strlen_user(const char __user *str);
569 extern __must_check long strnlen_user(const char __user *str, long n);
570
571 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
572 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
573
574 extern void __cmpxchg_wrong_size(void)
575         __compiletime_error("Bad argument size for cmpxchg");
576
577 #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size)       \
578 ({                                                                      \
579         int __ret = 0;                                                  \
580         __typeof__(ptr) __uval = (uval);                                \
581         __typeof__(*(ptr)) __old = (old);                               \
582         __typeof__(*(ptr)) __new = (new);                               \
583         __uaccess_begin();                                              \
584         switch (size) {                                                 \
585         case 1:                                                         \
586         {                                                               \
587                 asm volatile("\n"                                       \
588                         "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n"          \
589                         "2:\n"                                          \
590                         "\t.section .fixup, \"ax\"\n"                   \
591                         "3:\tmov     %3, %0\n"                          \
592                         "\tjmp     2b\n"                                \
593                         "\t.previous\n"                                 \
594                         _ASM_EXTABLE(1b, 3b)                            \
595                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
596                         : "i" (-EFAULT), "q" (__new), "1" (__old)       \
597                         : "memory"                                      \
598                 );                                                      \
599                 break;                                                  \
600         }                                                               \
601         case 2:                                                         \
602         {                                                               \
603                 asm volatile("\n"                                       \
604                         "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n"          \
605                         "2:\n"                                          \
606                         "\t.section .fixup, \"ax\"\n"                   \
607                         "3:\tmov     %3, %0\n"                          \
608                         "\tjmp     2b\n"                                \
609                         "\t.previous\n"                                 \
610                         _ASM_EXTABLE(1b, 3b)                            \
611                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
612                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
613                         : "memory"                                      \
614                 );                                                      \
615                 break;                                                  \
616         }                                                               \
617         case 4:                                                         \
618         {                                                               \
619                 asm volatile("\n"                                       \
620                         "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"          \
621                         "2:\n"                                          \
622                         "\t.section .fixup, \"ax\"\n"                   \
623                         "3:\tmov     %3, %0\n"                          \
624                         "\tjmp     2b\n"                                \
625                         "\t.previous\n"                                 \
626                         _ASM_EXTABLE(1b, 3b)                            \
627                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
628                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
629                         : "memory"                                      \
630                 );                                                      \
631                 break;                                                  \
632         }                                                               \
633         case 8:                                                         \
634         {                                                               \
635                 if (!IS_ENABLED(CONFIG_X86_64))                         \
636                         __cmpxchg_wrong_size();                         \
637                                                                         \
638                 asm volatile("\n"                                       \
639                         "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n"          \
640                         "2:\n"                                          \
641                         "\t.section .fixup, \"ax\"\n"                   \
642                         "3:\tmov     %3, %0\n"                          \
643                         "\tjmp     2b\n"                                \
644                         "\t.previous\n"                                 \
645                         _ASM_EXTABLE(1b, 3b)                            \
646                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
647                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
648                         : "memory"                                      \
649                 );                                                      \
650                 break;                                                  \
651         }                                                               \
652         default:                                                        \
653                 __cmpxchg_wrong_size();                                 \
654         }                                                               \
655         __uaccess_end();                                                \
656         *__uval = __old;                                                \
657         __ret;                                                          \
658 })
659
660 #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new)               \
661 ({                                                                      \
662         access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ?                \
663                 __user_atomic_cmpxchg_inatomic((uval), (ptr),           \
664                                 (old), (new), sizeof(*(ptr))) :         \
665                 -EFAULT;                                                \
666 })
667
668 /*
669  * movsl can be slow when source and dest are not both 8-byte aligned
670  */
671 #ifdef CONFIG_X86_INTEL_USERCOPY
672 extern struct movsl_mask {
673         int mask;
674 } ____cacheline_aligned_in_smp movsl_mask;
675 #endif
676
677 #define ARCH_HAS_NOCACHE_UACCESS 1
678
679 #ifdef CONFIG_X86_32
680 # include <asm/uaccess_32.h>
681 #else
682 # include <asm/uaccess_64.h>
683 #endif
684
685 /*
686  * We rely on the nested NMI work to allow atomic faults from the NMI path; the
687  * nested NMI paths are careful to preserve CR2.
688  *
689  * Caller must use pagefault_enable/disable, or run in interrupt context,
690  * and also do a uaccess_ok() check
691  */
692 #define __copy_from_user_nmi __copy_from_user_inatomic
693
694 /*
695  * The "unsafe" user accesses aren't really "unsafe", but the naming
696  * is a big fat warning: you have to not only do the access_ok()
697  * checking before using them, but you have to surround them with the
698  * user_access_begin/end() pair.
699  */
700 #define user_access_begin()     __uaccess_begin()
701 #define user_access_end()       __uaccess_end()
702
703 #define unsafe_put_user(x, ptr, err_label)                                      \
704 do {                                                                            \
705         int __pu_err;                                                           \
706         __typeof__(*(ptr)) __pu_val = (x);                                      \
707         __put_user_size(__pu_val, (ptr), sizeof(*(ptr)), __pu_err, -EFAULT);    \
708         if (unlikely(__pu_err)) goto err_label;                                 \
709 } while (0)
710
711 #define unsafe_get_user(x, ptr, err_label)                                      \
712 do {                                                                            \
713         int __gu_err;                                                           \
714         __inttype(*(ptr)) __gu_val;                                             \
715         __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT);    \
716         (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
717         if (unlikely(__gu_err)) goto err_label;                                 \
718 } while (0)
719
720 #endif /* _ASM_X86_UACCESS_H */
721