x86/mm/32: Add support for 64-bit __get_user() on 32-bit kernels
[sfrench/cifs-2.6.git] / arch / x86 / include / asm / uaccess.h
1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
3 /*
4  * User space memory access functions
5  */
6 #include <linux/errno.h>
7 #include <linux/compiler.h>
8 #include <linux/thread_info.h>
9 #include <linux/string.h>
10 #include <asm/asm.h>
11 #include <asm/page.h>
12 #include <asm/smap.h>
13
14 #define VERIFY_READ 0
15 #define VERIFY_WRITE 1
16
17 /*
18  * The fs value determines whether argument validity checking should be
19  * performed or not.  If get_fs() == USER_DS, checking is performed, with
20  * get_fs() == KERNEL_DS, checking is bypassed.
21  *
22  * For historical reasons, these macros are grossly misnamed.
23  */
24
25 #define MAKE_MM_SEG(s)  ((mm_segment_t) { (s) })
26
27 #define KERNEL_DS       MAKE_MM_SEG(-1UL)
28 #define USER_DS         MAKE_MM_SEG(TASK_SIZE_MAX)
29
30 #define get_ds()        (KERNEL_DS)
31 #define get_fs()        (current_thread_info()->addr_limit)
32 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
33
34 #define segment_eq(a, b)        ((a).seg == (b).seg)
35
36 #define user_addr_max() (current_thread_info()->addr_limit.seg)
37 #define __addr_ok(addr)         \
38         ((unsigned long __force)(addr) < user_addr_max())
39
40 /*
41  * Test whether a block of memory is a valid user space address.
42  * Returns 0 if the range is valid, nonzero otherwise.
43  */
44 static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
45 {
46         /*
47          * If we have used "sizeof()" for the size,
48          * we know it won't overflow the limit (but
49          * it might overflow the 'addr', so it's
50          * important to subtract the size from the
51          * limit, not add it to the address).
52          */
53         if (__builtin_constant_p(size))
54                 return unlikely(addr > limit - size);
55
56         /* Arbitrary sizes? Be careful about overflow */
57         addr += size;
58         if (unlikely(addr < size))
59                 return true;
60         return unlikely(addr > limit);
61 }
62
63 #define __range_not_ok(addr, size, limit)                               \
64 ({                                                                      \
65         __chk_user_ptr(addr);                                           \
66         __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
67 })
68
69 /**
70  * access_ok: - Checks if a user space pointer is valid
71  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
72  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
73  *        to write to a block, it is always safe to read from it.
74  * @addr: User space pointer to start of block to check
75  * @size: Size of block to check
76  *
77  * Context: User context only. This function may sleep if pagefaults are
78  *          enabled.
79  *
80  * Checks if a pointer to a block of memory in user space is valid.
81  *
82  * Returns true (nonzero) if the memory block may be valid, false (zero)
83  * if it is definitely invalid.
84  *
85  * Note that, depending on architecture, this function probably just
86  * checks that the pointer is in the user space range - after calling
87  * this function, memory access functions may still return -EFAULT.
88  */
89 #define access_ok(type, addr, size) \
90         likely(!__range_not_ok(addr, size, user_addr_max()))
91
92 /*
93  * The exception table consists of triples of addresses relative to the
94  * exception table entry itself. The first address is of an instruction
95  * that is allowed to fault, the second is the target at which the program
96  * should continue. The third is a handler function to deal with the fault
97  * caused by the instruction in the first field.
98  *
99  * All the routines below use bits of fixup code that are out of line
100  * with the main instruction path.  This means when everything is well,
101  * we don't even have to jump over them.  Further, they do not intrude
102  * on our cache or tlb entries.
103  */
104
105 struct exception_table_entry {
106         int insn, fixup, handler;
107 };
108
109 #define ARCH_HAS_RELATIVE_EXTABLE
110
111 extern int fixup_exception(struct pt_regs *regs, int trapnr);
112 extern bool ex_has_fault_handler(unsigned long ip);
113 extern int early_fixup_exception(unsigned long *ip);
114
115 /*
116  * These are the main single-value transfer routines.  They automatically
117  * use the right size if we just have the right pointer type.
118  *
119  * This gets kind of ugly. We want to return _two_ values in "get_user()"
120  * and yet we don't want to do any pointers, because that is too much
121  * of a performance impact. Thus we have a few rather ugly macros here,
122  * and hide all the ugliness from the user.
123  *
124  * The "__xxx" versions of the user access functions are versions that
125  * do not verify the address space, that must have been done previously
126  * with a separate "access_ok()" call (this is used when we do multiple
127  * accesses to the same area of user memory).
128  */
129
130 extern int __get_user_1(void);
131 extern int __get_user_2(void);
132 extern int __get_user_4(void);
133 extern int __get_user_8(void);
134 extern int __get_user_bad(void);
135
136 #define __uaccess_begin() stac()
137 #define __uaccess_end()   clac()
138
139 /*
140  * This is a type: either unsigned long, if the argument fits into
141  * that type, or otherwise unsigned long long.
142  */
143 #define __inttype(x) \
144 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
145
146 /**
147  * get_user: - Get a simple variable from user space.
148  * @x:   Variable to store result.
149  * @ptr: Source address, in user space.
150  *
151  * Context: User context only. This function may sleep if pagefaults are
152  *          enabled.
153  *
154  * This macro copies a single simple variable from user space to kernel
155  * space.  It supports simple types like char and int, but not larger
156  * data types like structures or arrays.
157  *
158  * @ptr must have pointer-to-simple-variable type, and the result of
159  * dereferencing @ptr must be assignable to @x without a cast.
160  *
161  * Returns zero on success, or -EFAULT on error.
162  * On error, the variable @x is set to zero.
163  */
164 /*
165  * Careful: we have to cast the result to the type of the pointer
166  * for sign reasons.
167  *
168  * The use of _ASM_DX as the register specifier is a bit of a
169  * simplification, as gcc only cares about it as the starting point
170  * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
171  * (%ecx being the next register in gcc's x86 register sequence), and
172  * %rdx on 64 bits.
173  *
174  * Clang/LLVM cares about the size of the register, but still wants
175  * the base register for something that ends up being a pair.
176  */
177 #define get_user(x, ptr)                                                \
178 ({                                                                      \
179         int __ret_gu;                                                   \
180         register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);            \
181         register void *__sp asm(_ASM_SP);                               \
182         __chk_user_ptr(ptr);                                            \
183         might_fault();                                                  \
184         asm volatile("call __get_user_%P4"                              \
185                      : "=a" (__ret_gu), "=r" (__val_gu), "+r" (__sp)    \
186                      : "0" (ptr), "i" (sizeof(*(ptr))));                \
187         (x) = (__force __typeof__(*(ptr))) __val_gu;                    \
188         __builtin_expect(__ret_gu, 0);                                  \
189 })
190
191 #define __put_user_x(size, x, ptr, __ret_pu)                    \
192         asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
193                      : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
194
195
196
197 #ifdef CONFIG_X86_32
198 #define __put_user_asm_u64(x, addr, err, errret)                        \
199         asm volatile("\n"                                               \
200                      "1:        movl %%eax,0(%2)\n"                     \
201                      "2:        movl %%edx,4(%2)\n"                     \
202                      "3:"                                               \
203                      ".section .fixup,\"ax\"\n"                         \
204                      "4:        movl %3,%0\n"                           \
205                      "  jmp 3b\n"                                       \
206                      ".previous\n"                                      \
207                      _ASM_EXTABLE(1b, 4b)                               \
208                      _ASM_EXTABLE(2b, 4b)                               \
209                      : "=r" (err)                                       \
210                      : "A" (x), "r" (addr), "i" (errret), "0" (err))
211
212 #define __put_user_asm_ex_u64(x, addr)                                  \
213         asm volatile("\n"                                               \
214                      "1:        movl %%eax,0(%1)\n"                     \
215                      "2:        movl %%edx,4(%1)\n"                     \
216                      "3:"                                               \
217                      _ASM_EXTABLE_EX(1b, 2b)                            \
218                      _ASM_EXTABLE_EX(2b, 3b)                            \
219                      : : "A" (x), "r" (addr))
220
221 #define __put_user_x8(x, ptr, __ret_pu)                         \
222         asm volatile("call __put_user_8" : "=a" (__ret_pu)      \
223                      : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
224 #else
225 #define __put_user_asm_u64(x, ptr, retval, errret) \
226         __put_user_asm(x, ptr, retval, "q", "", "er", errret)
227 #define __put_user_asm_ex_u64(x, addr)  \
228         __put_user_asm_ex(x, addr, "q", "", "er")
229 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
230 #endif
231
232 extern void __put_user_bad(void);
233
234 /*
235  * Strange magic calling convention: pointer in %ecx,
236  * value in %eax(:%edx), return value in %eax. clobbers %rbx
237  */
238 extern void __put_user_1(void);
239 extern void __put_user_2(void);
240 extern void __put_user_4(void);
241 extern void __put_user_8(void);
242
243 /**
244  * put_user: - Write a simple value into user space.
245  * @x:   Value to copy to user space.
246  * @ptr: Destination address, in user space.
247  *
248  * Context: User context only. This function may sleep if pagefaults are
249  *          enabled.
250  *
251  * This macro copies a single simple value from kernel space to user
252  * space.  It supports simple types like char and int, but not larger
253  * data types like structures or arrays.
254  *
255  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
256  * to the result of dereferencing @ptr.
257  *
258  * Returns zero on success, or -EFAULT on error.
259  */
260 #define put_user(x, ptr)                                        \
261 ({                                                              \
262         int __ret_pu;                                           \
263         __typeof__(*(ptr)) __pu_val;                            \
264         __chk_user_ptr(ptr);                                    \
265         might_fault();                                          \
266         __pu_val = x;                                           \
267         switch (sizeof(*(ptr))) {                               \
268         case 1:                                                 \
269                 __put_user_x(1, __pu_val, ptr, __ret_pu);       \
270                 break;                                          \
271         case 2:                                                 \
272                 __put_user_x(2, __pu_val, ptr, __ret_pu);       \
273                 break;                                          \
274         case 4:                                                 \
275                 __put_user_x(4, __pu_val, ptr, __ret_pu);       \
276                 break;                                          \
277         case 8:                                                 \
278                 __put_user_x8(__pu_val, ptr, __ret_pu);         \
279                 break;                                          \
280         default:                                                \
281                 __put_user_x(X, __pu_val, ptr, __ret_pu);       \
282                 break;                                          \
283         }                                                       \
284         __builtin_expect(__ret_pu, 0);                          \
285 })
286
287 #define __put_user_size(x, ptr, size, retval, errret)                   \
288 do {                                                                    \
289         retval = 0;                                                     \
290         __chk_user_ptr(ptr);                                            \
291         switch (size) {                                                 \
292         case 1:                                                         \
293                 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
294                 break;                                                  \
295         case 2:                                                         \
296                 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
297                 break;                                                  \
298         case 4:                                                         \
299                 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
300                 break;                                                  \
301         case 8:                                                         \
302                 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval,  \
303                                    errret);                             \
304                 break;                                                  \
305         default:                                                        \
306                 __put_user_bad();                                       \
307         }                                                               \
308 } while (0)
309
310 /*
311  * This doesn't do __uaccess_begin/end - the exception handling
312  * around it must do that.
313  */
314 #define __put_user_size_ex(x, ptr, size)                                \
315 do {                                                                    \
316         __chk_user_ptr(ptr);                                            \
317         switch (size) {                                                 \
318         case 1:                                                         \
319                 __put_user_asm_ex(x, ptr, "b", "b", "iq");              \
320                 break;                                                  \
321         case 2:                                                         \
322                 __put_user_asm_ex(x, ptr, "w", "w", "ir");              \
323                 break;                                                  \
324         case 4:                                                         \
325                 __put_user_asm_ex(x, ptr, "l", "k", "ir");              \
326                 break;                                                  \
327         case 8:                                                         \
328                 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr);      \
329                 break;                                                  \
330         default:                                                        \
331                 __put_user_bad();                                       \
332         }                                                               \
333 } while (0)
334
335 #ifdef CONFIG_X86_32
336 #define __get_user_asm_u64(x, ptr, retval, errret)                      \
337 ({                                                                      \
338         __typeof__(ptr) __ptr = (ptr);                                  \
339         asm volatile(ASM_STAC "\n"                                      \
340                      "1:        movl %2,%%eax\n"                        \
341                      "2:        movl %3,%%edx\n"                        \
342                      "3: " ASM_CLAC "\n"                                \
343                      ".section .fixup,\"ax\"\n"                         \
344                      "4:        mov %4,%0\n"                            \
345                      "  xorl %%eax,%%eax\n"                             \
346                      "  xorl %%edx,%%edx\n"                             \
347                      "  jmp 3b\n"                                       \
348                      ".previous\n"                                      \
349                      _ASM_EXTABLE(1b, 4b)                               \
350                      _ASM_EXTABLE(2b, 4b)                               \
351                      : "=r" (retval), "=A"(x)                           \
352                      : "m" (__m(__ptr)), "m" __m(((u32 *)(__ptr)) + 1), \
353                        "i" (errret), "0" (retval));                     \
354 })
355
356 #define __get_user_asm_ex_u64(x, ptr)                   (x) = __get_user_bad()
357 #else
358 #define __get_user_asm_u64(x, ptr, retval, errret) \
359          __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
360 #define __get_user_asm_ex_u64(x, ptr) \
361          __get_user_asm_ex(x, ptr, "q", "", "=r")
362 #endif
363
364 #define __get_user_size(x, ptr, size, retval, errret)                   \
365 do {                                                                    \
366         retval = 0;                                                     \
367         __chk_user_ptr(ptr);                                            \
368         switch (size) {                                                 \
369         case 1:                                                         \
370                 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
371                 break;                                                  \
372         case 2:                                                         \
373                 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
374                 break;                                                  \
375         case 4:                                                         \
376                 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
377                 break;                                                  \
378         case 8:                                                         \
379                 __get_user_asm_u64(x, ptr, retval, errret);             \
380                 break;                                                  \
381         default:                                                        \
382                 (x) = __get_user_bad();                                 \
383         }                                                               \
384 } while (0)
385
386 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)       \
387         asm volatile("\n"                                               \
388                      "1:        mov"itype" %2,%"rtype"1\n"              \
389                      "2:\n"                                             \
390                      ".section .fixup,\"ax\"\n"                         \
391                      "3:        mov %3,%0\n"                            \
392                      "  xor"itype" %"rtype"1,%"rtype"1\n"               \
393                      "  jmp 2b\n"                                       \
394                      ".previous\n"                                      \
395                      _ASM_EXTABLE(1b, 3b)                               \
396                      : "=r" (err), ltype(x)                             \
397                      : "m" (__m(addr)), "i" (errret), "0" (err))
398
399 /*
400  * This doesn't do __uaccess_begin/end - the exception handling
401  * around it must do that.
402  */
403 #define __get_user_size_ex(x, ptr, size)                                \
404 do {                                                                    \
405         __chk_user_ptr(ptr);                                            \
406         switch (size) {                                                 \
407         case 1:                                                         \
408                 __get_user_asm_ex(x, ptr, "b", "b", "=q");              \
409                 break;                                                  \
410         case 2:                                                         \
411                 __get_user_asm_ex(x, ptr, "w", "w", "=r");              \
412                 break;                                                  \
413         case 4:                                                         \
414                 __get_user_asm_ex(x, ptr, "l", "k", "=r");              \
415                 break;                                                  \
416         case 8:                                                         \
417                 __get_user_asm_ex_u64(x, ptr);                          \
418                 break;                                                  \
419         default:                                                        \
420                 (x) = __get_user_bad();                                 \
421         }                                                               \
422 } while (0)
423
424 #define __get_user_asm_ex(x, addr, itype, rtype, ltype)                 \
425         asm volatile("1:        mov"itype" %1,%"rtype"0\n"              \
426                      "2:\n"                                             \
427                      _ASM_EXTABLE_EX(1b, 2b)                            \
428                      : ltype(x) : "m" (__m(addr)))
429
430 #define __put_user_nocheck(x, ptr, size)                        \
431 ({                                                              \
432         int __pu_err;                                           \
433         __uaccess_begin();                                      \
434         __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
435         __uaccess_end();                                        \
436         __builtin_expect(__pu_err, 0);                          \
437 })
438
439 #define __get_user_nocheck(x, ptr, size)                                \
440 ({                                                                      \
441         int __gu_err;                                                   \
442         __inttype(*(ptr)) __gu_val;                                     \
443         __uaccess_begin();                                              \
444         __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);    \
445         __uaccess_end();                                                \
446         (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
447         __builtin_expect(__gu_err, 0);                                  \
448 })
449
450 /* FIXME: this hack is definitely wrong -AK */
451 struct __large_struct { unsigned long buf[100]; };
452 #define __m(x) (*(struct __large_struct __user *)(x))
453
454 /*
455  * Tell gcc we read from memory instead of writing: this is because
456  * we do not write to any memory gcc knows about, so there are no
457  * aliasing issues.
458  */
459 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)       \
460         asm volatile("\n"                                               \
461                      "1:        mov"itype" %"rtype"1,%2\n"              \
462                      "2:\n"                                             \
463                      ".section .fixup,\"ax\"\n"                         \
464                      "3:        mov %3,%0\n"                            \
465                      "  jmp 2b\n"                                       \
466                      ".previous\n"                                      \
467                      _ASM_EXTABLE(1b, 3b)                               \
468                      : "=r"(err)                                        \
469                      : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
470
471 #define __put_user_asm_ex(x, addr, itype, rtype, ltype)                 \
472         asm volatile("1:        mov"itype" %"rtype"0,%1\n"              \
473                      "2:\n"                                             \
474                      _ASM_EXTABLE_EX(1b, 2b)                            \
475                      : : ltype(x), "m" (__m(addr)))
476
477 /*
478  * uaccess_try and catch
479  */
480 #define uaccess_try     do {                                            \
481         current_thread_info()->uaccess_err = 0;                         \
482         __uaccess_begin();                                              \
483         barrier();
484
485 #define uaccess_catch(err)                                              \
486         __uaccess_end();                                                \
487         (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0);    \
488 } while (0)
489
490 /**
491  * __get_user: - Get a simple variable from user space, with less checking.
492  * @x:   Variable to store result.
493  * @ptr: Source address, in user space.
494  *
495  * Context: User context only. This function may sleep if pagefaults are
496  *          enabled.
497  *
498  * This macro copies a single simple variable from user space to kernel
499  * space.  It supports simple types like char and int, but not larger
500  * data types like structures or arrays.
501  *
502  * @ptr must have pointer-to-simple-variable type, and the result of
503  * dereferencing @ptr must be assignable to @x without a cast.
504  *
505  * Caller must check the pointer with access_ok() before calling this
506  * function.
507  *
508  * Returns zero on success, or -EFAULT on error.
509  * On error, the variable @x is set to zero.
510  */
511
512 #define __get_user(x, ptr)                                              \
513         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
514
515 /**
516  * __put_user: - Write a simple value into user space, with less checking.
517  * @x:   Value to copy to user space.
518  * @ptr: Destination address, in user space.
519  *
520  * Context: User context only. This function may sleep if pagefaults are
521  *          enabled.
522  *
523  * This macro copies a single simple value from kernel space to user
524  * space.  It supports simple types like char and int, but not larger
525  * data types like structures or arrays.
526  *
527  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
528  * to the result of dereferencing @ptr.
529  *
530  * Caller must check the pointer with access_ok() before calling this
531  * function.
532  *
533  * Returns zero on success, or -EFAULT on error.
534  */
535
536 #define __put_user(x, ptr)                                              \
537         __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
538
539 #define __get_user_unaligned __get_user
540 #define __put_user_unaligned __put_user
541
542 /*
543  * {get|put}_user_try and catch
544  *
545  * get_user_try {
546  *      get_user_ex(...);
547  * } get_user_catch(err)
548  */
549 #define get_user_try            uaccess_try
550 #define get_user_catch(err)     uaccess_catch(err)
551
552 #define get_user_ex(x, ptr)     do {                                    \
553         unsigned long __gue_val;                                        \
554         __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr))));       \
555         (x) = (__force __typeof__(*(ptr)))__gue_val;                    \
556 } while (0)
557
558 #define put_user_try            uaccess_try
559 #define put_user_catch(err)     uaccess_catch(err)
560
561 #define put_user_ex(x, ptr)                                             \
562         __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
563
564 extern unsigned long
565 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
566 extern __must_check long
567 strncpy_from_user(char *dst, const char __user *src, long count);
568
569 extern __must_check long strlen_user(const char __user *str);
570 extern __must_check long strnlen_user(const char __user *str, long n);
571
572 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
573 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
574
575 extern void __cmpxchg_wrong_size(void)
576         __compiletime_error("Bad argument size for cmpxchg");
577
578 #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size)       \
579 ({                                                                      \
580         int __ret = 0;                                                  \
581         __typeof__(ptr) __uval = (uval);                                \
582         __typeof__(*(ptr)) __old = (old);                               \
583         __typeof__(*(ptr)) __new = (new);                               \
584         __uaccess_begin();                                              \
585         switch (size) {                                                 \
586         case 1:                                                         \
587         {                                                               \
588                 asm volatile("\n"                                       \
589                         "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n"          \
590                         "2:\n"                                          \
591                         "\t.section .fixup, \"ax\"\n"                   \
592                         "3:\tmov     %3, %0\n"                          \
593                         "\tjmp     2b\n"                                \
594                         "\t.previous\n"                                 \
595                         _ASM_EXTABLE(1b, 3b)                            \
596                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
597                         : "i" (-EFAULT), "q" (__new), "1" (__old)       \
598                         : "memory"                                      \
599                 );                                                      \
600                 break;                                                  \
601         }                                                               \
602         case 2:                                                         \
603         {                                                               \
604                 asm volatile("\n"                                       \
605                         "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n"          \
606                         "2:\n"                                          \
607                         "\t.section .fixup, \"ax\"\n"                   \
608                         "3:\tmov     %3, %0\n"                          \
609                         "\tjmp     2b\n"                                \
610                         "\t.previous\n"                                 \
611                         _ASM_EXTABLE(1b, 3b)                            \
612                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
613                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
614                         : "memory"                                      \
615                 );                                                      \
616                 break;                                                  \
617         }                                                               \
618         case 4:                                                         \
619         {                                                               \
620                 asm volatile("\n"                                       \
621                         "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"          \
622                         "2:\n"                                          \
623                         "\t.section .fixup, \"ax\"\n"                   \
624                         "3:\tmov     %3, %0\n"                          \
625                         "\tjmp     2b\n"                                \
626                         "\t.previous\n"                                 \
627                         _ASM_EXTABLE(1b, 3b)                            \
628                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
629                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
630                         : "memory"                                      \
631                 );                                                      \
632                 break;                                                  \
633         }                                                               \
634         case 8:                                                         \
635         {                                                               \
636                 if (!IS_ENABLED(CONFIG_X86_64))                         \
637                         __cmpxchg_wrong_size();                         \
638                                                                         \
639                 asm volatile("\n"                                       \
640                         "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n"          \
641                         "2:\n"                                          \
642                         "\t.section .fixup, \"ax\"\n"                   \
643                         "3:\tmov     %3, %0\n"                          \
644                         "\tjmp     2b\n"                                \
645                         "\t.previous\n"                                 \
646                         _ASM_EXTABLE(1b, 3b)                            \
647                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
648                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
649                         : "memory"                                      \
650                 );                                                      \
651                 break;                                                  \
652         }                                                               \
653         default:                                                        \
654                 __cmpxchg_wrong_size();                                 \
655         }                                                               \
656         __uaccess_end();                                                \
657         *__uval = __old;                                                \
658         __ret;                                                          \
659 })
660
661 #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new)               \
662 ({                                                                      \
663         access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ?                \
664                 __user_atomic_cmpxchg_inatomic((uval), (ptr),           \
665                                 (old), (new), sizeof(*(ptr))) :         \
666                 -EFAULT;                                                \
667 })
668
669 /*
670  * movsl can be slow when source and dest are not both 8-byte aligned
671  */
672 #ifdef CONFIG_X86_INTEL_USERCOPY
673 extern struct movsl_mask {
674         int mask;
675 } ____cacheline_aligned_in_smp movsl_mask;
676 #endif
677
678 #define ARCH_HAS_NOCACHE_UACCESS 1
679
680 #ifdef CONFIG_X86_32
681 # include <asm/uaccess_32.h>
682 #else
683 # include <asm/uaccess_64.h>
684 #endif
685
686 unsigned long __must_check _copy_from_user(void *to, const void __user *from,
687                                            unsigned n);
688 unsigned long __must_check _copy_to_user(void __user *to, const void *from,
689                                          unsigned n);
690
691 #ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
692 # define copy_user_diag __compiletime_error
693 #else
694 # define copy_user_diag __compiletime_warning
695 #endif
696
697 extern void copy_user_diag("copy_from_user() buffer size is too small")
698 copy_from_user_overflow(void);
699 extern void copy_user_diag("copy_to_user() buffer size is too small")
700 copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
701
702 #undef copy_user_diag
703
704 #ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
705
706 extern void
707 __compiletime_warning("copy_from_user() buffer size is not provably correct")
708 __copy_from_user_overflow(void) __asm__("copy_from_user_overflow");
709 #define __copy_from_user_overflow(size, count) __copy_from_user_overflow()
710
711 extern void
712 __compiletime_warning("copy_to_user() buffer size is not provably correct")
713 __copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
714 #define __copy_to_user_overflow(size, count) __copy_to_user_overflow()
715
716 #else
717
718 static inline void
719 __copy_from_user_overflow(int size, unsigned long count)
720 {
721         WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
722 }
723
724 #define __copy_to_user_overflow __copy_from_user_overflow
725
726 #endif
727
728 static inline unsigned long __must_check
729 copy_from_user(void *to, const void __user *from, unsigned long n)
730 {
731         int sz = __compiletime_object_size(to);
732
733         might_fault();
734
735         /*
736          * While we would like to have the compiler do the checking for us
737          * even in the non-constant size case, any false positives there are
738          * a problem (especially when DEBUG_STRICT_USER_COPY_CHECKS, but even
739          * without - the [hopefully] dangerous looking nature of the warning
740          * would make people go look at the respecitive call sites over and
741          * over again just to find that there's no problem).
742          *
743          * And there are cases where it's just not realistic for the compiler
744          * to prove the count to be in range. For example when multiple call
745          * sites of a helper function - perhaps in different source files -
746          * all doing proper range checking, yet the helper function not doing
747          * so again.
748          *
749          * Therefore limit the compile time checking to the constant size
750          * case, and do only runtime checking for non-constant sizes.
751          */
752
753         if (likely(sz < 0 || sz >= n))
754                 n = _copy_from_user(to, from, n);
755         else if(__builtin_constant_p(n))
756                 copy_from_user_overflow();
757         else
758                 __copy_from_user_overflow(sz, n);
759
760         return n;
761 }
762
763 static inline unsigned long __must_check
764 copy_to_user(void __user *to, const void *from, unsigned long n)
765 {
766         int sz = __compiletime_object_size(from);
767
768         might_fault();
769
770         /* See the comment in copy_from_user() above. */
771         if (likely(sz < 0 || sz >= n))
772                 n = _copy_to_user(to, from, n);
773         else if(__builtin_constant_p(n))
774                 copy_to_user_overflow();
775         else
776                 __copy_to_user_overflow(sz, n);
777
778         return n;
779 }
780
781 #undef __copy_from_user_overflow
782 #undef __copy_to_user_overflow
783
784 /*
785  * We rely on the nested NMI work to allow atomic faults from the NMI path; the
786  * nested NMI paths are careful to preserve CR2.
787  *
788  * Caller must use pagefault_enable/disable, or run in interrupt context,
789  * and also do a uaccess_ok() check
790  */
791 #define __copy_from_user_nmi __copy_from_user_inatomic
792
793 /*
794  * The "unsafe" user accesses aren't really "unsafe", but the naming
795  * is a big fat warning: you have to not only do the access_ok()
796  * checking before using them, but you have to surround them with the
797  * user_access_begin/end() pair.
798  */
799 #define user_access_begin()     __uaccess_begin()
800 #define user_access_end()       __uaccess_end()
801
802 #define unsafe_put_user(x, ptr)                                         \
803 ({                                                                              \
804         int __pu_err;                                                           \
805         __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT);         \
806         __builtin_expect(__pu_err, 0);                                          \
807 })
808
809 #define unsafe_get_user(x, ptr)                                         \
810 ({                                                                              \
811         int __gu_err;                                                           \
812         unsigned long __gu_val;                                                 \
813         __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT);    \
814         (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
815         __builtin_expect(__gu_err, 0);                                          \
816 })
817
818 #endif /* _ASM_X86_UACCESS_H */
819