treewide: Replace GPLv2 boilerplate/reference with SPDX - rule 500
[sfrench/cifs-2.6.git] / arch / arm / include / asm / uaccess.h
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  *  arch/arm/include/asm/uaccess.h
4  */
5 #ifndef _ASMARM_UACCESS_H
6 #define _ASMARM_UACCESS_H
7
8 /*
9  * User space memory access functions
10  */
11 #include <linux/string.h>
12 #include <asm/memory.h>
13 #include <asm/domain.h>
14 #include <asm/unified.h>
15 #include <asm/compiler.h>
16
17 #include <asm/extable.h>
18
19 /*
20  * These two functions allow hooking accesses to userspace to increase
21  * system integrity by ensuring that the kernel can not inadvertantly
22  * perform such accesses (eg, via list poison values) which could then
23  * be exploited for priviledge escalation.
24  */
25 static inline unsigned int uaccess_save_and_enable(void)
26 {
27 #ifdef CONFIG_CPU_SW_DOMAIN_PAN
28         unsigned int old_domain = get_domain();
29
30         /* Set the current domain access to permit user accesses */
31         set_domain((old_domain & ~domain_mask(DOMAIN_USER)) |
32                    domain_val(DOMAIN_USER, DOMAIN_CLIENT));
33
34         return old_domain;
35 #else
36         return 0;
37 #endif
38 }
39
40 static inline void uaccess_restore(unsigned int flags)
41 {
42 #ifdef CONFIG_CPU_SW_DOMAIN_PAN
43         /* Restore the user access mask */
44         set_domain(flags);
45 #endif
46 }
47
48 /*
49  * These two are intentionally not defined anywhere - if the kernel
50  * code generates any references to them, that's a bug.
51  */
52 extern int __get_user_bad(void);
53 extern int __put_user_bad(void);
54
55 /*
56  * Note that this is actually 0x1,0000,0000
57  */
58 #define KERNEL_DS       0x00000000
59
60 #ifdef CONFIG_MMU
61
62 #define USER_DS         TASK_SIZE
63 #define get_fs()        (current_thread_info()->addr_limit)
64
65 static inline void set_fs(mm_segment_t fs)
66 {
67         current_thread_info()->addr_limit = fs;
68
69         /*
70          * Prevent a mispredicted conditional call to set_fs from forwarding
71          * the wrong address limit to access_ok under speculation.
72          */
73         dsb(nsh);
74         isb();
75
76         modify_domain(DOMAIN_KERNEL, fs ? DOMAIN_CLIENT : DOMAIN_MANAGER);
77 }
78
79 #define segment_eq(a, b)        ((a) == (b))
80
81 /* We use 33-bit arithmetic here... */
82 #define __range_ok(addr, size) ({ \
83         unsigned long flag, roksum; \
84         __chk_user_ptr(addr);   \
85         __asm__(".syntax unified\n" \
86                 "adds %1, %2, %3; sbcscc %1, %1, %0; movcc %0, #0" \
87                 : "=&r" (flag), "=&r" (roksum) \
88                 : "r" (addr), "Ir" (size), "0" (current_thread_info()->addr_limit) \
89                 : "cc"); \
90         flag; })
91
92 /*
93  * This is a type: either unsigned long, if the argument fits into
94  * that type, or otherwise unsigned long long.
95  */
96 #define __inttype(x) \
97         __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
98
99 /*
100  * Sanitise a uaccess pointer such that it becomes NULL if addr+size
101  * is above the current addr_limit.
102  */
103 #define uaccess_mask_range_ptr(ptr, size)                       \
104         ((__typeof__(ptr))__uaccess_mask_range_ptr(ptr, size))
105 static inline void __user *__uaccess_mask_range_ptr(const void __user *ptr,
106                                                     size_t size)
107 {
108         void __user *safe_ptr = (void __user *)ptr;
109         unsigned long tmp;
110
111         asm volatile(
112         "       .syntax unified\n"
113         "       sub     %1, %3, #1\n"
114         "       subs    %1, %1, %0\n"
115         "       addhs   %1, %1, #1\n"
116         "       subshs  %1, %1, %2\n"
117         "       movlo   %0, #0\n"
118         : "+r" (safe_ptr), "=&r" (tmp)
119         : "r" (size), "r" (current_thread_info()->addr_limit)
120         : "cc");
121
122         csdb();
123         return safe_ptr;
124 }
125
126 /*
127  * Single-value transfer routines.  They automatically use the right
128  * size if we just have the right pointer type.  Note that the functions
129  * which read from user space (*get_*) need to take care not to leak
130  * kernel data even if the calling code is buggy and fails to check
131  * the return value.  This means zeroing out the destination variable
132  * or buffer on error.  Normally this is done out of line by the
133  * fixup code, but there are a few places where it intrudes on the
134  * main code path.  When we only write to user space, there is no
135  * problem.
136  */
137 extern int __get_user_1(void *);
138 extern int __get_user_2(void *);
139 extern int __get_user_4(void *);
140 extern int __get_user_32t_8(void *);
141 extern int __get_user_8(void *);
142 extern int __get_user_64t_1(void *);
143 extern int __get_user_64t_2(void *);
144 extern int __get_user_64t_4(void *);
145
146 #define __GUP_CLOBBER_1 "lr", "cc"
147 #ifdef CONFIG_CPU_USE_DOMAINS
148 #define __GUP_CLOBBER_2 "ip", "lr", "cc"
149 #else
150 #define __GUP_CLOBBER_2 "lr", "cc"
151 #endif
152 #define __GUP_CLOBBER_4 "lr", "cc"
153 #define __GUP_CLOBBER_32t_8 "lr", "cc"
154 #define __GUP_CLOBBER_8 "lr", "cc"
155
156 #define __get_user_x(__r2, __p, __e, __l, __s)                          \
157            __asm__ __volatile__ (                                       \
158                 __asmeq("%0", "r0") __asmeq("%1", "r2")                 \
159                 __asmeq("%3", "r1")                                     \
160                 "bl     __get_user_" #__s                               \
161                 : "=&r" (__e), "=r" (__r2)                              \
162                 : "0" (__p), "r" (__l)                                  \
163                 : __GUP_CLOBBER_##__s)
164
165 /* narrowing a double-word get into a single 32bit word register: */
166 #ifdef __ARMEB__
167 #define __get_user_x_32t(__r2, __p, __e, __l, __s)                      \
168         __get_user_x(__r2, __p, __e, __l, 32t_8)
169 #else
170 #define __get_user_x_32t __get_user_x
171 #endif
172
173 /*
174  * storing result into proper least significant word of 64bit target var,
175  * different only for big endian case where 64 bit __r2 lsw is r3:
176  */
177 #ifdef __ARMEB__
178 #define __get_user_x_64t(__r2, __p, __e, __l, __s)                      \
179            __asm__ __volatile__ (                                       \
180                 __asmeq("%0", "r0") __asmeq("%1", "r2")                 \
181                 __asmeq("%3", "r1")                                     \
182                 "bl     __get_user_64t_" #__s                           \
183                 : "=&r" (__e), "=r" (__r2)                              \
184                 : "0" (__p), "r" (__l)                                  \
185                 : __GUP_CLOBBER_##__s)
186 #else
187 #define __get_user_x_64t __get_user_x
188 #endif
189
190
191 #define __get_user_check(x, p)                                          \
192         ({                                                              \
193                 unsigned long __limit = current_thread_info()->addr_limit - 1; \
194                 register typeof(*(p)) __user *__p asm("r0") = (p);      \
195                 register __inttype(x) __r2 asm("r2");                   \
196                 register unsigned long __l asm("r1") = __limit;         \
197                 register int __e asm("r0");                             \
198                 unsigned int __ua_flags = uaccess_save_and_enable();    \
199                 switch (sizeof(*(__p))) {                               \
200                 case 1:                                                 \
201                         if (sizeof((x)) >= 8)                           \
202                                 __get_user_x_64t(__r2, __p, __e, __l, 1); \
203                         else                                            \
204                                 __get_user_x(__r2, __p, __e, __l, 1);   \
205                         break;                                          \
206                 case 2:                                                 \
207                         if (sizeof((x)) >= 8)                           \
208                                 __get_user_x_64t(__r2, __p, __e, __l, 2); \
209                         else                                            \
210                                 __get_user_x(__r2, __p, __e, __l, 2);   \
211                         break;                                          \
212                 case 4:                                                 \
213                         if (sizeof((x)) >= 8)                           \
214                                 __get_user_x_64t(__r2, __p, __e, __l, 4); \
215                         else                                            \
216                                 __get_user_x(__r2, __p, __e, __l, 4);   \
217                         break;                                          \
218                 case 8:                                                 \
219                         if (sizeof((x)) < 8)                            \
220                                 __get_user_x_32t(__r2, __p, __e, __l, 4); \
221                         else                                            \
222                                 __get_user_x(__r2, __p, __e, __l, 8);   \
223                         break;                                          \
224                 default: __e = __get_user_bad(); break;                 \
225                 }                                                       \
226                 uaccess_restore(__ua_flags);                            \
227                 x = (typeof(*(p))) __r2;                                \
228                 __e;                                                    \
229         })
230
231 #define get_user(x, p)                                                  \
232         ({                                                              \
233                 might_fault();                                          \
234                 __get_user_check(x, p);                                 \
235          })
236
237 extern int __put_user_1(void *, unsigned int);
238 extern int __put_user_2(void *, unsigned int);
239 extern int __put_user_4(void *, unsigned int);
240 extern int __put_user_8(void *, unsigned long long);
241
242 #define __put_user_check(__pu_val, __ptr, __err, __s)                   \
243         ({                                                              \
244                 unsigned long __limit = current_thread_info()->addr_limit - 1; \
245                 register typeof(__pu_val) __r2 asm("r2") = __pu_val;    \
246                 register const void __user *__p asm("r0") = __ptr;      \
247                 register unsigned long __l asm("r1") = __limit;         \
248                 register int __e asm("r0");                             \
249                 __asm__ __volatile__ (                                  \
250                         __asmeq("%0", "r0") __asmeq("%2", "r2")         \
251                         __asmeq("%3", "r1")                             \
252                         "bl     __put_user_" #__s                       \
253                         : "=&r" (__e)                                   \
254                         : "0" (__p), "r" (__r2), "r" (__l)              \
255                         : "ip", "lr", "cc");                            \
256                 __err = __e;                                            \
257         })
258
259 #else /* CONFIG_MMU */
260
261 /*
262  * uClinux has only one addr space, so has simplified address limits.
263  */
264 #define USER_DS                 KERNEL_DS
265
266 #define segment_eq(a, b)                (1)
267 #define __addr_ok(addr)         ((void)(addr), 1)
268 #define __range_ok(addr, size)  ((void)(addr), 0)
269 #define get_fs()                (KERNEL_DS)
270
271 static inline void set_fs(mm_segment_t fs)
272 {
273 }
274
275 #define get_user(x, p)  __get_user(x, p)
276 #define __put_user_check __put_user_nocheck
277
278 #endif /* CONFIG_MMU */
279
280 #define access_ok(addr, size)   (__range_ok(addr, size) == 0)
281
282 #define user_addr_max() \
283         (uaccess_kernel() ? ~0UL : get_fs())
284
285 #ifdef CONFIG_CPU_SPECTRE
286 /*
287  * When mitigating Spectre variant 1, it is not worth fixing the non-
288  * verifying accessors, because we need to add verification of the
289  * address space there.  Force these to use the standard get_user()
290  * version instead.
291  */
292 #define __get_user(x, ptr) get_user(x, ptr)
293 #else
294
295 /*
296  * The "__xxx" versions of the user access functions do not verify the
297  * address space - it must have been done previously with a separate
298  * "access_ok()" call.
299  *
300  * The "xxx_error" versions set the third argument to EFAULT if an
301  * error occurs, and leave it unchanged on success.  Note that these
302  * versions are void (ie, don't return a value as such).
303  */
304 #define __get_user(x, ptr)                                              \
305 ({                                                                      \
306         long __gu_err = 0;                                              \
307         __get_user_err((x), (ptr), __gu_err);                           \
308         __gu_err;                                                       \
309 })
310
311 #define __get_user_err(x, ptr, err)                                     \
312 do {                                                                    \
313         unsigned long __gu_addr = (unsigned long)(ptr);                 \
314         unsigned long __gu_val;                                         \
315         unsigned int __ua_flags;                                        \
316         __chk_user_ptr(ptr);                                            \
317         might_fault();                                                  \
318         __ua_flags = uaccess_save_and_enable();                         \
319         switch (sizeof(*(ptr))) {                                       \
320         case 1: __get_user_asm_byte(__gu_val, __gu_addr, err);  break;  \
321         case 2: __get_user_asm_half(__gu_val, __gu_addr, err);  break;  \
322         case 4: __get_user_asm_word(__gu_val, __gu_addr, err);  break;  \
323         default: (__gu_val) = __get_user_bad();                         \
324         }                                                               \
325         uaccess_restore(__ua_flags);                                    \
326         (x) = (__typeof__(*(ptr)))__gu_val;                             \
327 } while (0)
328
329 #define __get_user_asm(x, addr, err, instr)                     \
330         __asm__ __volatile__(                                   \
331         "1:     " TUSER(instr) " %1, [%2], #0\n"                \
332         "2:\n"                                                  \
333         "       .pushsection .text.fixup,\"ax\"\n"              \
334         "       .align  2\n"                                    \
335         "3:     mov     %0, %3\n"                               \
336         "       mov     %1, #0\n"                               \
337         "       b       2b\n"                                   \
338         "       .popsection\n"                                  \
339         "       .pushsection __ex_table,\"a\"\n"                \
340         "       .align  3\n"                                    \
341         "       .long   1b, 3b\n"                               \
342         "       .popsection"                                    \
343         : "+r" (err), "=&r" (x)                                 \
344         : "r" (addr), "i" (-EFAULT)                             \
345         : "cc")
346
347 #define __get_user_asm_byte(x, addr, err)                       \
348         __get_user_asm(x, addr, err, ldrb)
349
350 #if __LINUX_ARM_ARCH__ >= 6
351
352 #define __get_user_asm_half(x, addr, err)                       \
353         __get_user_asm(x, addr, err, ldrh)
354
355 #else
356
357 #ifndef __ARMEB__
358 #define __get_user_asm_half(x, __gu_addr, err)                  \
359 ({                                                              \
360         unsigned long __b1, __b2;                               \
361         __get_user_asm_byte(__b1, __gu_addr, err);              \
362         __get_user_asm_byte(__b2, __gu_addr + 1, err);          \
363         (x) = __b1 | (__b2 << 8);                               \
364 })
365 #else
366 #define __get_user_asm_half(x, __gu_addr, err)                  \
367 ({                                                              \
368         unsigned long __b1, __b2;                               \
369         __get_user_asm_byte(__b1, __gu_addr, err);              \
370         __get_user_asm_byte(__b2, __gu_addr + 1, err);          \
371         (x) = (__b1 << 8) | __b2;                               \
372 })
373 #endif
374
375 #endif /* __LINUX_ARM_ARCH__ >= 6 */
376
377 #define __get_user_asm_word(x, addr, err)                       \
378         __get_user_asm(x, addr, err, ldr)
379 #endif
380
381
382 #define __put_user_switch(x, ptr, __err, __fn)                          \
383         do {                                                            \
384                 const __typeof__(*(ptr)) __user *__pu_ptr = (ptr);      \
385                 __typeof__(*(ptr)) __pu_val = (x);                      \
386                 unsigned int __ua_flags;                                \
387                 might_fault();                                          \
388                 __ua_flags = uaccess_save_and_enable();                 \
389                 switch (sizeof(*(ptr))) {                               \
390                 case 1: __fn(__pu_val, __pu_ptr, __err, 1); break;      \
391                 case 2: __fn(__pu_val, __pu_ptr, __err, 2); break;      \
392                 case 4: __fn(__pu_val, __pu_ptr, __err, 4); break;      \
393                 case 8: __fn(__pu_val, __pu_ptr, __err, 8); break;      \
394                 default: __err = __put_user_bad(); break;               \
395                 }                                                       \
396                 uaccess_restore(__ua_flags);                            \
397         } while (0)
398
399 #define put_user(x, ptr)                                                \
400 ({                                                                      \
401         int __pu_err = 0;                                               \
402         __put_user_switch((x), (ptr), __pu_err, __put_user_check);      \
403         __pu_err;                                                       \
404 })
405
406 #ifdef CONFIG_CPU_SPECTRE
407 /*
408  * When mitigating Spectre variant 1.1, all accessors need to include
409  * verification of the address space.
410  */
411 #define __put_user(x, ptr) put_user(x, ptr)
412
413 #else
414 #define __put_user(x, ptr)                                              \
415 ({                                                                      \
416         long __pu_err = 0;                                              \
417         __put_user_switch((x), (ptr), __pu_err, __put_user_nocheck);    \
418         __pu_err;                                                       \
419 })
420
421 #define __put_user_nocheck(x, __pu_ptr, __err, __size)                  \
422         do {                                                            \
423                 unsigned long __pu_addr = (unsigned long)__pu_ptr;      \
424                 __put_user_nocheck_##__size(x, __pu_addr, __err);       \
425         } while (0)
426
427 #define __put_user_nocheck_1 __put_user_asm_byte
428 #define __put_user_nocheck_2 __put_user_asm_half
429 #define __put_user_nocheck_4 __put_user_asm_word
430 #define __put_user_nocheck_8 __put_user_asm_dword
431
432 #define __put_user_asm(x, __pu_addr, err, instr)                \
433         __asm__ __volatile__(                                   \
434         "1:     " TUSER(instr) " %1, [%2], #0\n"                \
435         "2:\n"                                                  \
436         "       .pushsection .text.fixup,\"ax\"\n"              \
437         "       .align  2\n"                                    \
438         "3:     mov     %0, %3\n"                               \
439         "       b       2b\n"                                   \
440         "       .popsection\n"                                  \
441         "       .pushsection __ex_table,\"a\"\n"                \
442         "       .align  3\n"                                    \
443         "       .long   1b, 3b\n"                               \
444         "       .popsection"                                    \
445         : "+r" (err)                                            \
446         : "r" (x), "r" (__pu_addr), "i" (-EFAULT)               \
447         : "cc")
448
449 #define __put_user_asm_byte(x, __pu_addr, err)                  \
450         __put_user_asm(x, __pu_addr, err, strb)
451
452 #if __LINUX_ARM_ARCH__ >= 6
453
454 #define __put_user_asm_half(x, __pu_addr, err)                  \
455         __put_user_asm(x, __pu_addr, err, strh)
456
457 #else
458
459 #ifndef __ARMEB__
460 #define __put_user_asm_half(x, __pu_addr, err)                  \
461 ({                                                              \
462         unsigned long __temp = (__force unsigned long)(x);      \
463         __put_user_asm_byte(__temp, __pu_addr, err);            \
464         __put_user_asm_byte(__temp >> 8, __pu_addr + 1, err);   \
465 })
466 #else
467 #define __put_user_asm_half(x, __pu_addr, err)                  \
468 ({                                                              \
469         unsigned long __temp = (__force unsigned long)(x);      \
470         __put_user_asm_byte(__temp >> 8, __pu_addr, err);       \
471         __put_user_asm_byte(__temp, __pu_addr + 1, err);        \
472 })
473 #endif
474
475 #endif /* __LINUX_ARM_ARCH__ >= 6 */
476
477 #define __put_user_asm_word(x, __pu_addr, err)                  \
478         __put_user_asm(x, __pu_addr, err, str)
479
480 #ifndef __ARMEB__
481 #define __reg_oper0     "%R2"
482 #define __reg_oper1     "%Q2"
483 #else
484 #define __reg_oper0     "%Q2"
485 #define __reg_oper1     "%R2"
486 #endif
487
488 #define __put_user_asm_dword(x, __pu_addr, err)                 \
489         __asm__ __volatile__(                                   \
490  ARM(   "1:     " TUSER(str) "  " __reg_oper1 ", [%1], #4\n"    ) \
491  ARM(   "2:     " TUSER(str) "  " __reg_oper0 ", [%1]\n"        ) \
492  THUMB( "1:     " TUSER(str) "  " __reg_oper1 ", [%1]\n"        ) \
493  THUMB( "2:     " TUSER(str) "  " __reg_oper0 ", [%1, #4]\n"    ) \
494         "3:\n"                                                  \
495         "       .pushsection .text.fixup,\"ax\"\n"              \
496         "       .align  2\n"                                    \
497         "4:     mov     %0, %3\n"                               \
498         "       b       3b\n"                                   \
499         "       .popsection\n"                                  \
500         "       .pushsection __ex_table,\"a\"\n"                \
501         "       .align  3\n"                                    \
502         "       .long   1b, 4b\n"                               \
503         "       .long   2b, 4b\n"                               \
504         "       .popsection"                                    \
505         : "+r" (err), "+r" (__pu_addr)                          \
506         : "r" (x), "i" (-EFAULT)                                \
507         : "cc")
508
509 #endif /* !CONFIG_CPU_SPECTRE */
510
511 #ifdef CONFIG_MMU
512 extern unsigned long __must_check
513 arm_copy_from_user(void *to, const void __user *from, unsigned long n);
514
515 static inline unsigned long __must_check
516 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
517 {
518         unsigned int __ua_flags;
519
520         __ua_flags = uaccess_save_and_enable();
521         n = arm_copy_from_user(to, from, n);
522         uaccess_restore(__ua_flags);
523         return n;
524 }
525
526 extern unsigned long __must_check
527 arm_copy_to_user(void __user *to, const void *from, unsigned long n);
528 extern unsigned long __must_check
529 __copy_to_user_std(void __user *to, const void *from, unsigned long n);
530
531 static inline unsigned long __must_check
532 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
533 {
534 #ifndef CONFIG_UACCESS_WITH_MEMCPY
535         unsigned int __ua_flags;
536         __ua_flags = uaccess_save_and_enable();
537         n = arm_copy_to_user(to, from, n);
538         uaccess_restore(__ua_flags);
539         return n;
540 #else
541         return arm_copy_to_user(to, from, n);
542 #endif
543 }
544
545 extern unsigned long __must_check
546 arm_clear_user(void __user *addr, unsigned long n);
547 extern unsigned long __must_check
548 __clear_user_std(void __user *addr, unsigned long n);
549
550 static inline unsigned long __must_check
551 __clear_user(void __user *addr, unsigned long n)
552 {
553         unsigned int __ua_flags = uaccess_save_and_enable();
554         n = arm_clear_user(addr, n);
555         uaccess_restore(__ua_flags);
556         return n;
557 }
558
559 #else
560 static inline unsigned long
561 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
562 {
563         memcpy(to, (const void __force *)from, n);
564         return 0;
565 }
566 static inline unsigned long
567 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
568 {
569         memcpy((void __force *)to, from, n);
570         return 0;
571 }
572 #define __clear_user(addr, n)           (memset((void __force *)addr, 0, n), 0)
573 #endif
574 #define INLINE_COPY_TO_USER
575 #define INLINE_COPY_FROM_USER
576
577 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
578 {
579         if (access_ok(to, n))
580                 n = __clear_user(to, n);
581         return n;
582 }
583
584 /* These are from lib/ code, and use __get_user() and friends */
585 extern long strncpy_from_user(char *dest, const char __user *src, long count);
586
587 extern __must_check long strnlen_user(const char __user *str, long n);
588
589 #endif /* _ASMARM_UACCESS_H */