Merge branch 'work.set_fs' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs

[sfrench/cifs-2.6.git] / arch / x86 / lib / getuser.S

diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
index 2cd902e0606225ae839e7f2dfd3ecb8d101e8ca2..fa1bc2104b326ee36e41e9a89a74c2b8c5f511dc 100644 (file)
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -37,10 +37,19 @@
 
 #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
 
+#ifdef CONFIG_X86_5LEVEL
+#define LOAD_TASK_SIZE_MINUS_N(n) \
+       ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rdx), \
+                   __stringify(mov $((1 << 56) - 4096 - (n)),%rdx), X86_FEATURE_LA57
+#else
+#define LOAD_TASK_SIZE_MINUS_N(n) \
+       mov $(TASK_SIZE_MAX - (n)),%_ASM_DX
+#endif
+
        .text
 SYM_FUNC_START(__get_user_1)
-       mov PER_CPU_VAR(current_task), %_ASM_DX
-       cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
+       LOAD_TASK_SIZE_MINUS_N(0)
+       cmp %_ASM_DX,%_ASM_AX
        jae bad_get_user
        sbb %_ASM_DX, %_ASM_DX          /* array_index_mask_nospec() */
        and %_ASM_DX, %_ASM_AX
@@ -53,15 +62,13 @@ SYM_FUNC_END(__get_user_1)
 EXPORT_SYMBOL(__get_user_1)
 
 SYM_FUNC_START(__get_user_2)
-       add $1,%_ASM_AX
-       jc bad_get_user
-       mov PER_CPU_VAR(current_task), %_ASM_DX
-       cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
+       LOAD_TASK_SIZE_MINUS_N(1)
+       cmp %_ASM_DX,%_ASM_AX
        jae bad_get_user
        sbb %_ASM_DX, %_ASM_DX          /* array_index_mask_nospec() */
        and %_ASM_DX, %_ASM_AX
        ASM_STAC
-2:     movzwl -1(%_ASM_AX),%edx
+2:     movzwl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        ret
@@ -69,15 +76,13 @@ SYM_FUNC_END(__get_user_2)
 EXPORT_SYMBOL(__get_user_2)
 
 SYM_FUNC_START(__get_user_4)
-       add $3,%_ASM_AX
-       jc bad_get_user
-       mov PER_CPU_VAR(current_task), %_ASM_DX
-       cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
+       LOAD_TASK_SIZE_MINUS_N(3)
+       cmp %_ASM_DX,%_ASM_AX
        jae bad_get_user
        sbb %_ASM_DX, %_ASM_DX          /* array_index_mask_nospec() */
        and %_ASM_DX, %_ASM_AX
        ASM_STAC
-3:     movl -3(%_ASM_AX),%edx
+3:     movl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        ret
@@ -86,29 +91,25 @@ EXPORT_SYMBOL(__get_user_4)
 
 SYM_FUNC_START(__get_user_8)
 #ifdef CONFIG_X86_64
-       add $7,%_ASM_AX
-       jc bad_get_user
-       mov PER_CPU_VAR(current_task), %_ASM_DX
-       cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
+       LOAD_TASK_SIZE_MINUS_N(7)
+       cmp %_ASM_DX,%_ASM_AX
        jae bad_get_user
        sbb %_ASM_DX, %_ASM_DX          /* array_index_mask_nospec() */
        and %_ASM_DX, %_ASM_AX
        ASM_STAC
-4:     movq -7(%_ASM_AX),%rdx
+4:     movq (%_ASM_AX),%rdx
        xor %eax,%eax
        ASM_CLAC
        ret
 #else
-       add $7,%_ASM_AX
-       jc bad_get_user_8
-       mov PER_CPU_VAR(current_task), %_ASM_DX
-       cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
+       LOAD_TASK_SIZE_MINUS_N(7)
+       cmp %_ASM_DX,%_ASM_AX
        jae bad_get_user_8
        sbb %_ASM_DX, %_ASM_DX          /* array_index_mask_nospec() */
        and %_ASM_DX, %_ASM_AX
        ASM_STAC
-4:     movl -7(%_ASM_AX),%edx
-5:     movl -3(%_ASM_AX),%ecx
+4:     movl (%_ASM_AX),%edx
+5:     movl 4(%_ASM_AX),%ecx
        xor %eax,%eax
        ASM_CLAC
        ret