1 /* SPDX-License-Identifier: GPL-2.0 */
3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
4 #define _ASM_X86_NOSPEC_BRANCH_H_
6 #include <asm/alternative.h>
7 #include <asm/alternative-asm.h>
8 #include <asm/cpufeatures.h>
9 #include <asm/msr-index.h>
14 * This should be used immediately before a retpoline alternative. It tells
15 * objtool where the retpolines are so that it can make sense of the control
16 * flow by just reading the original instruction(s) and ignoring the
19 .macro ANNOTATE_NOSPEC_ALTERNATIVE
21 .pushsection .discard.nospec
22 .long .Lannotate_\@ - .
27 * These are the bare retpoline primitives for indirect jmp and call.
28 * Do not use these directly; they only exist to make the ALTERNATIVE
29 * invocation below less ugly.
31 .macro RETPOLINE_JMP reg:req
43 * This is a wrapper around RETPOLINE_JMP so the called function in reg
44 * returns to the instruction after the macro.
46 .macro RETPOLINE_CALL reg:req
48 .Ldo_retpoline_jmp_\@:
51 call .Ldo_retpoline_jmp_\@
55 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
56 * indirect jmp/call which may be susceptible to the Spectre variant 2
59 .macro JMP_NOSPEC reg:req
60 #ifdef CONFIG_RETPOLINE
61 ANNOTATE_NOSPEC_ALTERNATIVE
62 ALTERNATIVE_2 __stringify(jmp *\reg), \
63 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
64 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
70 .macro CALL_NOSPEC reg:req
71 #ifdef CONFIG_RETPOLINE
72 ANNOTATE_NOSPEC_ALTERNATIVE
73 ALTERNATIVE_2 __stringify(call *\reg), \
74 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
75 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
81 /* This clobbers the BX register */
82 .macro FILL_RETURN_BUFFER nr:req ftr:req
83 #ifdef CONFIG_RETPOLINE
84 ALTERNATIVE "", "call __clear_rsb", \ftr
88 #else /* __ASSEMBLY__ */
90 #define ANNOTATE_NOSPEC_ALTERNATIVE \
92 ".pushsection .discard.nospec\n\t" \
93 ".long 999b - .\n\t" \
96 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
99 * Since the inline asm uses the %V modifier which is only in newer GCC,
100 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
102 # define CALL_NOSPEC \
103 ANNOTATE_NOSPEC_ALTERNATIVE \
105 "call *%[thunk_target]\n", \
106 "call __x86_indirect_thunk_%V[thunk_target]\n", \
107 X86_FEATURE_RETPOLINE)
108 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
110 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
112 * For i386 we use the original ret-equivalent retpoline, because
113 * otherwise we'll run out of registers. We don't care about CET
116 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
119 "901: call 903f;\n" \
124 "903: addl $4, %%esp;\n" \
125 " pushl %[thunk_target];\n" \
128 "904: call 901b;\n", \
129 X86_FEATURE_RETPOLINE)
131 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
132 #else /* No retpoline for C / inline asm */
133 # define CALL_NOSPEC "call *%[thunk_target]\n"
134 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
137 /* The Spectre V2 mitigation variants */
138 enum spectre_v2_mitigation {
140 SPECTRE_V2_RETPOLINE_MINIMAL,
141 SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
142 SPECTRE_V2_RETPOLINE_GENERIC,
143 SPECTRE_V2_RETPOLINE_AMD,
147 extern char __indirect_thunk_start[];
148 extern char __indirect_thunk_end[];
151 * On VMEXIT we must ensure that no RSB predictions learned in the guest
152 * can be followed in the host, by overwriting the RSB completely. Both
153 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
154 * CPUs with IBRS_ALL *might* it be avoided.
156 static inline void vmexit_fill_RSB(void)
158 #ifdef CONFIG_RETPOLINE
159 alternative_input("",
161 X86_FEATURE_RETPOLINE,
162 ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory"));
166 static inline void indirect_branch_prediction_barrier(void)
168 asm volatile(ALTERNATIVE("",
169 "movl %[msr], %%ecx\n\t"
170 "movl %[val], %%eax\n\t"
173 X86_FEATURE_USE_IBPB)
174 : : [msr] "i" (MSR_IA32_PRED_CMD),
175 [val] "i" (PRED_CMD_IBPB)
176 : "eax", "ecx", "edx", "memory");
179 #endif /* __ASSEMBLY__ */
180 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */