1 /* SPDX-License-Identifier: GPL-2.0 */
3 #ifndef __NOSPEC_BRANCH_H__
4 #define __NOSPEC_BRANCH_H__
6 #include <asm/alternative.h>
7 #include <asm/alternative-asm.h>
8 #include <asm/cpufeatures.h>
11 * Fill the CPU return stack buffer.
13 * Each entry in the RSB, if used for a speculative 'ret', contains an
14 * infinite 'pause; jmp' loop to capture speculative execution.
16 * This is required in various cases for retpoline and IBRS-based
17 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
18 * eliminate potentially bogus entries from the RSB, and sometimes
19 * purely to ensure that it doesn't get empty, which on some CPUs would
20 * allow predictions from other (unwanted!) sources to be used.
22 * We define a CPP macro such that it can be used from both .S files and
23 * inline assembly. It's possible to do a .macro and then include that
24 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
27 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
28 #define RSB_FILL_LOOPS 16 /* To avoid underflow */
31 * Google experimented with loop-unrolling and this turned out to be
32 * the optimal version — two calls, each with their own speculation
33 * trap should their return address end up getting used, in a loop.
35 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
39 773: /* speculation trap */ \
44 775: /* speculation trap */ \
50 add $(BITS_PER_LONG/8) * nr, sp;
55 * This should be used immediately before a retpoline alternative. It tells
56 * objtool where the retpolines are so that it can make sense of the control
57 * flow by just reading the original instruction(s) and ignoring the
60 .macro ANNOTATE_NOSPEC_ALTERNATIVE
62 .pushsection .discard.nospec
63 .long .Lannotate_\@ - .
68 * These are the bare retpoline primitives for indirect jmp and call.
69 * Do not use these directly; they only exist to make the ALTERNATIVE
70 * invocation below less ugly.
72 .macro RETPOLINE_JMP reg:req
83 * This is a wrapper around RETPOLINE_JMP so the called function in reg
84 * returns to the instruction after the macro.
86 .macro RETPOLINE_CALL reg:req
88 .Ldo_retpoline_jmp_\@:
91 call .Ldo_retpoline_jmp_\@
95 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
96 * indirect jmp/call which may be susceptible to the Spectre variant 2
99 .macro JMP_NOSPEC reg:req
100 #ifdef CONFIG_RETPOLINE
101 ANNOTATE_NOSPEC_ALTERNATIVE
102 ALTERNATIVE_2 __stringify(jmp *\reg), \
103 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
104 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
110 .macro CALL_NOSPEC reg:req
111 #ifdef CONFIG_RETPOLINE
112 ANNOTATE_NOSPEC_ALTERNATIVE
113 ALTERNATIVE_2 __stringify(call *\reg), \
114 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
115 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
122 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
123 * monstrosity above, manually.
125 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
126 #ifdef CONFIG_RETPOLINE
127 ANNOTATE_NOSPEC_ALTERNATIVE
128 ALTERNATIVE "jmp .Lskip_rsb_\@", \
129 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
135 #else /* __ASSEMBLY__ */
137 #define ANNOTATE_NOSPEC_ALTERNATIVE \
139 ".pushsection .discard.nospec\n\t" \
140 ".long 999b - .\n\t" \
143 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
146 * Since the inline asm uses the %V modifier which is only in newer GCC,
147 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
149 # define CALL_NOSPEC \
150 ANNOTATE_NOSPEC_ALTERNATIVE \
152 "call *%[thunk_target]\n", \
153 "call __x86_indirect_thunk_%V[thunk_target]\n", \
154 X86_FEATURE_RETPOLINE)
155 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
157 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
159 * For i386 we use the original ret-equivalent retpoline, because
160 * otherwise we'll run out of registers. We don't care about CET
163 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
166 "901: call 903f;\n" \
170 "903: addl $4, %%esp;\n" \
171 " pushl %[thunk_target];\n" \
174 "904: call 901b;\n", \
175 X86_FEATURE_RETPOLINE)
177 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
178 #else /* No retpoline for C / inline asm */
179 # define CALL_NOSPEC "call *%[thunk_target]\n"
180 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
183 /* The Spectre V2 mitigation variants */
184 enum spectre_v2_mitigation {
186 SPECTRE_V2_RETPOLINE_MINIMAL,
187 SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
188 SPECTRE_V2_RETPOLINE_GENERIC,
189 SPECTRE_V2_RETPOLINE_AMD,
194 * On VMEXIT we must ensure that no RSB predictions learned in the guest
195 * can be followed in the host, by overwriting the RSB completely. Both
196 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
197 * CPUs with IBRS_ATT *might* it be avoided.
199 static inline void vmexit_fill_RSB(void)
201 #ifdef CONFIG_RETPOLINE
202 unsigned long loops = RSB_CLEAR_LOOPS / 2;
204 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
205 ALTERNATIVE("jmp 910f",
206 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
207 X86_FEATURE_RETPOLINE)
209 : "=&r" (loops), ASM_CALL_CONSTRAINT
210 : "r" (loops) : "memory" );
213 #endif /* __ASSEMBLY__ */
214 #endif /* __NOSPEC_BRANCH_H__ */