objtool/x86: Rewrite retpoline thunk calls

When the compiler emits: "CALL __x86_indirect_thunk_\reg" for an
indirect call, have objtool rewrite it to:

	ALTERNATIVE "call __x86_indirect_thunk_\reg",
		    "call *%reg", ALT_NOT(X86_FEATURE_RETPOLINE)

Additionally, in order to not emit endless identical
.altinst_replacement chunks, use a global symbol for them, see
__x86_indirect_alt_*.

This also avoids objtool from having to do code generation.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Link: https://lkml.kernel.org/r/20210326151300.320177914@infradead.org

diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index 0545b07895a52483130f5308f4424006352adb35..4cb726c71ed8c61f1d1aedd9dd3a7d6f36c9bb09 100644 (file)
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -19,11 +19,19 @@ extern void cmpxchg8b_emu(void);
 
 #ifdef CONFIG_RETPOLINE
 
-#define DECL_INDIRECT_THUNK(reg) \
+#undef GEN
+#define GEN(reg) \
        extern asmlinkage void __x86_indirect_thunk_ ## reg (void);
+#include <asm/GEN-for-each-reg.h>
+
+#undef GEN
+#define GEN(reg) \
+       extern asmlinkage void __x86_indirect_alt_call_ ## reg (void);
+#include <asm/GEN-for-each-reg.h>
 
 #undef GEN
-#define GEN(reg) DECL_INDIRECT_THUNK(reg)
+#define GEN(reg) \
+       extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);
 #include <asm/GEN-for-each-reg.h>
 
 #endif /* CONFIG_RETPOLINE */

diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index d2c0d14c1e221e20ef009d42a25e0e78e42ecc1d..4d32cb06ffd5b2b6174e49bdce58630dc6286be3 100644 (file)
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -10,6 +10,8 @@
 #include <asm/unwind_hints.h>
 #include <asm/frame.h>
 
+       .section .text.__x86.indirect_thunk
+
 .macro RETPOLINE reg
        ANNOTATE_INTRA_FUNCTION_CALL
        call    .Ldo_rop_\@
@@ -25,9 +27,9 @@
 .endm
 
 .macro THUNK reg
-       .section .text.__x86.indirect_thunk
 
        .align 32
+
 SYM_FUNC_START(__x86_indirect_thunk_\reg)
 
        ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
@@ -38,6 +40,32 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg)
 
 .endm
 
+/*
+ * This generates .altinstr_replacement symbols for use by objtool. They,
+ * however, must not actually live in .altinstr_replacement since that will be
+ * discarded after init, but module alternatives will also reference these
+ * symbols.
+ *
+ * Their names matches the "__x86_indirect_" prefix to mark them as retpolines.
+ */
+.macro ALT_THUNK reg
+
+       .align 1
+
+SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg)
+       ANNOTATE_RETPOLINE_SAFE
+1:     call    *%\reg
+2:     .skip   5-(2b-1b), 0x90
+SYM_FUNC_END(__x86_indirect_alt_call_\reg)
+
+SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg)
+       ANNOTATE_RETPOLINE_SAFE
+1:     jmp     *%\reg
+2:     .skip   5-(2b-1b), 0x90
+SYM_FUNC_END(__x86_indirect_alt_jmp_\reg)
+
+.endm
+
 /*
  * Despite being an assembler file we can't just use .irp here
  * because __KSYM_DEPS__ only uses the C preprocessor and would
@@ -61,3 +89,14 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg)
 #define GEN(reg) EXPORT_THUNK(reg)
 #include <asm/GEN-for-each-reg.h>
 
+#undef GEN
+#define GEN(reg) ALT_THUNK reg
+#include <asm/GEN-for-each-reg.h>
+
+#undef GEN
+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg)
+#include <asm/GEN-for-each-reg.h>
+
+#undef GEN
+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg)
+#include <asm/GEN-for-each-reg.h>

diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
index 782894eb351a49ce67b0c899f0f50d876931f86f..7e8b5bedd946cc7f244e3f4425d4c9fc46934058 100644 (file)
--- a/tools/objtool/arch/x86/decode.c
+++ b/tools/objtool/arch/x86/decode.c
@@ -19,6 +19,7 @@
 #include <objtool/elf.h>
 #include <objtool/arch.h>
 #include <objtool/warn.h>
+#include <arch/elf.h>
 
 static unsigned char op_to_cfi_reg[][2] = {
        {CFI_AX, CFI_R8},
@@ -613,6 +614,122 @@ const char *arch_nop_insn(int len)
        return nops[len-1];
 }
 
+/* asm/alternative.h ? */
+
+#define ALTINSTR_FLAG_INV      (1 << 15)
+#define ALT_NOT(feat)          ((feat) | ALTINSTR_FLAG_INV)
+
+struct alt_instr {
+       s32 instr_offset;       /* original instruction */
+       s32 repl_offset;        /* offset to replacement instruction */
+       u16 cpuid;              /* cpuid bit set for replacement */
+       u8  instrlen;           /* length of original instruction */
+       u8  replacementlen;     /* length of new instruction */
+} __packed;
+
+static int elf_add_alternative(struct elf *elf,
+                              struct instruction *orig, struct symbol *sym,
+                              int cpuid, u8 orig_len, u8 repl_len)
+{
+       const int size = sizeof(struct alt_instr);
+       struct alt_instr *alt;
+       struct section *sec;
+       Elf_Scn *s;
+
+       sec = find_section_by_name(elf, ".altinstructions");
+       if (!sec) {
+               sec = elf_create_section(elf, ".altinstructions",
+                                        SHF_WRITE, size, 0);
+
+               if (!sec) {
+                       WARN_ELF("elf_create_section");
+                       return -1;
+               }
+       }
+
+       s = elf_getscn(elf->elf, sec->idx);
+       if (!s) {
+               WARN_ELF("elf_getscn");
+               return -1;
+       }
+
+       sec->data = elf_newdata(s);
+       if (!sec->data) {
+               WARN_ELF("elf_newdata");
+               return -1;
+       }
+
+       sec->data->d_size = size;
+       sec->data->d_align = 1;
+
+       alt = sec->data->d_buf = malloc(size);
+       if (!sec->data->d_buf) {
+               perror("malloc");
+               return -1;
+       }
+       memset(sec->data->d_buf, 0, size);
+
+       if (elf_add_reloc_to_insn(elf, sec, sec->sh.sh_size,
+                                 R_X86_64_PC32, orig->sec, orig->offset)) {
+               WARN("elf_create_reloc: alt_instr::instr_offset");
+               return -1;
+       }
+
+       if (elf_add_reloc(elf, sec, sec->sh.sh_size + 4,
+                         R_X86_64_PC32, sym, 0)) {
+               WARN("elf_create_reloc: alt_instr::repl_offset");
+               return -1;
+       }
+
+       alt->cpuid = cpuid;
+       alt->instrlen = orig_len;
+       alt->replacementlen = repl_len;
+
+       sec->sh.sh_size += size;
+       sec->changed = true;
+
+       return 0;
+}
+
+#define X86_FEATURE_RETPOLINE                ( 7*32+12)
+
+int arch_rewrite_retpolines(struct objtool_file *file)
+{
+       struct instruction *insn;
+       struct reloc *reloc;
+       struct symbol *sym;
+       char name[32] = "";
+
+       list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
+
+               if (!strcmp(insn->sec->name, ".text.__x86.indirect_thunk"))
+                       continue;
+
+               reloc = insn->reloc;
+
+               sprintf(name, "__x86_indirect_alt_%s_%s",
+                       insn->type == INSN_JUMP_DYNAMIC ? "jmp" : "call",
+                       reloc->sym->name + 21);
+
+               sym = find_symbol_by_name(file->elf, name);
+               if (!sym) {
+                       sym = elf_create_undef_symbol(file->elf, name);
+                       if (!sym) {
+                               WARN("elf_create_undef_symbol");
+                               return -1;
+                       }
+               }
+
+               if (elf_add_alternative(file->elf, insn, sym,
+                                       ALT_NOT(X86_FEATURE_RETPOLINE), 5, 5)) {
+                       WARN("elf_add_alternative");
+                       return -1;
+               }
+       }
+
+       return 0;
+}
+
 int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg)
 {
        struct cfi_reg *cfa = &insn->cfi.cfa;