arch/arm64/include/asm/alternative.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_ALTERNATIVE_H
   3 #define __ASM_ALTERNATIVE_H
   4
   5 #include <asm/cpucaps.h>
   6 #include <asm/insn.h>
   7
   8 #ifndef __ASSEMBLY__
   9
  10 #include <linux/init.h>
  11 #include <linux/types.h>
  12 #include <linux/stddef.h>
  13 #include <linux/stringify.h>
  14
  15 struct alt_instr {
  16         s32 orig_offset;        /* offset to original instruction */
  17         s32 alt_offset;         /* offset to replacement instruction */
  18         u16 cpufeature;         /* cpufeature bit set for replacement */
  19         u8  orig_len;           /* size of original instruction(s) */
  20         u8  alt_len;            /* size of new instruction(s), <= orig_len */
  21 };
  22
  23 void __init apply_alternatives_all(void);
  24 void apply_alternatives(void *start, size_t length);
  25
  26 #define ALTINSTR_ENTRY(feature)                                               \
  27         " .word 661b - .\n"                             /* label           */ \
  28         " .word 663f - .\n"                             /* new instruction */ \
  29         " .hword " __stringify(feature) "\n"            /* feature bit     */ \
  30         " .byte 662b-661b\n"                            /* source len      */ \
  31         " .byte 664f-663f\n"                            /* replacement len */
  32
  33 /*
  34  * alternative assembly primitive:
  35  *
  36  * If any of these .org directive fail, it means that insn1 and insn2
  37  * don't have the same length. This used to be written as
  38  *
  39  * .if ((664b-663b) != (662b-661b))
  40  *      .error "Alternatives instruction length mismatch"
  41  * .endif
  42  *
  43  * but most assemblers die if insn1 or insn2 have a .inst. This should
  44  * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
  45  * containing commit 4e4d08cf7399b606 or c1baaddf8861).
  46  */
  47 #define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)     \
  48         ".if "__stringify(cfg_enabled)" == 1\n"                         \
  49         "661:\n\t"                                                      \
  50         oldinstr "\n"                                                   \
  51         "662:\n"                                                        \
  52         ".pushsection .altinstructions,\"a\"\n"                         \
  53         ALTINSTR_ENTRY(feature)                                         \
  54         ".popsection\n"                                                 \
  55         ".pushsection .altinstr_replacement, \"a\"\n"                   \
  56         "663:\n\t"                                                      \
  57         newinstr "\n"                                                   \
  58         "664:\n\t"                                                      \
  59         ".popsection\n\t"                                               \
  60         ".org   . - (664b-663b) + (662b-661b)\n\t"                      \
  61         ".org   . - (662b-661b) + (664b-663b)\n"                        \
  62         ".endif\n"
  63
  64 #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
  65         __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
  66
  67 #else
  68
  69 #include <asm/assembler.h>
  70
  71 .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
  72         .word \orig_offset - .
  73         .word \alt_offset - .
  74         .hword \feature
  75         .byte \orig_len
  76         .byte \alt_len
  77 .endm
  78
  79 .macro alternative_insn insn1, insn2, cap, enable = 1
  80         .if \enable
  81 661:    \insn1
  82 662:    .pushsection .altinstructions, "a"
  83         altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
  84         .popsection
  85         .pushsection .altinstr_replacement, "ax"
  86 663:    \insn2
  87 664:    .popsection
  88         .org    . - (664b-663b) + (662b-661b)
  89         .org    . - (662b-661b) + (664b-663b)
  90         .endif
  91 .endm
  92
  93 /*
  94  * Alternative sequences
  95  *
  96  * The code for the case where the capability is not present will be
  97  * assembled and linked as normal. There are no restrictions on this
  98  * code.
  99  *
 100  * The code for the case where the capability is present will be
 101  * assembled into a special section to be used for dynamic patching.
 102  * Code for that case must:
 103  *
 104  * 1. Be exactly the same length (in bytes) as the default code
 105  *    sequence.
 106  *
 107  * 2. Not contain a branch target that is used outside of the
 108  *    alternative sequence it is defined in (branches into an
 109  *    alternative sequence are not fixed up).
 110  */
 111
 112 /*
 113  * Begin an alternative code sequence.
 114  */
 115 .macro alternative_if_not cap
 116         .set .Lasm_alt_mode, 0
 117         .pushsection .altinstructions, "a"
 118         altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
 119         .popsection
 120 661:
 121 .endm
 122
 123 .macro alternative_if cap
 124         .set .Lasm_alt_mode, 1
 125         .pushsection .altinstructions, "a"
 126         altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
 127         .popsection
 128         .pushsection .altinstr_replacement, "ax"
 129         .align 2        /* So GAS knows label 661 is suitably aligned */
 130 661:
 131 .endm
 132
 133 /*
 134  * Provide the other half of the alternative code sequence.
 135  */
 136 .macro alternative_else
 137 662:
 138         .if .Lasm_alt_mode==0
 139         .pushsection .altinstr_replacement, "ax"
 140         .else
 141         .popsection
 142         .endif
 143 663:
 144 .endm
 145
 146 /*
 147  * Complete an alternative code sequence.
 148  */
 149 .macro alternative_endif
 150 664:
 151         .if .Lasm_alt_mode==0
 152         .popsection
 153         .endif
 154         .org    . - (664b-663b) + (662b-661b)
 155         .org    . - (662b-661b) + (664b-663b)
 156 .endm
 157
 158 /*
 159  * Provides a trivial alternative or default sequence consisting solely
 160  * of NOPs. The number of NOPs is chosen automatically to match the
 161  * previous case.
 162  */
 163 .macro alternative_else_nop_endif
 164 alternative_else
 165         nops    (662b-661b) / AARCH64_INSN_SIZE
 166 alternative_endif
 167 .endm
 168
 169 #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)   \
 170         alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
 171
 172 .macro user_alt, label, oldinstr, newinstr, cond
 173 9999:   alternative_insn "\oldinstr", "\newinstr", \cond
 174         _ASM_EXTABLE 9999b, \label
 175 .endm
 176
 177 /*
 178  * Generate the assembly for UAO alternatives with exception table entries.
 179  * This is complicated as there is no post-increment or pair versions of the
 180  * unprivileged instructions, and USER() only works for single instructions.
 181  */
 182 #ifdef CONFIG_ARM64_UAO
 183         .macro uao_ldp l, reg1, reg2, addr, post_inc
 184                 alternative_if_not ARM64_HAS_UAO
 185 8888:                   ldp     \reg1, \reg2, [\addr], \post_inc;
 186 8889:                   nop;
 187                         nop;
 188                 alternative_else
 189                         ldtr    \reg1, [\addr];
 190                         ldtr    \reg2, [\addr, #8];
 191                         add     \addr, \addr, \post_inc;
 192                 alternative_endif
 193
 194                 _asm_extable    8888b,\l;
 195                 _asm_extable    8889b,\l;
 196         .endm
 197
 198         .macro uao_stp l, reg1, reg2, addr, post_inc
 199                 alternative_if_not ARM64_HAS_UAO
 200 8888:                   stp     \reg1, \reg2, [\addr], \post_inc;
 201 8889:                   nop;
 202                         nop;
 203                 alternative_else
 204                         sttr    \reg1, [\addr];
 205                         sttr    \reg2, [\addr, #8];
 206                         add     \addr, \addr, \post_inc;
 207                 alternative_endif
 208
 209                 _asm_extable    8888b,\l;
 210                 _asm_extable    8889b,\l;
 211         .endm
 212
 213         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 214                 alternative_if_not ARM64_HAS_UAO
 215 8888:                   \inst   \reg, [\addr], \post_inc;
 216                         nop;
 217                 alternative_else
 218                         \alt_inst       \reg, [\addr];
 219                         add             \addr, \addr, \post_inc;
 220                 alternative_endif
 221
 222                 _asm_extable    8888b,\l;
 223         .endm
 224 #else
 225         .macro uao_ldp l, reg1, reg2, addr, post_inc
 226                 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
 227         .endm
 228         .macro uao_stp l, reg1, reg2, addr, post_inc
 229                 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
 230         .endm
 231         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 232                 USER(\l, \inst \reg, [\addr], \post_inc)
 233         .endm
 234 #endif
 235
 236 #endif  /*  __ASSEMBLY__  */
 237
 238 /*
 239  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
 240  *
 241  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
 242  * N.B. If CONFIG_FOO is specified, but not selected, the whole block
 243  *      will be omitted, including oldinstr.
 244  */
 245 #define ALTERNATIVE(oldinstr, newinstr, ...)   \
 246         _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
 247
 248 #endif /* __ASM_ALTERNATIVE_H */