arch/loongarch/kernel/alternative.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/mm.h>
   3 #include <linux/module.h>
   4 #include <asm/alternative.h>
   5 #include <asm/cacheflush.h>
   6 #include <asm/inst.h>
   7 #include <asm/sections.h>
   8
   9 int __read_mostly alternatives_patched;
  10
  11 EXPORT_SYMBOL_GPL(alternatives_patched);
  12
  13 #define MAX_PATCH_SIZE (((u8)(-1)) / LOONGARCH_INSN_SIZE)
  14
  15 static int __initdata_or_module debug_alternative;
  16
  17 static int __init debug_alt(char *str)
  18 {
  19         debug_alternative = 1;
  20         return 1;
  21 }
  22 __setup("debug-alternative", debug_alt);
  23
  24 #define DPRINTK(fmt, args...)                                           \
  25 do {                                                                    \
  26         if (debug_alternative)                                          \
  27                 printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args);   \
  28 } while (0)
  29
  30 #define DUMP_WORDS(buf, count, fmt, args...)                            \
  31 do {                                                                    \
  32         if (unlikely(debug_alternative)) {                              \
  33                 int _j;                                                 \
  34                 union loongarch_instruction *_buf = buf;                \
  35                                                                         \
  36                 if (!(count))                                           \
  37                         break;                                          \
  38                                                                         \
  39                 printk(KERN_DEBUG fmt, ##args);                         \
  40                 for (_j = 0; _j < count - 1; _j++)                      \
  41                         printk(KERN_CONT "<%08x> ", _buf[_j].word);     \
  42                 printk(KERN_CONT "<%08x>\n", _buf[_j].word);            \
  43         }                                                               \
  44 } while (0)
  45
  46 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
  47 static void __init_or_module add_nops(union loongarch_instruction *insn, int count)
  48 {
  49         while (count--) {
  50                 insn->word = INSN_NOP;
  51                 insn++;
  52         }
  53 }
  54
  55 /* Is the jump addr in local .altinstructions */
  56 static inline bool in_alt_jump(unsigned long jump, void *start, void *end)
  57 {
  58         return jump >= (unsigned long)start && jump < (unsigned long)end;
  59 }
  60
  61 static void __init_or_module recompute_jump(union loongarch_instruction *buf,
  62                 union loongarch_instruction *dest, union loongarch_instruction *src,
  63                 void *start, void *end)
  64 {
  65         unsigned int si, si_l, si_h;
  66         unsigned long cur_pc, jump_addr, pc;
  67         long offset;
  68
  69         cur_pc = (unsigned long)src;
  70         pc = (unsigned long)dest;
  71
  72         si_l = src->reg0i26_format.immediate_l;
  73         si_h = src->reg0i26_format.immediate_h;
  74         switch (src->reg0i26_format.opcode) {
  75         case b_op:
  76         case bl_op:
  77                 jump_addr = cur_pc + sign_extend((si_h << 16 | si_l) << 2, 27);
  78                 if (in_alt_jump(jump_addr, start, end))
  79                         return;
  80                 offset = jump_addr - pc;
  81                 BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
  82                 offset >>= 2;
  83                 buf->reg0i26_format.immediate_h = offset >> 16;
  84                 buf->reg0i26_format.immediate_l = offset;
  85                 return;
  86         }
  87
  88         si_l = src->reg1i21_format.immediate_l;
  89         si_h = src->reg1i21_format.immediate_h;
  90         switch (src->reg1i21_format.opcode) {
  91         case bceqz_op: /* bceqz_op = bcnez_op */
  92                 BUG_ON(buf->reg1i21_format.rj & BIT(4));
  93                 fallthrough;
  94         case beqz_op:
  95         case bnez_op:
  96                 jump_addr = cur_pc + sign_extend((si_h << 16 | si_l) << 2, 22);
  97                 if (in_alt_jump(jump_addr, start, end))
  98                         return;
  99                 offset = jump_addr - pc;
 100                 BUG_ON(offset < -SZ_4M || offset >= SZ_4M);
 101                 offset >>= 2;
 102                 buf->reg1i21_format.immediate_h = offset >> 16;
 103                 buf->reg1i21_format.immediate_l = offset;
 104                 return;
 105         }
 106
 107         si = src->reg2i16_format.immediate;
 108         switch (src->reg2i16_format.opcode) {
 109         case beq_op:
 110         case bne_op:
 111         case blt_op:
 112         case bge_op:
 113         case bltu_op:
 114         case bgeu_op:
 115                 jump_addr = cur_pc + sign_extend(si << 2, 17);
 116                 if (in_alt_jump(jump_addr, start, end))
 117                         return;
 118                 offset = jump_addr - pc;
 119                 BUG_ON(offset < -SZ_128K || offset >= SZ_128K);
 120                 offset >>= 2;
 121                 buf->reg2i16_format.immediate = offset;
 122                 return;
 123         }
 124 }
 125
 126 static int __init_or_module copy_alt_insns(union loongarch_instruction *buf,
 127         union loongarch_instruction *dest, union loongarch_instruction *src, int nr)
 128 {
 129         int i;
 130
 131         for (i = 0; i < nr; i++) {
 132                 buf[i].word = src[i].word;
 133
 134                 if (is_pc_ins(&src[i])) {
 135                         pr_err("Not support pcrel instruction at present!");
 136                         return -EINVAL;
 137                 }
 138
 139                 if (is_branch_ins(&src[i]) &&
 140                     src[i].reg2i16_format.opcode != jirl_op) {
 141                         recompute_jump(&buf[i], &dest[i], &src[i], src, src + nr);
 142                 }
 143         }
 144
 145         return 0;
 146 }
 147
 148 /*
 149  * text_poke_early - Update instructions on a live kernel at boot time
 150  *
 151  * When you use this code to patch more than one byte of an instruction
 152  * you need to make sure that other CPUs cannot execute this code in parallel.
 153  * Also no thread must be currently preempted in the middle of these
 154  * instructions. And on the local CPU you need to be protected again NMI or MCE
 155  * handlers seeing an inconsistent instruction while you patch.
 156  */
 157 static void *__init_or_module text_poke_early(union loongarch_instruction *insn,
 158                               union loongarch_instruction *buf, unsigned int nr)
 159 {
 160         int i;
 161         unsigned long flags;
 162
 163         local_irq_save(flags);
 164
 165         for (i = 0; i < nr; i++)
 166                 insn[i].word = buf[i].word;
 167
 168         local_irq_restore(flags);
 169
 170         wbflush();
 171         flush_icache_range((unsigned long)insn, (unsigned long)(insn + nr));
 172
 173         return insn;
 174 }
 175
 176 /*
 177  * Replace instructions with better alternatives for this CPU type. This runs
 178  * before SMP is initialized to avoid SMP problems with self modifying code.
 179  * This implies that asymmetric systems where APs have less capabilities than
 180  * the boot processor are not handled. Tough. Make sure you disable such
 181  * features by hand.
 182  */
 183 void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 184 {
 185         struct alt_instr *a;
 186         unsigned int nr_instr, nr_repl, nr_insnbuf;
 187         union loongarch_instruction *instr, *replacement;
 188         union loongarch_instruction insnbuf[MAX_PATCH_SIZE];
 189
 190         DPRINTK("alt table %px, -> %px", start, end);
 191         /*
 192          * The scan order should be from start to end. A later scanned
 193          * alternative code can overwrite previously scanned alternative code.
 194          * Some kernel functions (e.g. memcpy, memset, etc) use this order to
 195          * patch code.
 196          *
 197          * So be careful if you want to change the scan order to any other
 198          * order.
 199          */
 200         for (a = start; a < end; a++) {
 201                 nr_insnbuf = 0;
 202
 203                 instr = (void *)&a->instr_offset + a->instr_offset;
 204                 replacement = (void *)&a->replace_offset + a->replace_offset;
 205
 206                 BUG_ON(a->instrlen > sizeof(insnbuf));
 207                 BUG_ON(a->instrlen & 0x3);
 208                 BUG_ON(a->replacementlen & 0x3);
 209
 210                 nr_instr = a->instrlen / LOONGARCH_INSN_SIZE;
 211                 nr_repl = a->replacementlen / LOONGARCH_INSN_SIZE;
 212
 213                 if (!cpu_has(a->feature)) {
 214                         DPRINTK("feat not exist: %d, old: (%px len: %d), repl: (%px, len: %d)",
 215                                 a->feature, instr, a->instrlen,
 216                                 replacement, a->replacementlen);
 217
 218                         continue;
 219                 }
 220
 221                 DPRINTK("feat: %d, old: (%px len: %d), repl: (%px, len: %d)",
 222                         a->feature, instr, a->instrlen,
 223                         replacement, a->replacementlen);
 224
 225                 DUMP_WORDS(instr, nr_instr, "%px: old_insn: ", instr);
 226                 DUMP_WORDS(replacement, nr_repl, "%px: rpl_insn: ", replacement);
 227
 228                 copy_alt_insns(insnbuf, instr, replacement, nr_repl);
 229                 nr_insnbuf = nr_repl;
 230
 231                 if (nr_instr > nr_repl) {
 232                         add_nops(insnbuf + nr_repl, nr_instr - nr_repl);
 233                         nr_insnbuf += nr_instr - nr_repl;
 234                 }
 235                 DUMP_WORDS(insnbuf, nr_insnbuf, "%px: final_insn: ", instr);
 236
 237                 text_poke_early(instr, insnbuf, nr_insnbuf);
 238         }
 239 }
 240
 241 void __init alternative_instructions(void)
 242 {
 243         apply_alternatives(__alt_instructions, __alt_instructions_end);
 244
 245         alternatives_patched = 1;
 246 }