arch/riscv/kernel/patch.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 SiFive
   4  */
   5
   6 #include <linux/spinlock.h>
   7 #include <linux/mm.h>
   8 #include <linux/memory.h>
   9 #include <linux/string.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/stop_machine.h>
  12 #include <asm/kprobes.h>
  13 #include <asm/cacheflush.h>
  14 #include <asm/fixmap.h>
  15 #include <asm/ftrace.h>
  16 #include <asm/patch.h>
  17
  18 struct patch_insn {
  19         void *addr;
  20         u32 *insns;
  21         int ninsns;
  22         atomic_t cpu_count;
  23 };
  24
  25 int riscv_patch_in_stop_machine = false;
  26
  27 #ifdef CONFIG_MMU
  28 /*
  29  * The fix_to_virt(, idx) needs a const value (not a dynamic variable of
  30  * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses".
  31  * So use '__always_inline' and 'const unsigned int fixmap' here.
  32  */
  33 static __always_inline void *patch_map(void *addr, const unsigned int fixmap)
  34 {
  35         uintptr_t uintaddr = (uintptr_t) addr;
  36         struct page *page;
  37
  38         if (core_kernel_text(uintaddr))
  39                 page = phys_to_page(__pa_symbol(addr));
  40         else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
  41                 page = vmalloc_to_page(addr);
  42         else
  43                 return addr;
  44
  45         BUG_ON(!page);
  46
  47         return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
  48                                          (uintaddr & ~PAGE_MASK));
  49 }
  50
  51 static void patch_unmap(int fixmap)
  52 {
  53         clear_fixmap(fixmap);
  54 }
  55 NOKPROBE_SYMBOL(patch_unmap);
  56
  57 static int __patch_insn_set(void *addr, u8 c, size_t len)
  58 {
  59         void *waddr = addr;
  60         bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE;
  61
  62         /*
  63          * Only two pages can be mapped at a time for writing.
  64          */
  65         if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
  66                 return -EINVAL;
  67         /*
  68          * Before reaching here, it was expected to lock the text_mutex
  69          * already, so we don't need to give another lock here and could
  70          * ensure that it was safe between each cores.
  71          */
  72         lockdep_assert_held(&text_mutex);
  73
  74         if (across_pages)
  75                 patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
  76
  77         waddr = patch_map(addr, FIX_TEXT_POKE0);
  78
  79         memset(waddr, c, len);
  80
  81         patch_unmap(FIX_TEXT_POKE0);
  82
  83         if (across_pages)
  84                 patch_unmap(FIX_TEXT_POKE1);
  85
  86         return 0;
  87 }
  88 NOKPROBE_SYMBOL(__patch_insn_set);
  89
  90 static int __patch_insn_write(void *addr, const void *insn, size_t len)
  91 {
  92         void *waddr = addr;
  93         bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
  94         int ret;
  95
  96         /*
  97          * Only two pages can be mapped at a time for writing.
  98          */
  99         if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
 100                 return -EINVAL;
 101
 102         /*
 103          * Before reaching here, it was expected to lock the text_mutex
 104          * already, so we don't need to give another lock here and could
 105          * ensure that it was safe between each cores.
 106          *
 107          * We're currently using stop_machine() for ftrace & kprobes, and while
 108          * that ensures text_mutex is held before installing the mappings it
 109          * does not ensure text_mutex is held by the calling thread.  That's
 110          * safe but triggers a lockdep failure, so just elide it for that
 111          * specific case.
 112          */
 113         if (!riscv_patch_in_stop_machine)
 114                 lockdep_assert_held(&text_mutex);
 115
 116         if (across_pages)
 117                 patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
 118
 119         waddr = patch_map(addr, FIX_TEXT_POKE0);
 120
 121         ret = copy_to_kernel_nofault(waddr, insn, len);
 122
 123         patch_unmap(FIX_TEXT_POKE0);
 124
 125         if (across_pages)
 126                 patch_unmap(FIX_TEXT_POKE1);
 127
 128         return ret;
 129 }
 130 NOKPROBE_SYMBOL(__patch_insn_write);
 131 #else
 132 static int __patch_insn_set(void *addr, u8 c, size_t len)
 133 {
 134         memset(addr, c, len);
 135
 136         return 0;
 137 }
 138 NOKPROBE_SYMBOL(__patch_insn_set);
 139
 140 static int __patch_insn_write(void *addr, const void *insn, size_t len)
 141 {
 142         return copy_to_kernel_nofault(addr, insn, len);
 143 }
 144 NOKPROBE_SYMBOL(__patch_insn_write);
 145 #endif /* CONFIG_MMU */
 146
 147 static int patch_insn_set(void *addr, u8 c, size_t len)
 148 {
 149         size_t patched = 0;
 150         size_t size;
 151         int ret = 0;
 152
 153         /*
 154          * __patch_insn_set() can only work on 2 pages at a time so call it in a
 155          * loop with len <= 2 * PAGE_SIZE.
 156          */
 157         while (patched < len && !ret) {
 158                 size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
 159                 ret = __patch_insn_set(addr + patched, c, size);
 160
 161                 patched += size;
 162         }
 163
 164         return ret;
 165 }
 166 NOKPROBE_SYMBOL(patch_insn_set);
 167
 168 int patch_text_set_nosync(void *addr, u8 c, size_t len)
 169 {
 170         u32 *tp = addr;
 171         int ret;
 172
 173         ret = patch_insn_set(tp, c, len);
 174
 175         if (!ret)
 176                 flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len);
 177
 178         return ret;
 179 }
 180 NOKPROBE_SYMBOL(patch_text_set_nosync);
 181
 182 static int patch_insn_write(void *addr, const void *insn, size_t len)
 183 {
 184         size_t patched = 0;
 185         size_t size;
 186         int ret = 0;
 187
 188         /*
 189          * Copy the instructions to the destination address, two pages at a time
 190          * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
 191          */
 192         while (patched < len && !ret) {
 193                 size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
 194                 ret = __patch_insn_write(addr + patched, insn + patched, size);
 195
 196                 patched += size;
 197         }
 198
 199         return ret;
 200 }
 201 NOKPROBE_SYMBOL(patch_insn_write);
 202
 203 int patch_text_nosync(void *addr, const void *insns, size_t len)
 204 {
 205         u32 *tp = addr;
 206         int ret;
 207
 208         ret = patch_insn_write(tp, insns, len);
 209
 210         if (!ret)
 211                 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
 212
 213         return ret;
 214 }
 215 NOKPROBE_SYMBOL(patch_text_nosync);
 216
 217 static int patch_text_cb(void *data)
 218 {
 219         struct patch_insn *patch = data;
 220         unsigned long len;
 221         int i, ret = 0;
 222
 223         if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
 224                 for (i = 0; ret == 0 && i < patch->ninsns; i++) {
 225                         len = GET_INSN_LENGTH(patch->insns[i]);
 226                         ret = patch_text_nosync(patch->addr + i * len,
 227                                                 &patch->insns[i], len);
 228                 }
 229                 atomic_inc(&patch->cpu_count);
 230         } else {
 231                 while (atomic_read(&patch->cpu_count) <= num_online_cpus())
 232                         cpu_relax();
 233                 smp_mb();
 234         }
 235
 236         return ret;
 237 }
 238 NOKPROBE_SYMBOL(patch_text_cb);
 239
 240 int patch_text(void *addr, u32 *insns, int ninsns)
 241 {
 242         int ret;
 243         struct patch_insn patch = {
 244                 .addr = addr,
 245                 .insns = insns,
 246                 .ninsns = ninsns,
 247                 .cpu_count = ATOMIC_INIT(0),
 248         };
 249
 250         /*
 251          * kprobes takes text_mutex, before calling patch_text(), but as we call
 252          * calls stop_machine(), the lockdep assertion in patch_insn_write()
 253          * gets confused by the context in which the lock is taken.
 254          * Instead, ensure the lock is held before calling stop_machine(), and
 255          * set riscv_patch_in_stop_machine to skip the check in
 256          * patch_insn_write().
 257          */
 258         lockdep_assert_held(&text_mutex);
 259         riscv_patch_in_stop_machine = true;
 260         ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask);
 261         riscv_patch_in_stop_machine = false;
 262         return ret;
 263 }
 264 NOKPROBE_SYMBOL(patch_text);