arch/riscv/kernel/elf_kexec.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Load ELF vmlinux file for the kexec_file_load syscall.
   4  *
   5  * Copyright (C) 2021 Huawei Technologies Co, Ltd.
   6  *
   7  * Author: Liao Chang (liaochang1@huawei.com)
   8  *
   9  * Based on kexec-tools' kexec-elf-riscv.c, heavily modified
  10  * for kernel.
  11  */
  12
  13 #define pr_fmt(fmt)     "kexec_image: " fmt
  14
  15 #include <linux/elf.h>
  16 #include <linux/kexec.h>
  17 #include <linux/slab.h>
  18 #include <linux/of.h>
  19 #include <linux/libfdt.h>
  20 #include <linux/types.h>
  21 #include <linux/memblock.h>
  22 #include <asm/setup.h>
  23
  24 static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr,
  25                                 struct kexec_elf_info *elf_info, unsigned long old_pbase,
  26                                 unsigned long new_pbase)
  27 {
  28         int i;
  29         int ret = 0;
  30         size_t size;
  31         struct kexec_buf kbuf;
  32         const struct elf_phdr *phdr;
  33
  34         kbuf.image = image;
  35
  36         for (i = 0; i < ehdr->e_phnum; i++) {
  37                 phdr = &elf_info->proghdrs[i];
  38                 if (phdr->p_type != PT_LOAD)
  39                         continue;
  40
  41                 size = phdr->p_filesz;
  42                 if (size > phdr->p_memsz)
  43                         size = phdr->p_memsz;
  44
  45                 kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
  46                 kbuf.bufsz = size;
  47                 kbuf.buf_align = phdr->p_align;
  48                 kbuf.mem = phdr->p_paddr - old_pbase + new_pbase;
  49                 kbuf.memsz = phdr->p_memsz;
  50                 kbuf.top_down = false;
  51                 ret = kexec_add_buffer(&kbuf);
  52                 if (ret)
  53                         break;
  54         }
  55
  56         return ret;
  57 }
  58
  59 /*
  60  * Go through the available phsyical memory regions and find one that hold
  61  * an image of the specified size.
  62  */
  63 static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
  64                           struct elfhdr *ehdr, struct kexec_elf_info *elf_info,
  65                           unsigned long *old_pbase, unsigned long *new_pbase)
  66 {
  67         int i;
  68         int ret;
  69         struct kexec_buf kbuf;
  70         const struct elf_phdr *phdr;
  71         unsigned long lowest_paddr = ULONG_MAX;
  72         unsigned long lowest_vaddr = ULONG_MAX;
  73
  74         for (i = 0; i < ehdr->e_phnum; i++) {
  75                 phdr = &elf_info->proghdrs[i];
  76                 if (phdr->p_type != PT_LOAD)
  77                         continue;
  78
  79                 if (lowest_paddr > phdr->p_paddr)
  80                         lowest_paddr = phdr->p_paddr;
  81
  82                 if (lowest_vaddr > phdr->p_vaddr)
  83                         lowest_vaddr = phdr->p_vaddr;
  84         }
  85
  86         kbuf.image = image;
  87         kbuf.buf_min = lowest_paddr;
  88         kbuf.buf_max = ULONG_MAX;
  89         kbuf.buf_align = PAGE_SIZE;
  90         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
  91         kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
  92         kbuf.top_down = false;
  93         ret = arch_kexec_locate_mem_hole(&kbuf);
  94         if (!ret) {
  95                 *old_pbase = lowest_paddr;
  96                 *new_pbase = kbuf.mem;
  97                 image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem;
  98         }
  99         return ret;
 100 }
 101
 102 static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
 103 {
 104         unsigned int *nr_ranges = arg;
 105
 106         (*nr_ranges)++;
 107         return 0;
 108 }
 109
 110 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
 111 {
 112         struct crash_mem *cmem = arg;
 113
 114         cmem->ranges[cmem->nr_ranges].start = res->start;
 115         cmem->ranges[cmem->nr_ranges].end = res->end;
 116         cmem->nr_ranges++;
 117
 118         return 0;
 119 }
 120
 121 static int prepare_elf_headers(void **addr, unsigned long *sz)
 122 {
 123         struct crash_mem *cmem;
 124         unsigned int nr_ranges;
 125         int ret;
 126
 127         nr_ranges = 1; /* For exclusion of crashkernel region */
 128         walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
 129
 130         cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
 131         if (!cmem)
 132                 return -ENOMEM;
 133
 134         cmem->max_nr_ranges = nr_ranges;
 135         cmem->nr_ranges = 0;
 136         ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
 137         if (ret)
 138                 goto out;
 139
 140         /* Exclude crashkernel region */
 141         ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
 142         if (!ret)
 143                 ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
 144
 145 out:
 146         kfree(cmem);
 147         return ret;
 148 }
 149
 150 static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
 151                                  unsigned long cmdline_len)
 152 {
 153         int elfcorehdr_strlen;
 154         char *cmdline_ptr;
 155
 156         cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
 157         if (!cmdline_ptr)
 158                 return NULL;
 159
 160         elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
 161                 image->elf_load_addr);
 162
 163         if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
 164                 pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
 165                 kfree(cmdline_ptr);
 166                 return NULL;
 167         }
 168
 169         memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
 170         /* Ensure it's nul terminated */
 171         cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
 172         return cmdline_ptr;
 173 }
 174
 175 static void *elf_kexec_load(struct kimage *image, char *kernel_buf,
 176                             unsigned long kernel_len, char *initrd,
 177                             unsigned long initrd_len, char *cmdline,
 178                             unsigned long cmdline_len)
 179 {
 180         int ret;
 181         unsigned long old_kernel_pbase = ULONG_MAX;
 182         unsigned long new_kernel_pbase = 0UL;
 183         unsigned long initrd_pbase = 0UL;
 184         unsigned long headers_sz;
 185         unsigned long kernel_start;
 186         void *fdt, *headers;
 187         struct elfhdr ehdr;
 188         struct kexec_buf kbuf;
 189         struct kexec_elf_info elf_info;
 190         char *modified_cmdline = NULL;
 191
 192         ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info);
 193         if (ret)
 194                 return ERR_PTR(ret);
 195
 196         ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info,
 197                              &old_kernel_pbase, &new_kernel_pbase);
 198         if (ret)
 199                 goto out;
 200         kernel_start = image->start;
 201         pr_notice("The entry point of kernel at 0x%lx\n", image->start);
 202
 203         /* Add the kernel binary to the image */
 204         ret = riscv_kexec_elf_load(image, &ehdr, &elf_info,
 205                                    old_kernel_pbase, new_kernel_pbase);
 206         if (ret)
 207                 goto out;
 208
 209         kbuf.image = image;
 210         kbuf.buf_min = new_kernel_pbase + kernel_len;
 211         kbuf.buf_max = ULONG_MAX;
 212
 213         /* Add elfcorehdr */
 214         if (image->type == KEXEC_TYPE_CRASH) {
 215                 ret = prepare_elf_headers(&headers, &headers_sz);
 216                 if (ret) {
 217                         pr_err("Preparing elf core header failed\n");
 218                         goto out;
 219                 }
 220
 221                 kbuf.buffer = headers;
 222                 kbuf.bufsz = headers_sz;
 223                 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 224                 kbuf.memsz = headers_sz;
 225                 kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
 226                 kbuf.top_down = true;
 227
 228                 ret = kexec_add_buffer(&kbuf);
 229                 if (ret) {
 230                         vfree(headers);
 231                         goto out;
 232                 }
 233                 image->elf_headers = headers;
 234                 image->elf_load_addr = kbuf.mem;
 235                 image->elf_headers_sz = headers_sz;
 236
 237                 pr_debug("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
 238                          image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
 239
 240                 /* Setup cmdline for kdump kernel case */
 241                 modified_cmdline = setup_kdump_cmdline(image, cmdline,
 242                                                        cmdline_len);
 243                 if (!modified_cmdline) {
 244                         pr_err("Setting up cmdline for kdump kernel failed\n");
 245                         ret = -EINVAL;
 246                         goto out;
 247                 }
 248                 cmdline = modified_cmdline;
 249         }
 250
 251 #ifdef CONFIG_ARCH_HAS_KEXEC_PURGATORY
 252         /* Add purgatory to the image */
 253         kbuf.top_down = true;
 254         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 255         ret = kexec_load_purgatory(image, &kbuf);
 256         if (ret) {
 257                 pr_err("Error loading purgatory ret=%d\n", ret);
 258                 goto out;
 259         }
 260         ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
 261                                              &kernel_start,
 262                                              sizeof(kernel_start), 0);
 263         if (ret)
 264                 pr_err("Error update purgatory ret=%d\n", ret);
 265 #endif /* CONFIG_ARCH_HAS_KEXEC_PURGATORY */
 266
 267         /* Add the initrd to the image */
 268         if (initrd != NULL) {
 269                 kbuf.buffer = initrd;
 270                 kbuf.bufsz = kbuf.memsz = initrd_len;
 271                 kbuf.buf_align = PAGE_SIZE;
 272                 kbuf.top_down = false;
 273                 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 274                 ret = kexec_add_buffer(&kbuf);
 275                 if (ret)
 276                         goto out;
 277                 initrd_pbase = kbuf.mem;
 278                 pr_notice("Loaded initrd at 0x%lx\n", initrd_pbase);
 279         }
 280
 281         /* Add the DTB to the image */
 282         fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
 283                                            initrd_len, cmdline, 0);
 284         if (!fdt) {
 285                 pr_err("Error setting up the new device tree.\n");
 286                 ret = -EINVAL;
 287                 goto out;
 288         }
 289
 290         fdt_pack(fdt);
 291         kbuf.buffer = fdt;
 292         kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
 293         kbuf.buf_align = PAGE_SIZE;
 294         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 295         kbuf.top_down = true;
 296         ret = kexec_add_buffer(&kbuf);
 297         if (ret) {
 298                 pr_err("Error add DTB kbuf ret=%d\n", ret);
 299                 goto out_free_fdt;
 300         }
 301         pr_notice("Loaded device tree at 0x%lx\n", kbuf.mem);
 302         goto out;
 303
 304 out_free_fdt:
 305         kvfree(fdt);
 306 out:
 307         kfree(modified_cmdline);
 308         kexec_free_elf_info(&elf_info);
 309         return ret ? ERR_PTR(ret) : NULL;
 310 }
 311
 312 #define RV_X(x, s, n)  (((x) >> (s)) & ((1 << (n)) - 1))
 313 #define RISCV_IMM_BITS 12
 314 #define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
 315 #define RISCV_CONST_HIGH_PART(x) \
 316         (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
 317 #define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
 318
 319 #define ENCODE_ITYPE_IMM(x) \
 320         (RV_X(x, 0, 12) << 20)
 321 #define ENCODE_BTYPE_IMM(x) \
 322         ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
 323         (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
 324 #define ENCODE_UTYPE_IMM(x) \
 325         (RV_X(x, 12, 20) << 12)
 326 #define ENCODE_JTYPE_IMM(x) \
 327         ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
 328         (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
 329 #define ENCODE_CBTYPE_IMM(x) \
 330         ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
 331         (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
 332 #define ENCODE_CJTYPE_IMM(x) \
 333         ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
 334         (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
 335         (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
 336 #define ENCODE_UJTYPE_IMM(x) \
 337         (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
 338         (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
 339 #define ENCODE_UITYPE_IMM(x) \
 340         (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))
 341
 342 #define CLEAN_IMM(type, x) \
 343         ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
 344
 345 int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
 346                                      Elf_Shdr *section,
 347                                      const Elf_Shdr *relsec,
 348                                      const Elf_Shdr *symtab)
 349 {
 350         const char *strtab, *name, *shstrtab;
 351         const Elf_Shdr *sechdrs;
 352         Elf64_Rela *relas;
 353         int i, r_type;
 354
 355         /* String & section header string table */
 356         sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
 357         strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
 358         shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
 359
 360         relas = (void *)pi->ehdr + relsec->sh_offset;
 361
 362         for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
 363                 const Elf_Sym *sym;     /* symbol to relocate */
 364                 unsigned long addr;     /* final location after relocation */
 365                 unsigned long val;      /* relocated symbol value */
 366                 unsigned long sec_base; /* relocated symbol value */
 367                 void *loc;              /* tmp location to modify */
 368
 369                 sym = (void *)pi->ehdr + symtab->sh_offset;
 370                 sym += ELF64_R_SYM(relas[i].r_info);
 371
 372                 if (sym->st_name)
 373                         name = strtab + sym->st_name;
 374                 else
 375                         name = shstrtab + sechdrs[sym->st_shndx].sh_name;
 376
 377                 loc = pi->purgatory_buf;
 378                 loc += section->sh_offset;
 379                 loc += relas[i].r_offset;
 380
 381                 if (sym->st_shndx == SHN_ABS)
 382                         sec_base = 0;
 383                 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
 384                         pr_err("Invalid section %d for symbol %s\n",
 385                                sym->st_shndx, name);
 386                         return -ENOEXEC;
 387                 } else
 388                         sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
 389
 390                 val = sym->st_value;
 391                 val += sec_base;
 392                 val += relas[i].r_addend;
 393
 394                 addr = section->sh_addr + relas[i].r_offset;
 395
 396                 r_type = ELF64_R_TYPE(relas[i].r_info);
 397
 398                 switch (r_type) {
 399                 case R_RISCV_BRANCH:
 400                         *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
 401                                  ENCODE_BTYPE_IMM(val - addr);
 402                         break;
 403                 case R_RISCV_JAL:
 404                         *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
 405                                  ENCODE_JTYPE_IMM(val - addr);
 406                         break;
 407                 /*
 408                  * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
 409                  * sym is expected to be next to R_RISCV_PCREL_HI20
 410                  * in purgatory relsec. Handle it like R_RISCV_CALL
 411                  * sym, instead of searching the whole relsec.
 412                  */
 413                 case R_RISCV_PCREL_HI20:
 414                 case R_RISCV_CALL:
 415                         *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
 416                                  ENCODE_UJTYPE_IMM(val - addr);
 417                         break;
 418                 case R_RISCV_RVC_BRANCH:
 419                         *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
 420                                  ENCODE_CBTYPE_IMM(val - addr);
 421                         break;
 422                 case R_RISCV_RVC_JUMP:
 423                         *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
 424                                  ENCODE_CJTYPE_IMM(val - addr);
 425                         break;
 426                 case R_RISCV_ADD32:
 427                         *(u32 *)loc += val;
 428                         break;
 429                 case R_RISCV_SUB32:
 430                         *(u32 *)loc -= val;
 431                         break;
 432                 /* It has been applied by R_RISCV_PCREL_HI20 sym */
 433                 case R_RISCV_PCREL_LO12_I:
 434                 case R_RISCV_ALIGN:
 435                 case R_RISCV_RELAX:
 436                         break;
 437                 default:
 438                         pr_err("Unknown rela relocation: %d\n", r_type);
 439                         return -ENOEXEC;
 440                 }
 441         }
 442         return 0;
 443 }
 444
 445 const struct kexec_file_ops elf_kexec_ops = {
 446         .probe = kexec_elf_probe,
 447         .load  = elf_kexec_load,
 448 };