arch/powerpc/kernel/kexec_elf_64.c

   1 /*
   2  * Load ELF vmlinux file for the kexec_file_load syscall.
   3  *
   4  * Copyright (C) 2004  Adam Litke (agl@us.ibm.com)
   5  * Copyright (C) 2004  IBM Corp.
   6  * Copyright (C) 2005  R Sharada (sharada@in.ibm.com)
   7  * Copyright (C) 2006  Mohan Kumar M (mohan@in.ibm.com)
   8  * Copyright (C) 2016  IBM Corporation
   9  *
  10  * Based on kexec-tools' kexec-elf-exec.c and kexec-elf-ppc64.c.
  11  * Heavily modified for the kernel by
  12  * Thiago Jung Bauermann <bauerman@linux.vnet.ibm.com>.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation (version 2 of the License).
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  */
  23
  24 #define pr_fmt(fmt)     "kexec_elf: " fmt
  25
  26 #include <linux/elf.h>
  27 #include <linux/kexec.h>
  28 #include <linux/libfdt.h>
  29 #include <linux/module.h>
  30 #include <linux/of_fdt.h>
  31 #include <linux/slab.h>
  32 #include <linux/types.h>
  33
  34 #define PURGATORY_STACK_SIZE    (16 * 1024)
  35
  36 #define elf_addr_to_cpu elf64_to_cpu
  37
  38 #ifndef Elf_Rel
  39 #define Elf_Rel         Elf64_Rel
  40 #endif /* Elf_Rel */
  41
  42 struct elf_info {
  43         /*
  44          * Where the ELF binary contents are kept.
  45          * Memory managed by the user of the struct.
  46          */
  47         const char *buffer;
  48
  49         const struct elfhdr *ehdr;
  50         const struct elf_phdr *proghdrs;
  51         struct elf_shdr *sechdrs;
  52 };
  53
  54 static inline bool elf_is_elf_file(const struct elfhdr *ehdr)
  55 {
  56        return memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0;
  57 }
  58
  59 static uint64_t elf64_to_cpu(const struct elfhdr *ehdr, uint64_t value)
  60 {
  61         if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
  62                 value = le64_to_cpu(value);
  63         else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
  64                 value = be64_to_cpu(value);
  65
  66         return value;
  67 }
  68
  69 static uint16_t elf16_to_cpu(const struct elfhdr *ehdr, uint16_t value)
  70 {
  71         if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
  72                 value = le16_to_cpu(value);
  73         else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
  74                 value = be16_to_cpu(value);
  75
  76         return value;
  77 }
  78
  79 static uint32_t elf32_to_cpu(const struct elfhdr *ehdr, uint32_t value)
  80 {
  81         if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
  82                 value = le32_to_cpu(value);
  83         else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
  84                 value = be32_to_cpu(value);
  85
  86         return value;
  87 }
  88
  89 /**
  90  * elf_is_ehdr_sane - check that it is safe to use the ELF header
  91  * @buf_len:    size of the buffer in which the ELF file is loaded.
  92  */
  93 static bool elf_is_ehdr_sane(const struct elfhdr *ehdr, size_t buf_len)
  94 {
  95         if (ehdr->e_phnum > 0 && ehdr->e_phentsize != sizeof(struct elf_phdr)) {
  96                 pr_debug("Bad program header size.\n");
  97                 return false;
  98         } else if (ehdr->e_shnum > 0 &&
  99                    ehdr->e_shentsize != sizeof(struct elf_shdr)) {
 100                 pr_debug("Bad section header size.\n");
 101                 return false;
 102         } else if (ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
 103                    ehdr->e_version != EV_CURRENT) {
 104                 pr_debug("Unknown ELF version.\n");
 105                 return false;
 106         }
 107
 108         if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
 109                 size_t phdr_size;
 110
 111                 /*
 112                  * e_phnum is at most 65535 so calculating the size of the
 113                  * program header cannot overflow.
 114                  */
 115                 phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
 116
 117                 /* Sanity check the program header table location. */
 118                 if (ehdr->e_phoff + phdr_size < ehdr->e_phoff) {
 119                         pr_debug("Program headers at invalid location.\n");
 120                         return false;
 121                 } else if (ehdr->e_phoff + phdr_size > buf_len) {
 122                         pr_debug("Program headers truncated.\n");
 123                         return false;
 124                 }
 125         }
 126
 127         if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
 128                 size_t shdr_size;
 129
 130                 /*
 131                  * e_shnum is at most 65536 so calculating
 132                  * the size of the section header cannot overflow.
 133                  */
 134                 shdr_size = sizeof(struct elf_shdr) * ehdr->e_shnum;
 135
 136                 /* Sanity check the section header table location. */
 137                 if (ehdr->e_shoff + shdr_size < ehdr->e_shoff) {
 138                         pr_debug("Section headers at invalid location.\n");
 139                         return false;
 140                 } else if (ehdr->e_shoff + shdr_size > buf_len) {
 141                         pr_debug("Section headers truncated.\n");
 142                         return false;
 143                 }
 144         }
 145
 146         return true;
 147 }
 148
 149 static int elf_read_ehdr(const char *buf, size_t len, struct elfhdr *ehdr)
 150 {
 151         struct elfhdr *buf_ehdr;
 152
 153         if (len < sizeof(*buf_ehdr)) {
 154                 pr_debug("Buffer is too small to hold ELF header.\n");
 155                 return -ENOEXEC;
 156         }
 157
 158         memset(ehdr, 0, sizeof(*ehdr));
 159         memcpy(ehdr->e_ident, buf, sizeof(ehdr->e_ident));
 160         if (!elf_is_elf_file(ehdr)) {
 161                 pr_debug("No ELF header magic.\n");
 162                 return -ENOEXEC;
 163         }
 164
 165         if (ehdr->e_ident[EI_CLASS] != ELF_CLASS) {
 166                 pr_debug("Not a supported ELF class.\n");
 167                 return -ENOEXEC;
 168         } else  if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
 169                 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
 170                 pr_debug("Not a supported ELF data format.\n");
 171                 return -ENOEXEC;
 172         }
 173
 174         buf_ehdr = (struct elfhdr *) buf;
 175         if (elf16_to_cpu(ehdr, buf_ehdr->e_ehsize) != sizeof(*buf_ehdr)) {
 176                 pr_debug("Bad ELF header size.\n");
 177                 return -ENOEXEC;
 178         }
 179
 180         ehdr->e_type      = elf16_to_cpu(ehdr, buf_ehdr->e_type);
 181         ehdr->e_machine   = elf16_to_cpu(ehdr, buf_ehdr->e_machine);
 182         ehdr->e_version   = elf32_to_cpu(ehdr, buf_ehdr->e_version);
 183         ehdr->e_entry     = elf_addr_to_cpu(ehdr, buf_ehdr->e_entry);
 184         ehdr->e_phoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_phoff);
 185         ehdr->e_shoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_shoff);
 186         ehdr->e_flags     = elf32_to_cpu(ehdr, buf_ehdr->e_flags);
 187         ehdr->e_phentsize = elf16_to_cpu(ehdr, buf_ehdr->e_phentsize);
 188         ehdr->e_phnum     = elf16_to_cpu(ehdr, buf_ehdr->e_phnum);
 189         ehdr->e_shentsize = elf16_to_cpu(ehdr, buf_ehdr->e_shentsize);
 190         ehdr->e_shnum     = elf16_to_cpu(ehdr, buf_ehdr->e_shnum);
 191         ehdr->e_shstrndx  = elf16_to_cpu(ehdr, buf_ehdr->e_shstrndx);
 192
 193         return elf_is_ehdr_sane(ehdr, len) ? 0 : -ENOEXEC;
 194 }
 195
 196 /**
 197  * elf_is_phdr_sane - check that it is safe to use the program header
 198  * @buf_len:    size of the buffer in which the ELF file is loaded.
 199  */
 200 static bool elf_is_phdr_sane(const struct elf_phdr *phdr, size_t buf_len)
 201 {
 202
 203         if (phdr->p_offset + phdr->p_filesz < phdr->p_offset) {
 204                 pr_debug("ELF segment location wraps around.\n");
 205                 return false;
 206         } else if (phdr->p_offset + phdr->p_filesz > buf_len) {
 207                 pr_debug("ELF segment not in file.\n");
 208                 return false;
 209         } else if (phdr->p_paddr + phdr->p_memsz < phdr->p_paddr) {
 210                 pr_debug("ELF segment address wraps around.\n");
 211                 return false;
 212         }
 213
 214         return true;
 215 }
 216
 217 static int elf_read_phdr(const char *buf, size_t len, struct elf_info *elf_info,
 218                          int idx)
 219 {
 220         /* Override the const in proghdrs, we are the ones doing the loading. */
 221         struct elf_phdr *phdr = (struct elf_phdr *) &elf_info->proghdrs[idx];
 222         const char *pbuf;
 223         struct elf_phdr *buf_phdr;
 224
 225         pbuf = buf + elf_info->ehdr->e_phoff + (idx * sizeof(*buf_phdr));
 226         buf_phdr = (struct elf_phdr *) pbuf;
 227
 228         phdr->p_type   = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_type);
 229         phdr->p_offset = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_offset);
 230         phdr->p_paddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_paddr);
 231         phdr->p_vaddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_vaddr);
 232         phdr->p_flags  = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_flags);
 233
 234         /*
 235          * The following fields have a type equivalent to Elf_Addr
 236          * both in 32 bit and 64 bit ELF.
 237          */
 238         phdr->p_filesz = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_filesz);
 239         phdr->p_memsz  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_memsz);
 240         phdr->p_align  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_align);
 241
 242         return elf_is_phdr_sane(phdr, len) ? 0 : -ENOEXEC;
 243 }
 244
 245 /**
 246  * elf_read_phdrs - read the program headers from the buffer
 247  *
 248  * This function assumes that the program header table was checked for sanity.
 249  * Use elf_is_ehdr_sane() if it wasn't.
 250  */
 251 static int elf_read_phdrs(const char *buf, size_t len,
 252                           struct elf_info *elf_info)
 253 {
 254         size_t phdr_size, i;
 255         const struct elfhdr *ehdr = elf_info->ehdr;
 256
 257         /*
 258          * e_phnum is at most 65535 so calculating the size of the
 259          * program header cannot overflow.
 260          */
 261         phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
 262
 263         elf_info->proghdrs = kzalloc(phdr_size, GFP_KERNEL);
 264         if (!elf_info->proghdrs)
 265                 return -ENOMEM;
 266
 267         for (i = 0; i < ehdr->e_phnum; i++) {
 268                 int ret;
 269
 270                 ret = elf_read_phdr(buf, len, elf_info, i);
 271                 if (ret) {
 272                         kfree(elf_info->proghdrs);
 273                         elf_info->proghdrs = NULL;
 274                         return ret;
 275                 }
 276         }
 277
 278         return 0;
 279 }
 280
 281 /**
 282  * elf_is_shdr_sane - check that it is safe to use the section header
 283  * @buf_len:    size of the buffer in which the ELF file is loaded.
 284  */
 285 static bool elf_is_shdr_sane(const struct elf_shdr *shdr, size_t buf_len)
 286 {
 287         bool size_ok;
 288
 289         /* SHT_NULL headers have undefined values, so we can't check them. */
 290         if (shdr->sh_type == SHT_NULL)
 291                 return true;
 292
 293         /* Now verify sh_entsize */
 294         switch (shdr->sh_type) {
 295         case SHT_SYMTAB:
 296                 size_ok = shdr->sh_entsize == sizeof(Elf_Sym);
 297                 break;
 298         case SHT_RELA:
 299                 size_ok = shdr->sh_entsize == sizeof(Elf_Rela);
 300                 break;
 301         case SHT_DYNAMIC:
 302                 size_ok = shdr->sh_entsize == sizeof(Elf_Dyn);
 303                 break;
 304         case SHT_REL:
 305                 size_ok = shdr->sh_entsize == sizeof(Elf_Rel);
 306                 break;
 307         case SHT_NOTE:
 308         case SHT_PROGBITS:
 309         case SHT_HASH:
 310         case SHT_NOBITS:
 311         default:
 312                 /*
 313                  * This is a section whose entsize requirements
 314                  * I don't care about.  If I don't know about
 315                  * the section I can't care about it's entsize
 316                  * requirements.
 317                  */
 318                 size_ok = true;
 319                 break;
 320         }
 321
 322         if (!size_ok) {
 323                 pr_debug("ELF section with wrong entry size.\n");
 324                 return false;
 325         } else if (shdr->sh_addr + shdr->sh_size < shdr->sh_addr) {
 326                 pr_debug("ELF section address wraps around.\n");
 327                 return false;
 328         }
 329
 330         if (shdr->sh_type != SHT_NOBITS) {
 331                 if (shdr->sh_offset + shdr->sh_size < shdr->sh_offset) {
 332                         pr_debug("ELF section location wraps around.\n");
 333                         return false;
 334                 } else if (shdr->sh_offset + shdr->sh_size > buf_len) {
 335                         pr_debug("ELF section not in file.\n");
 336                         return false;
 337                 }
 338         }
 339
 340         return true;
 341 }
 342
 343 static int elf_read_shdr(const char *buf, size_t len, struct elf_info *elf_info,
 344                          int idx)
 345 {
 346         struct elf_shdr *shdr = &elf_info->sechdrs[idx];
 347         const struct elfhdr *ehdr = elf_info->ehdr;
 348         const char *sbuf;
 349         struct elf_shdr *buf_shdr;
 350
 351         sbuf = buf + ehdr->e_shoff + idx * sizeof(*buf_shdr);
 352         buf_shdr = (struct elf_shdr *) sbuf;
 353
 354         shdr->sh_name      = elf32_to_cpu(ehdr, buf_shdr->sh_name);
 355         shdr->sh_type      = elf32_to_cpu(ehdr, buf_shdr->sh_type);
 356         shdr->sh_addr      = elf_addr_to_cpu(ehdr, buf_shdr->sh_addr);
 357         shdr->sh_offset    = elf_addr_to_cpu(ehdr, buf_shdr->sh_offset);
 358         shdr->sh_link      = elf32_to_cpu(ehdr, buf_shdr->sh_link);
 359         shdr->sh_info      = elf32_to_cpu(ehdr, buf_shdr->sh_info);
 360
 361         /*
 362          * The following fields have a type equivalent to Elf_Addr
 363          * both in 32 bit and 64 bit ELF.
 364          */
 365         shdr->sh_flags     = elf_addr_to_cpu(ehdr, buf_shdr->sh_flags);
 366         shdr->sh_size      = elf_addr_to_cpu(ehdr, buf_shdr->sh_size);
 367         shdr->sh_addralign = elf_addr_to_cpu(ehdr, buf_shdr->sh_addralign);
 368         shdr->sh_entsize   = elf_addr_to_cpu(ehdr, buf_shdr->sh_entsize);
 369
 370         return elf_is_shdr_sane(shdr, len) ? 0 : -ENOEXEC;
 371 }
 372
 373 /**
 374  * elf_read_shdrs - read the section headers from the buffer
 375  *
 376  * This function assumes that the section header table was checked for sanity.
 377  * Use elf_is_ehdr_sane() if it wasn't.
 378  */
 379 static int elf_read_shdrs(const char *buf, size_t len,
 380                           struct elf_info *elf_info)
 381 {
 382         size_t shdr_size, i;
 383
 384         /*
 385          * e_shnum is at most 65536 so calculating
 386          * the size of the section header cannot overflow.
 387          */
 388         shdr_size = sizeof(struct elf_shdr) * elf_info->ehdr->e_shnum;
 389
 390         elf_info->sechdrs = kzalloc(shdr_size, GFP_KERNEL);
 391         if (!elf_info->sechdrs)
 392                 return -ENOMEM;
 393
 394         for (i = 0; i < elf_info->ehdr->e_shnum; i++) {
 395                 int ret;
 396
 397                 ret = elf_read_shdr(buf, len, elf_info, i);
 398                 if (ret) {
 399                         kfree(elf_info->sechdrs);
 400                         elf_info->sechdrs = NULL;
 401                         return ret;
 402                 }
 403         }
 404
 405         return 0;
 406 }
 407
 408 /**
 409  * elf_read_from_buffer - read ELF file and sets up ELF header and ELF info
 410  * @buf:        Buffer to read ELF file from.
 411  * @len:        Size of @buf.
 412  * @ehdr:       Pointer to existing struct which will be populated.
 413  * @elf_info:   Pointer to existing struct which will be populated.
 414  *
 415  * This function allows reading ELF files with different byte order than
 416  * the kernel, byte-swapping the fields as needed.
 417  *
 418  * Return:
 419  * On success returns 0, and the caller should call elf_free_info(elf_info) to
 420  * free the memory allocated for the section and program headers.
 421  */
 422 int elf_read_from_buffer(const char *buf, size_t len, struct elfhdr *ehdr,
 423                          struct elf_info *elf_info)
 424 {
 425         int ret;
 426
 427         ret = elf_read_ehdr(buf, len, ehdr);
 428         if (ret)
 429                 return ret;
 430
 431         elf_info->buffer = buf;
 432         elf_info->ehdr = ehdr;
 433         if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
 434                 ret = elf_read_phdrs(buf, len, elf_info);
 435                 if (ret)
 436                         return ret;
 437         }
 438         if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
 439                 ret = elf_read_shdrs(buf, len, elf_info);
 440                 if (ret) {
 441                         kfree(elf_info->proghdrs);
 442                         return ret;
 443                 }
 444         }
 445
 446         return 0;
 447 }
 448
 449 /**
 450  * elf_free_info - free memory allocated by elf_read_from_buffer
 451  */
 452 void elf_free_info(struct elf_info *elf_info)
 453 {
 454         kfree(elf_info->proghdrs);
 455         kfree(elf_info->sechdrs);
 456         memset(elf_info, 0, sizeof(*elf_info));
 457 }
 458 /**
 459  * build_elf_exec_info - read ELF executable and check that we can use it
 460  */
 461 static int build_elf_exec_info(const char *buf, size_t len, struct elfhdr *ehdr,
 462                                struct elf_info *elf_info)
 463 {
 464         int i;
 465         int ret;
 466
 467         ret = elf_read_from_buffer(buf, len, ehdr, elf_info);
 468         if (ret)
 469                 return ret;
 470
 471         /* Big endian vmlinux has type ET_DYN. */
 472         if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) {
 473                 pr_err("Not an ELF executable.\n");
 474                 goto error;
 475         } else if (!elf_info->proghdrs) {
 476                 pr_err("No ELF program header.\n");
 477                 goto error;
 478         }
 479
 480         for (i = 0; i < ehdr->e_phnum; i++) {
 481                 /*
 482                  * Kexec does not support loading interpreters.
 483                  * In addition this check keeps us from attempting
 484                  * to kexec ordinay executables.
 485                  */
 486                 if (elf_info->proghdrs[i].p_type == PT_INTERP) {
 487                         pr_err("Requires an ELF interpreter.\n");
 488                         goto error;
 489                 }
 490         }
 491
 492         return 0;
 493 error:
 494         elf_free_info(elf_info);
 495         return -ENOEXEC;
 496 }
 497
 498 static int elf64_probe(const char *buf, unsigned long len)
 499 {
 500         struct elfhdr ehdr;
 501         struct elf_info elf_info;
 502         int ret;
 503
 504         ret = build_elf_exec_info(buf, len, &ehdr, &elf_info);
 505         if (ret)
 506                 return ret;
 507
 508         elf_free_info(&elf_info);
 509
 510         return elf_check_arch(&ehdr) ? 0 : -ENOEXEC;
 511 }
 512
 513 /**
 514  * elf_exec_load - load ELF executable image
 515  * @lowest_load_addr:   On return, will be the address where the first PT_LOAD
 516  *                      section will be loaded in memory.
 517  *
 518  * Return:
 519  * 0 on success, negative value on failure.
 520  */
 521 static int elf_exec_load(struct kimage *image, struct elfhdr *ehdr,
 522                          struct elf_info *elf_info,
 523                          unsigned long *lowest_load_addr)
 524 {
 525         unsigned long base = 0, lowest_addr = UINT_MAX;
 526         int ret;
 527         size_t i;
 528         struct kexec_buf kbuf = { .image = image, .buf_max = ppc64_rma_size,
 529                                   .top_down = false };
 530
 531         /* Read in the PT_LOAD segments. */
 532         for (i = 0; i < ehdr->e_phnum; i++) {
 533                 unsigned long load_addr;
 534                 size_t size;
 535                 const struct elf_phdr *phdr;
 536
 537                 phdr = &elf_info->proghdrs[i];
 538                 if (phdr->p_type != PT_LOAD)
 539                         continue;
 540
 541                 size = phdr->p_filesz;
 542                 if (size > phdr->p_memsz)
 543                         size = phdr->p_memsz;
 544
 545                 kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
 546                 kbuf.bufsz = size;
 547                 kbuf.memsz = phdr->p_memsz;
 548                 kbuf.buf_align = phdr->p_align;
 549                 kbuf.buf_min = phdr->p_paddr + base;
 550                 ret = kexec_add_buffer(&kbuf);
 551                 if (ret)
 552                         goto out;
 553                 load_addr = kbuf.mem;
 554
 555                 if (load_addr < lowest_addr)
 556                         lowest_addr = load_addr;
 557         }
 558
 559         /* Update entry point to reflect new load address. */
 560         ehdr->e_entry += base;
 561
 562         *lowest_load_addr = lowest_addr;
 563         ret = 0;
 564  out:
 565         return ret;
 566 }
 567
 568 static void *elf64_load(struct kimage *image, char *kernel_buf,
 569                         unsigned long kernel_len, char *initrd,
 570                         unsigned long initrd_len, char *cmdline,
 571                         unsigned long cmdline_len)
 572 {
 573         int ret;
 574         unsigned int fdt_size;
 575         unsigned long kernel_load_addr;
 576         unsigned long initrd_load_addr = 0, fdt_load_addr;
 577         void *fdt;
 578         const void *slave_code;
 579         struct elfhdr ehdr;
 580         struct elf_info elf_info;
 581         struct kexec_buf kbuf = { .image = image, .buf_min = 0,
 582                                   .buf_max = ppc64_rma_size };
 583         struct kexec_buf pbuf = { .image = image, .buf_min = 0,
 584                                   .buf_max = ppc64_rma_size, .top_down = true };
 585
 586         ret = build_elf_exec_info(kernel_buf, kernel_len, &ehdr, &elf_info);
 587         if (ret)
 588                 goto out;
 589
 590         ret = elf_exec_load(image, &ehdr, &elf_info, &kernel_load_addr);
 591         if (ret)
 592                 goto out;
 593
 594         pr_debug("Loaded the kernel at 0x%lx\n", kernel_load_addr);
 595
 596         ret = kexec_load_purgatory(image, &pbuf);
 597         if (ret) {
 598                 pr_err("Loading purgatory failed.\n");
 599                 goto out;
 600         }
 601
 602         pr_debug("Loaded purgatory at 0x%lx\n", pbuf.mem);
 603
 604         if (initrd != NULL) {
 605                 kbuf.buffer = initrd;
 606                 kbuf.bufsz = kbuf.memsz = initrd_len;
 607                 kbuf.buf_align = PAGE_SIZE;
 608                 kbuf.top_down = false;
 609                 ret = kexec_add_buffer(&kbuf);
 610                 if (ret)
 611                         goto out;
 612                 initrd_load_addr = kbuf.mem;
 613
 614                 pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr);
 615         }
 616
 617         fdt_size = fdt_totalsize(initial_boot_params) * 2;
 618         fdt = kmalloc(fdt_size, GFP_KERNEL);
 619         if (!fdt) {
 620                 pr_err("Not enough memory for the device tree.\n");
 621                 ret = -ENOMEM;
 622                 goto out;
 623         }
 624         ret = fdt_open_into(initial_boot_params, fdt, fdt_size);
 625         if (ret < 0) {
 626                 pr_err("Error setting up the new device tree.\n");
 627                 ret = -EINVAL;
 628                 goto out;
 629         }
 630
 631         ret = setup_new_fdt(image, fdt, initrd_load_addr, initrd_len, cmdline);
 632         if (ret)
 633                 goto out;
 634
 635         fdt_pack(fdt);
 636
 637         kbuf.buffer = fdt;
 638         kbuf.bufsz = kbuf.memsz = fdt_size;
 639         kbuf.buf_align = PAGE_SIZE;
 640         kbuf.top_down = true;
 641         ret = kexec_add_buffer(&kbuf);
 642         if (ret)
 643                 goto out;
 644         fdt_load_addr = kbuf.mem;
 645
 646         pr_debug("Loaded device tree at 0x%lx\n", fdt_load_addr);
 647
 648         slave_code = elf_info.buffer + elf_info.proghdrs[0].p_offset;
 649         ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
 650                               fdt_load_addr);
 651         if (ret)
 652                 pr_err("Error setting up the purgatory.\n");
 653
 654 out:
 655         elf_free_info(&elf_info);
 656
 657         /* Make kimage_file_post_load_cleanup free the fdt buffer for us. */
 658         return ret ? ERR_PTR(ret) : fdt;
 659 }
 660
 661 const struct kexec_file_ops kexec_elf64_ops = {
 662         .probe = elf64_probe,
 663         .load = elf64_load,
 664 };