arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/e820/api.h>
  51 #include <asm/time.h>
  52 #include <asm/set_memory.h>
  53 #include <asm/tlbflush.h>
  54 #include <asm/x86_init.h>
  55 #include <asm/uv/uv.h>
  56
  57 static struct efi efi_phys __initdata;
  58 static efi_system_table_t efi_systab __initdata;
  59
  60 static efi_config_table_type_t arch_tables[] __initdata = {
  61 #ifdef CONFIG_X86_UV
  62         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  63 #endif
  64         {NULL_GUID, NULL, NULL},
  65 };
  66
  67 u64 efi_setup;          /* efi setup_data physical address */
  68
  69 static int add_efi_memmap __initdata;
  70 static int __init setup_add_efi_memmap(char *arg)
  71 {
  72         add_efi_memmap = 1;
  73         return 0;
  74 }
  75 early_param("add_efi_memmap", setup_add_efi_memmap);
  76
  77 static efi_status_t __init phys_efi_set_virtual_address_map(
  78         unsigned long memory_map_size,
  79         unsigned long descriptor_size,
  80         u32 descriptor_version,
  81         efi_memory_desc_t *virtual_map)
  82 {
  83         efi_status_t status;
  84         unsigned long flags;
  85         pgd_t *save_pgd;
  86
  87         save_pgd = efi_call_phys_prolog();
  88
  89         /* Disable interrupts around EFI calls: */
  90         local_irq_save(flags);
  91         status = efi_call_phys(efi_phys.set_virtual_address_map,
  92                                memory_map_size, descriptor_size,
  93                                descriptor_version, virtual_map);
  94         local_irq_restore(flags);
  95
  96         efi_call_phys_epilog(save_pgd);
  97
  98         return status;
  99 }
 100
 101 void __init efi_find_mirror(void)
 102 {
 103         efi_memory_desc_t *md;
 104         u64 mirror_size = 0, total_size = 0;
 105
 106         for_each_efi_memory_desc(md) {
 107                 unsigned long long start = md->phys_addr;
 108                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 109
 110                 total_size += size;
 111                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 112                         memblock_mark_mirror(start, size);
 113                         mirror_size += size;
 114                 }
 115         }
 116         if (mirror_size)
 117                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 118                         mirror_size>>20, total_size>>20);
 119 }
 120
 121 /*
 122  * Tell the kernel about the EFI memory map.  This might include
 123  * more than the max 128 entries that can fit in the e820 legacy
 124  * (zeropage) memory map.
 125  */
 126
 127 static void __init do_add_efi_memmap(void)
 128 {
 129         efi_memory_desc_t *md;
 130
 131         for_each_efi_memory_desc(md) {
 132                 unsigned long long start = md->phys_addr;
 133                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 134                 int e820_type;
 135
 136                 switch (md->type) {
 137                 case EFI_LOADER_CODE:
 138                 case EFI_LOADER_DATA:
 139                 case EFI_BOOT_SERVICES_CODE:
 140                 case EFI_BOOT_SERVICES_DATA:
 141                 case EFI_CONVENTIONAL_MEMORY:
 142                         if (md->attribute & EFI_MEMORY_WB)
 143                                 e820_type = E820_TYPE_RAM;
 144                         else
 145                                 e820_type = E820_TYPE_RESERVED;
 146                         break;
 147                 case EFI_ACPI_RECLAIM_MEMORY:
 148                         e820_type = E820_TYPE_ACPI;
 149                         break;
 150                 case EFI_ACPI_MEMORY_NVS:
 151                         e820_type = E820_TYPE_NVS;
 152                         break;
 153                 case EFI_UNUSABLE_MEMORY:
 154                         e820_type = E820_TYPE_UNUSABLE;
 155                         break;
 156                 case EFI_PERSISTENT_MEMORY:
 157                         e820_type = E820_TYPE_PMEM;
 158                         break;
 159                 default:
 160                         /*
 161                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 162                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 163                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 164                          */
 165                         e820_type = E820_TYPE_RESERVED;
 166                         break;
 167                 }
 168                 e820__range_add(start, size, e820_type);
 169         }
 170         e820__update_table(e820_table);
 171 }
 172
 173 int __init efi_memblock_x86_reserve_range(void)
 174 {
 175         struct efi_info *e = &boot_params.efi_info;
 176         struct efi_memory_map_data data;
 177         phys_addr_t pmap;
 178         int rv;
 179
 180         if (efi_enabled(EFI_PARAVIRT))
 181                 return 0;
 182
 183 #ifdef CONFIG_X86_32
 184         /* Can't handle data above 4GB at this time */
 185         if (e->efi_memmap_hi) {
 186                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 187                 return -EINVAL;
 188         }
 189         pmap =  e->efi_memmap;
 190 #else
 191         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 192 #endif
 193         data.phys_map           = pmap;
 194         data.size               = e->efi_memmap_size;
 195         data.desc_size          = e->efi_memdesc_size;
 196         data.desc_version       = e->efi_memdesc_version;
 197
 198         rv = efi_memmap_init_early(&data);
 199         if (rv)
 200                 return rv;
 201
 202         if (add_efi_memmap)
 203                 do_add_efi_memmap();
 204
 205         WARN(efi.memmap.desc_version != 1,
 206              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 207              efi.memmap.desc_version);
 208
 209         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 210
 211         return 0;
 212 }
 213
 214 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
 215 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
 216 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
 217
 218 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
 219 {
 220         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
 221         u64 end_hi = 0;
 222         char buf[64];
 223
 224         if (md->num_pages == 0) {
 225                 end = 0;
 226         } else if (md->num_pages > EFI_PAGES_MAX ||
 227                    EFI_PAGES_MAX - md->num_pages <
 228                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
 229                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
 230                         >> OVERFLOW_ADDR_SHIFT;
 231
 232                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
 233                         end_hi += 1;
 234         } else {
 235                 return true;
 236         }
 237
 238         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
 239
 240         if (end_hi) {
 241                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
 242                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 243                         md->phys_addr, end_hi, end);
 244         } else {
 245                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
 246                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 247                         md->phys_addr, end);
 248         }
 249         return false;
 250 }
 251
 252 static void __init efi_clean_memmap(void)
 253 {
 254         efi_memory_desc_t *out = efi.memmap.map;
 255         const efi_memory_desc_t *in = out;
 256         const efi_memory_desc_t *end = efi.memmap.map_end;
 257         int i, n_removal;
 258
 259         for (i = n_removal = 0; in < end; i++) {
 260                 if (efi_memmap_entry_valid(in, i)) {
 261                         if (out != in)
 262                                 memcpy(out, in, efi.memmap.desc_size);
 263                         out = (void *)out + efi.memmap.desc_size;
 264                 } else {
 265                         n_removal++;
 266                 }
 267                 in = (void *)in + efi.memmap.desc_size;
 268         }
 269
 270         if (n_removal > 0) {
 271                 u64 size = efi.memmap.nr_map - n_removal;
 272
 273                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
 274                 efi_memmap_install(efi.memmap.phys_map, size);
 275         }
 276 }
 277
 278 void __init efi_print_memmap(void)
 279 {
 280         efi_memory_desc_t *md;
 281         int i = 0;
 282
 283         for_each_efi_memory_desc(md) {
 284                 char buf[64];
 285
 286                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 287                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 288                         md->phys_addr,
 289                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 290                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 291         }
 292 }
 293
 294 static int __init efi_systab_init(void *phys)
 295 {
 296         if (efi_enabled(EFI_64BIT)) {
 297                 efi_system_table_64_t *systab64;
 298                 struct efi_setup_data *data = NULL;
 299                 u64 tmp = 0;
 300
 301                 if (efi_setup) {
 302                         data = early_memremap(efi_setup, sizeof(*data));
 303                         if (!data)
 304                                 return -ENOMEM;
 305                 }
 306                 systab64 = early_memremap((unsigned long)phys,
 307                                          sizeof(*systab64));
 308                 if (systab64 == NULL) {
 309                         pr_err("Couldn't map the system table!\n");
 310                         if (data)
 311                                 early_memunmap(data, sizeof(*data));
 312                         return -ENOMEM;
 313                 }
 314
 315                 efi_systab.hdr = systab64->hdr;
 316                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 317                                               systab64->fw_vendor;
 318                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 319                 efi_systab.fw_revision = systab64->fw_revision;
 320                 efi_systab.con_in_handle = systab64->con_in_handle;
 321                 tmp |= systab64->con_in_handle;
 322                 efi_systab.con_in = systab64->con_in;
 323                 tmp |= systab64->con_in;
 324                 efi_systab.con_out_handle = systab64->con_out_handle;
 325                 tmp |= systab64->con_out_handle;
 326                 efi_systab.con_out = systab64->con_out;
 327                 tmp |= systab64->con_out;
 328                 efi_systab.stderr_handle = systab64->stderr_handle;
 329                 tmp |= systab64->stderr_handle;
 330                 efi_systab.stderr = systab64->stderr;
 331                 tmp |= systab64->stderr;
 332                 efi_systab.runtime = data ?
 333                                      (void *)(unsigned long)data->runtime :
 334                                      (void *)(unsigned long)systab64->runtime;
 335                 tmp |= data ? data->runtime : systab64->runtime;
 336                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 337                 tmp |= systab64->boottime;
 338                 efi_systab.nr_tables = systab64->nr_tables;
 339                 efi_systab.tables = data ? (unsigned long)data->tables :
 340                                            systab64->tables;
 341                 tmp |= data ? data->tables : systab64->tables;
 342
 343                 early_memunmap(systab64, sizeof(*systab64));
 344                 if (data)
 345                         early_memunmap(data, sizeof(*data));
 346 #ifdef CONFIG_X86_32
 347                 if (tmp >> 32) {
 348                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 349                         return -EINVAL;
 350                 }
 351 #endif
 352         } else {
 353                 efi_system_table_32_t *systab32;
 354
 355                 systab32 = early_memremap((unsigned long)phys,
 356                                          sizeof(*systab32));
 357                 if (systab32 == NULL) {
 358                         pr_err("Couldn't map the system table!\n");
 359                         return -ENOMEM;
 360                 }
 361
 362                 efi_systab.hdr = systab32->hdr;
 363                 efi_systab.fw_vendor = systab32->fw_vendor;
 364                 efi_systab.fw_revision = systab32->fw_revision;
 365                 efi_systab.con_in_handle = systab32->con_in_handle;
 366                 efi_systab.con_in = systab32->con_in;
 367                 efi_systab.con_out_handle = systab32->con_out_handle;
 368                 efi_systab.con_out = systab32->con_out;
 369                 efi_systab.stderr_handle = systab32->stderr_handle;
 370                 efi_systab.stderr = systab32->stderr;
 371                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 372                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 373                 efi_systab.nr_tables = systab32->nr_tables;
 374                 efi_systab.tables = systab32->tables;
 375
 376                 early_memunmap(systab32, sizeof(*systab32));
 377         }
 378
 379         efi.systab = &efi_systab;
 380
 381         /*
 382          * Verify the EFI Table
 383          */
 384         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 385                 pr_err("System table signature incorrect!\n");
 386                 return -EINVAL;
 387         }
 388         if ((efi.systab->hdr.revision >> 16) == 0)
 389                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 390                        efi.systab->hdr.revision >> 16,
 391                        efi.systab->hdr.revision & 0xffff);
 392
 393         return 0;
 394 }
 395
 396 static int __init efi_runtime_init32(void)
 397 {
 398         efi_runtime_services_32_t *runtime;
 399
 400         runtime = early_memremap((unsigned long)efi.systab->runtime,
 401                         sizeof(efi_runtime_services_32_t));
 402         if (!runtime) {
 403                 pr_err("Could not map the runtime service table!\n");
 404                 return -ENOMEM;
 405         }
 406
 407         /*
 408          * We will only need *early* access to the SetVirtualAddressMap
 409          * EFI runtime service. All other runtime services will be called
 410          * via the virtual mapping.
 411          */
 412         efi_phys.set_virtual_address_map =
 413                         (efi_set_virtual_address_map_t *)
 414                         (unsigned long)runtime->set_virtual_address_map;
 415         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 416
 417         return 0;
 418 }
 419
 420 static int __init efi_runtime_init64(void)
 421 {
 422         efi_runtime_services_64_t *runtime;
 423
 424         runtime = early_memremap((unsigned long)efi.systab->runtime,
 425                         sizeof(efi_runtime_services_64_t));
 426         if (!runtime) {
 427                 pr_err("Could not map the runtime service table!\n");
 428                 return -ENOMEM;
 429         }
 430
 431         /*
 432          * We will only need *early* access to the SetVirtualAddressMap
 433          * EFI runtime service. All other runtime services will be called
 434          * via the virtual mapping.
 435          */
 436         efi_phys.set_virtual_address_map =
 437                         (efi_set_virtual_address_map_t *)
 438                         (unsigned long)runtime->set_virtual_address_map;
 439         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 440
 441         return 0;
 442 }
 443
 444 static int __init efi_runtime_init(void)
 445 {
 446         int rv;
 447
 448         /*
 449          * Check out the runtime services table. We need to map
 450          * the runtime services table so that we can grab the physical
 451          * address of several of the EFI runtime functions, needed to
 452          * set the firmware into virtual mode.
 453          *
 454          * When EFI_PARAVIRT is in force then we could not map runtime
 455          * service memory region because we do not have direct access to it.
 456          * However, runtime services are available through proxy functions
 457          * (e.g. in case of Xen dom0 EFI implementation they call special
 458          * hypercall which executes relevant EFI functions) and that is why
 459          * they are always enabled.
 460          */
 461
 462         if (!efi_enabled(EFI_PARAVIRT)) {
 463                 if (efi_enabled(EFI_64BIT))
 464                         rv = efi_runtime_init64();
 465                 else
 466                         rv = efi_runtime_init32();
 467
 468                 if (rv)
 469                         return rv;
 470         }
 471
 472         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 473
 474         return 0;
 475 }
 476
 477 void __init efi_init(void)
 478 {
 479         efi_char16_t *c16;
 480         char vendor[100] = "unknown";
 481         int i = 0;
 482         void *tmp;
 483
 484 #ifdef CONFIG_X86_32
 485         if (boot_params.efi_info.efi_systab_hi ||
 486             boot_params.efi_info.efi_memmap_hi) {
 487                 pr_info("Table located above 4GB, disabling EFI.\n");
 488                 return;
 489         }
 490         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 491 #else
 492         efi_phys.systab = (efi_system_table_t *)
 493                           (boot_params.efi_info.efi_systab |
 494                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 495 #endif
 496
 497         if (efi_systab_init(efi_phys.systab))
 498                 return;
 499
 500         efi.config_table = (unsigned long)efi.systab->tables;
 501         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 502         efi.runtime      = (unsigned long)efi.systab->runtime;
 503
 504         /*
 505          * Show what we know for posterity
 506          */
 507         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 508         if (c16) {
 509                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 510                         vendor[i] = *c16++;
 511                 vendor[i] = '\0';
 512         } else
 513                 pr_err("Could not map the firmware vendor!\n");
 514         early_memunmap(tmp, 2);
 515
 516         pr_info("EFI v%u.%.02u by %s\n",
 517                 efi.systab->hdr.revision >> 16,
 518                 efi.systab->hdr.revision & 0xffff, vendor);
 519
 520         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 521                 return;
 522
 523         if (efi_config_init(arch_tables))
 524                 return;
 525
 526         /*
 527          * Note: We currently don't support runtime services on an EFI
 528          * that doesn't match the kernel 32/64-bit mode.
 529          */
 530
 531         if (!efi_runtime_supported())
 532                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 533         else {
 534                 if (efi_runtime_disabled() || efi_runtime_init()) {
 535                         efi_memmap_unmap();
 536                         return;
 537                 }
 538         }
 539
 540         efi_clean_memmap();
 541
 542         if (efi_enabled(EFI_DBG))
 543                 efi_print_memmap();
 544 }
 545
 546 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 547 {
 548         u64 addr, npages;
 549
 550         addr = md->virt_addr;
 551         npages = md->num_pages;
 552
 553         memrange_efi_to_native(&addr, &npages);
 554
 555         if (executable)
 556                 set_memory_x(addr, npages);
 557         else
 558                 set_memory_nx(addr, npages);
 559 }
 560
 561 void __init runtime_code_page_mkexec(void)
 562 {
 563         efi_memory_desc_t *md;
 564
 565         /* Make EFI runtime service code area executable */
 566         for_each_efi_memory_desc(md) {
 567                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 568                         continue;
 569
 570                 efi_set_executable(md, true);
 571         }
 572 }
 573
 574 void __init efi_memory_uc(u64 addr, unsigned long size)
 575 {
 576         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 577         u64 npages;
 578
 579         npages = round_up(size, page_shift) / page_shift;
 580         memrange_efi_to_native(&addr, &npages);
 581         set_memory_uc(addr, npages);
 582 }
 583
 584 void __init old_map_region(efi_memory_desc_t *md)
 585 {
 586         u64 start_pfn, end_pfn, end;
 587         unsigned long size;
 588         void *va;
 589
 590         start_pfn = PFN_DOWN(md->phys_addr);
 591         size      = md->num_pages << PAGE_SHIFT;
 592         end       = md->phys_addr + size;
 593         end_pfn   = PFN_UP(end);
 594
 595         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 596                 va = __va(md->phys_addr);
 597
 598                 if (!(md->attribute & EFI_MEMORY_WB))
 599                         efi_memory_uc((u64)(unsigned long)va, size);
 600         } else
 601                 va = efi_ioremap(md->phys_addr, size,
 602                                  md->type, md->attribute);
 603
 604         md->virt_addr = (u64) (unsigned long) va;
 605         if (!va)
 606                 pr_err("ioremap of 0x%llX failed!\n",
 607                        (unsigned long long)md->phys_addr);
 608 }
 609
 610 /* Merge contiguous regions of the same type and attribute */
 611 static void __init efi_merge_regions(void)
 612 {
 613         efi_memory_desc_t *md, *prev_md = NULL;
 614
 615         for_each_efi_memory_desc(md) {
 616                 u64 prev_size;
 617
 618                 if (!prev_md) {
 619                         prev_md = md;
 620                         continue;
 621                 }
 622
 623                 if (prev_md->type != md->type ||
 624                     prev_md->attribute != md->attribute) {
 625                         prev_md = md;
 626                         continue;
 627                 }
 628
 629                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 630
 631                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 632                         prev_md->num_pages += md->num_pages;
 633                         md->type = EFI_RESERVED_TYPE;
 634                         md->attribute = 0;
 635                         continue;
 636                 }
 637                 prev_md = md;
 638         }
 639 }
 640
 641 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 642 {
 643         unsigned long size;
 644         u64 end, systab;
 645
 646         size = md->num_pages << EFI_PAGE_SHIFT;
 647         end = md->phys_addr + size;
 648         systab = (u64)(unsigned long)efi_phys.systab;
 649         if (md->phys_addr <= systab && systab < end) {
 650                 systab += md->virt_addr - md->phys_addr;
 651                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 652         }
 653 }
 654
 655 static void *realloc_pages(void *old_memmap, int old_shift)
 656 {
 657         void *ret;
 658
 659         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 660         if (!ret)
 661                 goto out;
 662
 663         /*
 664          * A first-time allocation doesn't have anything to copy.
 665          */
 666         if (!old_memmap)
 667                 return ret;
 668
 669         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 670
 671 out:
 672         free_pages((unsigned long)old_memmap, old_shift);
 673         return ret;
 674 }
 675
 676 /*
 677  * Iterate the EFI memory map in reverse order because the regions
 678  * will be mapped top-down. The end result is the same as if we had
 679  * mapped things forward, but doesn't require us to change the
 680  * existing implementation of efi_map_region().
 681  */
 682 static inline void *efi_map_next_entry_reverse(void *entry)
 683 {
 684         /* Initial call */
 685         if (!entry)
 686                 return efi.memmap.map_end - efi.memmap.desc_size;
 687
 688         entry -= efi.memmap.desc_size;
 689         if (entry < efi.memmap.map)
 690                 return NULL;
 691
 692         return entry;
 693 }
 694
 695 /*
 696  * efi_map_next_entry - Return the next EFI memory map descriptor
 697  * @entry: Previous EFI memory map descriptor
 698  *
 699  * This is a helper function to iterate over the EFI memory map, which
 700  * we do in different orders depending on the current configuration.
 701  *
 702  * To begin traversing the memory map @entry must be %NULL.
 703  *
 704  * Returns %NULL when we reach the end of the memory map.
 705  */
 706 static void *efi_map_next_entry(void *entry)
 707 {
 708         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 709                 /*
 710                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 711                  * config table feature requires us to map all entries
 712                  * in the same order as they appear in the EFI memory
 713                  * map. That is to say, entry N must have a lower
 714                  * virtual address than entry N+1. This is because the
 715                  * firmware toolchain leaves relative references in
 716                  * the code/data sections, which are split and become
 717                  * separate EFI memory regions. Mapping things
 718                  * out-of-order leads to the firmware accessing
 719                  * unmapped addresses.
 720                  *
 721                  * Since we need to map things this way whether or not
 722                  * the kernel actually makes use of
 723                  * EFI_PROPERTIES_TABLE, let's just switch to this
 724                  * scheme by default for 64-bit.
 725                  */
 726                 return efi_map_next_entry_reverse(entry);
 727         }
 728
 729         /* Initial call */
 730         if (!entry)
 731                 return efi.memmap.map;
 732
 733         entry += efi.memmap.desc_size;
 734         if (entry >= efi.memmap.map_end)
 735                 return NULL;
 736
 737         return entry;
 738 }
 739
 740 static bool should_map_region(efi_memory_desc_t *md)
 741 {
 742         /*
 743          * Runtime regions always require runtime mappings (obviously).
 744          */
 745         if (md->attribute & EFI_MEMORY_RUNTIME)
 746                 return true;
 747
 748         /*
 749          * 32-bit EFI doesn't suffer from the bug that requires us to
 750          * reserve boot services regions, and mixed mode support
 751          * doesn't exist for 32-bit kernels.
 752          */
 753         if (IS_ENABLED(CONFIG_X86_32))
 754                 return false;
 755
 756         /*
 757          * Map all of RAM so that we can access arguments in the 1:1
 758          * mapping when making EFI runtime calls.
 759          */
 760         if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
 761                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
 762                     md->type == EFI_LOADER_DATA ||
 763                     md->type == EFI_LOADER_CODE)
 764                         return true;
 765         }
 766
 767         /*
 768          * Map boot services regions as a workaround for buggy
 769          * firmware that accesses them even when they shouldn't.
 770          *
 771          * See efi_{reserve,free}_boot_services().
 772          */
 773         if (md->type == EFI_BOOT_SERVICES_CODE ||
 774             md->type == EFI_BOOT_SERVICES_DATA)
 775                 return true;
 776
 777         return false;
 778 }
 779
 780 /*
 781  * Map the efi memory ranges of the runtime services and update new_mmap with
 782  * virtual addresses.
 783  */
 784 static void * __init efi_map_regions(int *count, int *pg_shift)
 785 {
 786         void *p, *new_memmap = NULL;
 787         unsigned long left = 0;
 788         unsigned long desc_size;
 789         efi_memory_desc_t *md;
 790
 791         desc_size = efi.memmap.desc_size;
 792
 793         p = NULL;
 794         while ((p = efi_map_next_entry(p))) {
 795                 md = p;
 796
 797                 if (!should_map_region(md))
 798                         continue;
 799
 800                 efi_map_region(md);
 801                 get_systab_virt_addr(md);
 802
 803                 if (left < desc_size) {
 804                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 805                         if (!new_memmap)
 806                                 return NULL;
 807
 808                         left += PAGE_SIZE << *pg_shift;
 809                         (*pg_shift)++;
 810                 }
 811
 812                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 813
 814                 left -= desc_size;
 815                 (*count)++;
 816         }
 817
 818         return new_memmap;
 819 }
 820
 821 static void __init kexec_enter_virtual_mode(void)
 822 {
 823 #ifdef CONFIG_KEXEC_CORE
 824         efi_memory_desc_t *md;
 825         unsigned int num_pages;
 826
 827         efi.systab = NULL;
 828
 829         /*
 830          * We don't do virtual mode, since we don't do runtime services, on
 831          * non-native EFI. With efi=old_map, we don't do runtime services in
 832          * kexec kernel because in the initial boot something else might
 833          * have been mapped at these virtual addresses.
 834          */
 835         if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
 836                 efi_memmap_unmap();
 837                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 838                 return;
 839         }
 840
 841         if (efi_alloc_page_tables()) {
 842                 pr_err("Failed to allocate EFI page tables\n");
 843                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 844                 return;
 845         }
 846
 847         /*
 848         * Map efi regions which were passed via setup_data. The virt_addr is a
 849         * fixed addr which was used in first kernel of a kexec boot.
 850         */
 851         for_each_efi_memory_desc(md) {
 852                 efi_map_region_fixed(md); /* FIXME: add error handling */
 853                 get_systab_virt_addr(md);
 854         }
 855
 856         /*
 857          * Unregister the early EFI memmap from efi_init() and install
 858          * the new EFI memory map.
 859          */
 860         efi_memmap_unmap();
 861
 862         if (efi_memmap_init_late(efi.memmap.phys_map,
 863                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
 864                 pr_err("Failed to remap late EFI memory map\n");
 865                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 866                 return;
 867         }
 868
 869         BUG_ON(!efi.systab);
 870
 871         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 872         num_pages >>= PAGE_SHIFT;
 873
 874         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 875                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 876                 return;
 877         }
 878
 879         efi_sync_low_kernel_mappings();
 880
 881         /*
 882          * Now that EFI is in virtual mode, update the function
 883          * pointers in the runtime service table to the new virtual addresses.
 884          *
 885          * Call EFI services through wrapper functions.
 886          */
 887         efi.runtime_version = efi_systab.hdr.revision;
 888
 889         efi_native_runtime_setup();
 890
 891         efi.set_virtual_address_map = NULL;
 892
 893         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 894                 runtime_code_page_mkexec();
 895
 896         /* clean DUMMY object */
 897         efi_delete_dummy_variable();
 898 #endif
 899 }
 900
 901 /*
 902  * This function will switch the EFI runtime services to virtual mode.
 903  * Essentially, we look through the EFI memmap and map every region that
 904  * has the runtime attribute bit set in its memory descriptor into the
 905  * efi_pgd page table.
 906  *
 907  * The old method which used to update that memory descriptor with the
 908  * virtual address obtained from ioremap() is still supported when the
 909  * kernel is booted with efi=old_map on its command line. Same old
 910  * method enabled the runtime services to be called without having to
 911  * thunk back into physical mode for every invocation.
 912  *
 913  * The new method does a pagetable switch in a preemption-safe manner
 914  * so that we're in a different address space when calling a runtime
 915  * function. For function arguments passing we do copy the PUDs of the
 916  * kernel page table into efi_pgd prior to each call.
 917  *
 918  * Specially for kexec boot, efi runtime maps in previous kernel should
 919  * be passed in via setup_data. In that case runtime ranges will be mapped
 920  * to the same virtual addresses as the first kernel, see
 921  * kexec_enter_virtual_mode().
 922  */
 923 static void __init __efi_enter_virtual_mode(void)
 924 {
 925         int count = 0, pg_shift = 0;
 926         void *new_memmap = NULL;
 927         efi_status_t status;
 928         unsigned long pa;
 929
 930         efi.systab = NULL;
 931
 932         if (efi_alloc_page_tables()) {
 933                 pr_err("Failed to allocate EFI page tables\n");
 934                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 935                 return;
 936         }
 937
 938         efi_merge_regions();
 939         new_memmap = efi_map_regions(&count, &pg_shift);
 940         if (!new_memmap) {
 941                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 942                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 943                 return;
 944         }
 945
 946         pa = __pa(new_memmap);
 947
 948         /*
 949          * Unregister the early EFI memmap from efi_init() and install
 950          * the new EFI memory map that we are about to pass to the
 951          * firmware via SetVirtualAddressMap().
 952          */
 953         efi_memmap_unmap();
 954
 955         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
 956                 pr_err("Failed to remap late EFI memory map\n");
 957                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 958                 return;
 959         }
 960
 961         if (efi_enabled(EFI_DBG)) {
 962                 pr_info("EFI runtime memory map:\n");
 963                 efi_print_memmap();
 964         }
 965
 966         BUG_ON(!efi.systab);
 967
 968         if (efi_setup_page_tables(pa, 1 << pg_shift)) {
 969                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 970                 return;
 971         }
 972
 973         efi_sync_low_kernel_mappings();
 974
 975         if (efi_is_native()) {
 976                 status = phys_efi_set_virtual_address_map(
 977                                 efi.memmap.desc_size * count,
 978                                 efi.memmap.desc_size,
 979                                 efi.memmap.desc_version,
 980                                 (efi_memory_desc_t *)pa);
 981         } else {
 982                 status = efi_thunk_set_virtual_address_map(
 983                                 efi_phys.set_virtual_address_map,
 984                                 efi.memmap.desc_size * count,
 985                                 efi.memmap.desc_size,
 986                                 efi.memmap.desc_version,
 987                                 (efi_memory_desc_t *)pa);
 988         }
 989
 990         if (status != EFI_SUCCESS) {
 991                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 992                          status);
 993                 panic("EFI call to SetVirtualAddressMap() failed!");
 994         }
 995
 996         /*
 997          * Now that EFI is in virtual mode, update the function
 998          * pointers in the runtime service table to the new virtual addresses.
 999          *
1000          * Call EFI services through wrapper functions.
1001          */
1002         efi.runtime_version = efi_systab.hdr.revision;
1003
1004         if (efi_is_native())
1005                 efi_native_runtime_setup();
1006         else
1007                 efi_thunk_runtime_setup();
1008
1009         efi.set_virtual_address_map = NULL;
1010
1011         /*
1012          * Apply more restrictive page table mapping attributes now that
1013          * SVAM() has been called and the firmware has performed all
1014          * necessary relocation fixups for the new virtual addresses.
1015          */
1016         efi_runtime_update_mappings();
1017         efi_dump_pagetable();
1018
1019         /* clean DUMMY object */
1020         efi_delete_dummy_variable();
1021 }
1022
1023 void __init efi_enter_virtual_mode(void)
1024 {
1025         if (efi_enabled(EFI_PARAVIRT))
1026                 return;
1027
1028         if (efi_setup)
1029                 kexec_enter_virtual_mode();
1030         else
1031                 __efi_enter_virtual_mode();
1032 }
1033
1034 /*
1035  * Convenience functions to obtain memory types and attributes
1036  */
1037 u32 efi_mem_type(unsigned long phys_addr)
1038 {
1039         efi_memory_desc_t *md;
1040
1041         if (!efi_enabled(EFI_MEMMAP))
1042                 return 0;
1043
1044         for_each_efi_memory_desc(md) {
1045                 if ((md->phys_addr <= phys_addr) &&
1046                     (phys_addr < (md->phys_addr +
1047                                   (md->num_pages << EFI_PAGE_SHIFT))))
1048                         return md->type;
1049         }
1050         return 0;
1051 }
1052
1053 static int __init arch_parse_efi_cmdline(char *str)
1054 {
1055         if (!str) {
1056                 pr_warn("need at least one option\n");
1057                 return -EINVAL;
1058         }
1059
1060         if (parse_option_str(str, "old_map"))
1061                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1062
1063         return 0;
1064 }
1065 early_param("efi", arch_parse_efi_cmdline);