arch/x86/platform/efi/efi_64.c

   1 /*
   2  * x86_64 specific EFI support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 2005-2008 Intel Co.
   6  *      Fenghua Yu <fenghua.yu@intel.com>
   7  *      Bibo Mao <bibo.mao@intel.com>
   8  *      Chandramouli Narayanan <mouli@linux.intel.com>
   9  *      Huang Ying <ying.huang@intel.com>
  10  *
  11  * Code to convert EFI to E820 map has been implemented in elilo bootloader
  12  * based on a EFI patch by Edgar Hucek. Based on the E820 map, the page table
  13  * is setup appropriately for EFI runtime code.
  14  * - mouli 06/14/2007.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/init.h>
  20 #include <linux/mm.h>
  21 #include <linux/types.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/bootmem.h>
  24 #include <linux/ioport.h>
  25 #include <linux/module.h>
  26 #include <linux/efi.h>
  27 #include <linux/uaccess.h>
  28 #include <linux/io.h>
  29 #include <linux/reboot.h>
  30 #include <linux/slab.h>
  31
  32 #include <asm/setup.h>
  33 #include <asm/page.h>
  34 #include <asm/e820.h>
  35 #include <asm/pgtable.h>
  36 #include <asm/tlbflush.h>
  37 #include <asm/proto.h>
  38 #include <asm/efi.h>
  39 #include <asm/cacheflush.h>
  40 #include <asm/fixmap.h>
  41 #include <asm/realmode.h>
  42 #include <asm/time.h>
  43
  44 static pgd_t *save_pgd __initdata;
  45 static unsigned long efi_flags __initdata;
  46
  47 /*
  48  * We allocate runtime services regions bottom-up, starting from -4G, i.e.
  49  * 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G.
  50  */
  51 static u64 efi_va       = -4 * (1UL << 30);
  52 #define EFI_VA_END      (-68 * (1UL << 30))
  53
  54 /*
  55  * Scratch space used for switching the pagetable in the EFI stub
  56  */
  57 struct efi_scratch {
  58         u64 r15;
  59         u64 prev_cr3;
  60         pgd_t *efi_pgt;
  61         bool use_pgd;
  62         u64 phys_stack;
  63 } __packed;
  64
  65 static void __init early_code_mapping_set_exec(int executable)
  66 {
  67         efi_memory_desc_t *md;
  68         void *p;
  69
  70         if (!(__supported_pte_mask & _PAGE_NX))
  71                 return;
  72
  73         /* Make EFI service code area executable */
  74         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
  75                 md = p;
  76                 if (md->type == EFI_RUNTIME_SERVICES_CODE ||
  77                     md->type == EFI_BOOT_SERVICES_CODE)
  78                         efi_set_executable(md, executable);
  79         }
  80 }
  81
  82 void __init efi_call_phys_prolog(void)
  83 {
  84         unsigned long vaddress;
  85         int pgd;
  86         int n_pgds;
  87
  88         if (!efi_enabled(EFI_OLD_MEMMAP))
  89                 return;
  90
  91         early_code_mapping_set_exec(1);
  92         local_irq_save(efi_flags);
  93
  94         n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE);
  95         save_pgd = kmalloc(n_pgds * sizeof(pgd_t), GFP_KERNEL);
  96
  97         for (pgd = 0; pgd < n_pgds; pgd++) {
  98                 save_pgd[pgd] = *pgd_offset_k(pgd * PGDIR_SIZE);
  99                 vaddress = (unsigned long)__va(pgd * PGDIR_SIZE);
 100                 set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress));
 101         }
 102         __flush_tlb_all();
 103 }
 104
 105 void __init efi_call_phys_epilog(void)
 106 {
 107         /*
 108          * After the lock is released, the original page table is restored.
 109          */
 110         int pgd;
 111         int n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE);
 112
 113         if (!efi_enabled(EFI_OLD_MEMMAP))
 114                 return;
 115
 116         for (pgd = 0; pgd < n_pgds; pgd++)
 117                 set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), save_pgd[pgd]);
 118         kfree(save_pgd);
 119         __flush_tlb_all();
 120         local_irq_restore(efi_flags);
 121         early_code_mapping_set_exec(0);
 122 }
 123
 124 /*
 125  * Add low kernel mappings for passing arguments to EFI functions.
 126  */
 127 void efi_sync_low_kernel_mappings(void)
 128 {
 129         unsigned num_pgds;
 130         pgd_t *pgd = (pgd_t *)__va(real_mode_header->trampoline_pgd);
 131
 132         if (efi_enabled(EFI_OLD_MEMMAP))
 133                 return;
 134
 135         num_pgds = pgd_index(MODULES_END - 1) - pgd_index(PAGE_OFFSET);
 136
 137         memcpy(pgd + pgd_index(PAGE_OFFSET),
 138                 init_mm.pgd + pgd_index(PAGE_OFFSET),
 139                 sizeof(pgd_t) * num_pgds);
 140 }
 141
 142 int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
 143 {
 144         unsigned long text;
 145         struct page *page;
 146         unsigned npages;
 147         pgd_t *pgd;
 148
 149         if (efi_enabled(EFI_OLD_MEMMAP))
 150                 return 0;
 151
 152         efi_scratch.efi_pgt = (pgd_t *)(unsigned long)real_mode_header->trampoline_pgd;
 153         pgd = __va(efi_scratch.efi_pgt);
 154
 155         /*
 156          * It can happen that the physical address of new_memmap lands in memory
 157          * which is not mapped in the EFI page table. Therefore we need to go
 158          * and ident-map those pages containing the map before calling
 159          * phys_efi_set_virtual_address_map().
 160          */
 161         if (kernel_map_pages_in_pgd(pgd, pa_memmap, pa_memmap, num_pages, _PAGE_NX)) {
 162                 pr_err("Error ident-mapping new memmap (0x%lx)!\n", pa_memmap);
 163                 return 1;
 164         }
 165
 166         efi_scratch.use_pgd = true;
 167
 168         /*
 169          * When making calls to the firmware everything needs to be 1:1
 170          * mapped and addressable with 32-bit pointers. Map the kernel
 171          * text and allocate a new stack because we can't rely on the
 172          * stack pointer being < 4GB.
 173          */
 174         if (!IS_ENABLED(CONFIG_EFI_MIXED))
 175                 return 0;
 176
 177         page = alloc_page(GFP_KERNEL|__GFP_DMA32);
 178         if (!page)
 179                 panic("Unable to allocate EFI runtime stack < 4GB\n");
 180
 181         efi_scratch.phys_stack = virt_to_phys(page_address(page));
 182         efi_scratch.phys_stack += PAGE_SIZE; /* stack grows down */
 183
 184         npages = (_end - _text) >> PAGE_SHIFT;
 185         text = __pa(_text);
 186
 187         if (kernel_map_pages_in_pgd(pgd, text >> PAGE_SHIFT, text, npages, 0)) {
 188                 pr_err("Failed to map kernel text 1:1\n");
 189                 return 1;
 190         }
 191
 192         return 0;
 193 }
 194
 195 void __init efi_cleanup_page_tables(unsigned long pa_memmap, unsigned num_pages)
 196 {
 197         pgd_t *pgd = (pgd_t *)__va(real_mode_header->trampoline_pgd);
 198
 199         kernel_unmap_pages_in_pgd(pgd, pa_memmap, num_pages);
 200 }
 201
 202 static void __init __map_region(efi_memory_desc_t *md, u64 va)
 203 {
 204         pgd_t *pgd = (pgd_t *)__va(real_mode_header->trampoline_pgd);
 205         unsigned long pf = 0;
 206
 207         if (!(md->attribute & EFI_MEMORY_WB))
 208                 pf |= _PAGE_PCD;
 209
 210         if (kernel_map_pages_in_pgd(pgd, md->phys_addr, va, md->num_pages, pf))
 211                 pr_warn("Error mapping PA 0x%llx -> VA 0x%llx!\n",
 212                            md->phys_addr, va);
 213 }
 214
 215 void __init efi_map_region(efi_memory_desc_t *md)
 216 {
 217         unsigned long size = md->num_pages << PAGE_SHIFT;
 218         u64 pa = md->phys_addr;
 219
 220         if (efi_enabled(EFI_OLD_MEMMAP))
 221                 return old_map_region(md);
 222
 223         /*
 224          * Make sure the 1:1 mappings are present as a catch-all for b0rked
 225          * firmware which doesn't update all internal pointers after switching
 226          * to virtual mode and would otherwise crap on us.
 227          */
 228         __map_region(md, md->phys_addr);
 229
 230         /*
 231          * Enforce the 1:1 mapping as the default virtual address when
 232          * booting in EFI mixed mode, because even though we may be
 233          * running a 64-bit kernel, the firmware may only be 32-bit.
 234          */
 235         if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) {
 236                 md->virt_addr = md->phys_addr;
 237                 return;
 238         }
 239
 240         efi_va -= size;
 241
 242         /* Is PA 2M-aligned? */
 243         if (!(pa & (PMD_SIZE - 1))) {
 244                 efi_va &= PMD_MASK;
 245         } else {
 246                 u64 pa_offset = pa & (PMD_SIZE - 1);
 247                 u64 prev_va = efi_va;
 248
 249                 /* get us the same offset within this 2M page */
 250                 efi_va = (efi_va & PMD_MASK) + pa_offset;
 251
 252                 if (efi_va > prev_va)
 253                         efi_va -= PMD_SIZE;
 254         }
 255
 256         if (efi_va < EFI_VA_END) {
 257                 pr_warn(FW_WARN "VA address range overflow!\n");
 258                 return;
 259         }
 260
 261         /* Do the VA map */
 262         __map_region(md, efi_va);
 263         md->virt_addr = efi_va;
 264 }
 265
 266 /*
 267  * kexec kernel will use efi_map_region_fixed to map efi runtime memory ranges.
 268  * md->virt_addr is the original virtual address which had been mapped in kexec
 269  * 1st kernel.
 270  */
 271 void __init efi_map_region_fixed(efi_memory_desc_t *md)
 272 {
 273         __map_region(md, md->virt_addr);
 274 }
 275
 276 void __iomem *__init efi_ioremap(unsigned long phys_addr, unsigned long size,
 277                                  u32 type, u64 attribute)
 278 {
 279         unsigned long last_map_pfn;
 280
 281         if (type == EFI_MEMORY_MAPPED_IO)
 282                 return ioremap(phys_addr, size);
 283
 284         last_map_pfn = init_memory_mapping(phys_addr, phys_addr + size);
 285         if ((last_map_pfn << PAGE_SHIFT) < phys_addr + size) {
 286                 unsigned long top = last_map_pfn << PAGE_SHIFT;
 287                 efi_ioremap(top, size - (top - phys_addr), type, attribute);
 288         }
 289
 290         if (!(attribute & EFI_MEMORY_WB))
 291                 efi_memory_uc((u64)(unsigned long)__va(phys_addr), size);
 292
 293         return (void __iomem *)__va(phys_addr);
 294 }
 295
 296 void __init parse_efi_setup(u64 phys_addr, u32 data_len)
 297 {
 298         efi_setup = phys_addr + sizeof(struct setup_data);
 299 }
 300
 301 void __init efi_runtime_mkexec(void)
 302 {
 303         if (!efi_enabled(EFI_OLD_MEMMAP))
 304                 return;
 305
 306         if (__supported_pte_mask & _PAGE_NX)
 307                 runtime_code_page_mkexec();
 308 }
 309
 310 void __init efi_dump_pagetable(void)
 311 {
 312 #ifdef CONFIG_EFI_PGT_DUMP
 313         pgd_t *pgd = (pgd_t *)__va(real_mode_header->trampoline_pgd);
 314
 315         ptdump_walk_pgd_level(NULL, pgd);
 316 #endif
 317 }
 318
 319 #ifdef CONFIG_EFI_MIXED
 320 extern efi_status_t efi64_thunk(u32, ...);
 321
 322 #define runtime_service32(func)                                          \
 323 ({                                                                       \
 324         u32 table = (u32)(unsigned long)efi.systab;                      \
 325         u32 *rt, *___f;                                                  \
 326                                                                          \
 327         rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime));  \
 328         ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \
 329         *___f;                                                           \
 330 })
 331
 332 /*
 333  * Switch to the EFI page tables early so that we can access the 1:1
 334  * runtime services mappings which are not mapped in any other page
 335  * tables. This function must be called before runtime_service32().
 336  *
 337  * Also, disable interrupts because the IDT points to 64-bit handlers,
 338  * which aren't going to function correctly when we switch to 32-bit.
 339  */
 340 #define efi_thunk(f, ...)                                               \
 341 ({                                                                      \
 342         efi_status_t __s;                                               \
 343         unsigned long flags;                                            \
 344         u32 func;                                                       \
 345                                                                         \
 346         efi_sync_low_kernel_mappings();                                 \
 347         local_irq_save(flags);                                          \
 348                                                                         \
 349         efi_scratch.prev_cr3 = read_cr3();                              \
 350         write_cr3((unsigned long)efi_scratch.efi_pgt);                  \
 351         __flush_tlb_all();                                              \
 352                                                                         \
 353         func = runtime_service32(f);                                    \
 354         __s = efi64_thunk(func, __VA_ARGS__);                   \
 355                                                                         \
 356         write_cr3(efi_scratch.prev_cr3);                                \
 357         __flush_tlb_all();                                              \
 358         local_irq_restore(flags);                                       \
 359                                                                         \
 360         __s;                                                            \
 361 })
 362
 363 efi_status_t efi_thunk_set_virtual_address_map(
 364         void *phys_set_virtual_address_map,
 365         unsigned long memory_map_size,
 366         unsigned long descriptor_size,
 367         u32 descriptor_version,
 368         efi_memory_desc_t *virtual_map)
 369 {
 370         efi_status_t status;
 371         unsigned long flags;
 372         u32 func;
 373
 374         efi_sync_low_kernel_mappings();
 375         local_irq_save(flags);
 376
 377         efi_scratch.prev_cr3 = read_cr3();
 378         write_cr3((unsigned long)efi_scratch.efi_pgt);
 379         __flush_tlb_all();
 380
 381         func = (u32)(unsigned long)phys_set_virtual_address_map;
 382         status = efi64_thunk(func, memory_map_size, descriptor_size,
 383                              descriptor_version, virtual_map);
 384
 385         write_cr3(efi_scratch.prev_cr3);
 386         __flush_tlb_all();
 387         local_irq_restore(flags);
 388
 389         return status;
 390 }
 391
 392 static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc)
 393 {
 394         efi_status_t status;
 395         u32 phys_tm, phys_tc;
 396
 397         spin_lock(&rtc_lock);
 398
 399         phys_tm = virt_to_phys(tm);
 400         phys_tc = virt_to_phys(tc);
 401
 402         status = efi_thunk(get_time, phys_tm, phys_tc);
 403
 404         spin_unlock(&rtc_lock);
 405
 406         return status;
 407 }
 408
 409 static efi_status_t efi_thunk_set_time(efi_time_t *tm)
 410 {
 411         efi_status_t status;
 412         u32 phys_tm;
 413
 414         spin_lock(&rtc_lock);
 415
 416         phys_tm = virt_to_phys(tm);
 417
 418         status = efi_thunk(set_time, phys_tm);
 419
 420         spin_unlock(&rtc_lock);
 421
 422         return status;
 423 }
 424
 425 static efi_status_t
 426 efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending,
 427                           efi_time_t *tm)
 428 {
 429         efi_status_t status;
 430         u32 phys_enabled, phys_pending, phys_tm;
 431
 432         spin_lock(&rtc_lock);
 433
 434         phys_enabled = virt_to_phys(enabled);
 435         phys_pending = virt_to_phys(pending);
 436         phys_tm = virt_to_phys(tm);
 437
 438         status = efi_thunk(get_wakeup_time, phys_enabled,
 439                              phys_pending, phys_tm);
 440
 441         spin_unlock(&rtc_lock);
 442
 443         return status;
 444 }
 445
 446 static efi_status_t
 447 efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm)
 448 {
 449         efi_status_t status;
 450         u32 phys_tm;
 451
 452         spin_lock(&rtc_lock);
 453
 454         phys_tm = virt_to_phys(tm);
 455
 456         status = efi_thunk(set_wakeup_time, enabled, phys_tm);
 457
 458         spin_unlock(&rtc_lock);
 459
 460         return status;
 461 }
 462
 463
 464 static efi_status_t
 465 efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor,
 466                        u32 *attr, unsigned long *data_size, void *data)
 467 {
 468         efi_status_t status;
 469         u32 phys_name, phys_vendor, phys_attr;
 470         u32 phys_data_size, phys_data;
 471
 472         phys_data_size = virt_to_phys(data_size);
 473         phys_vendor = virt_to_phys(vendor);
 474         phys_name = virt_to_phys(name);
 475         phys_attr = virt_to_phys(attr);
 476         phys_data = virt_to_phys(data);
 477
 478         status = efi_thunk(get_variable, phys_name, phys_vendor,
 479                            phys_attr, phys_data_size, phys_data);
 480
 481         return status;
 482 }
 483
 484 static efi_status_t
 485 efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor,
 486                        u32 attr, unsigned long data_size, void *data)
 487 {
 488         u32 phys_name, phys_vendor, phys_data;
 489         efi_status_t status;
 490
 491         phys_name = virt_to_phys(name);
 492         phys_vendor = virt_to_phys(vendor);
 493         phys_data = virt_to_phys(data);
 494
 495         /* If data_size is > sizeof(u32) we've got problems */
 496         status = efi_thunk(set_variable, phys_name, phys_vendor,
 497                            attr, data_size, phys_data);
 498
 499         return status;
 500 }
 501
 502 static efi_status_t
 503 efi_thunk_get_next_variable(unsigned long *name_size,
 504                             efi_char16_t *name,
 505                             efi_guid_t *vendor)
 506 {
 507         efi_status_t status;
 508         u32 phys_name_size, phys_name, phys_vendor;
 509
 510         phys_name_size = virt_to_phys(name_size);
 511         phys_vendor = virt_to_phys(vendor);
 512         phys_name = virt_to_phys(name);
 513
 514         status = efi_thunk(get_next_variable, phys_name_size,
 515                            phys_name, phys_vendor);
 516
 517         return status;
 518 }
 519
 520 static efi_status_t
 521 efi_thunk_get_next_high_mono_count(u32 *count)
 522 {
 523         efi_status_t status;
 524         u32 phys_count;
 525
 526         phys_count = virt_to_phys(count);
 527         status = efi_thunk(get_next_high_mono_count, phys_count);
 528
 529         return status;
 530 }
 531
 532 static void
 533 efi_thunk_reset_system(int reset_type, efi_status_t status,
 534                        unsigned long data_size, efi_char16_t *data)
 535 {
 536         u32 phys_data;
 537
 538         phys_data = virt_to_phys(data);
 539
 540         efi_thunk(reset_system, reset_type, status, data_size, phys_data);
 541 }
 542
 543 static efi_status_t
 544 efi_thunk_update_capsule(efi_capsule_header_t **capsules,
 545                          unsigned long count, unsigned long sg_list)
 546 {
 547         /*
 548          * To properly support this function we would need to repackage
 549          * 'capsules' because the firmware doesn't understand 64-bit
 550          * pointers.
 551          */
 552         return EFI_UNSUPPORTED;
 553 }
 554
 555 static efi_status_t
 556 efi_thunk_query_variable_info(u32 attr, u64 *storage_space,
 557                               u64 *remaining_space,
 558                               u64 *max_variable_size)
 559 {
 560         efi_status_t status;
 561         u32 phys_storage, phys_remaining, phys_max;
 562
 563         if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
 564                 return EFI_UNSUPPORTED;
 565
 566         phys_storage = virt_to_phys(storage_space);
 567         phys_remaining = virt_to_phys(remaining_space);
 568         phys_max = virt_to_phys(max_variable_size);
 569
 570         status = efi_thunk(query_variable_info, attr, phys_storage,
 571                            phys_remaining, phys_max);
 572
 573         return status;
 574 }
 575
 576 static efi_status_t
 577 efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules,
 578                              unsigned long count, u64 *max_size,
 579                              int *reset_type)
 580 {
 581         /*
 582          * To properly support this function we would need to repackage
 583          * 'capsules' because the firmware doesn't understand 64-bit
 584          * pointers.
 585          */
 586         return EFI_UNSUPPORTED;
 587 }
 588
 589 void efi_thunk_runtime_setup(void)
 590 {
 591         efi.get_time = efi_thunk_get_time;
 592         efi.set_time = efi_thunk_set_time;
 593         efi.get_wakeup_time = efi_thunk_get_wakeup_time;
 594         efi.set_wakeup_time = efi_thunk_set_wakeup_time;
 595         efi.get_variable = efi_thunk_get_variable;
 596         efi.get_next_variable = efi_thunk_get_next_variable;
 597         efi.set_variable = efi_thunk_set_variable;
 598         efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count;
 599         efi.reset_system = efi_thunk_reset_system;
 600         efi.query_variable_info = efi_thunk_query_variable_info;
 601         efi.update_capsule = efi_thunk_update_capsule;
 602         efi.query_capsule_caps = efi_thunk_query_capsule_caps;
 603 }
 604 #endif /* CONFIG_EFI_MIXED */