arch/x86/mm/pageattr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2002 Andi Kleen, SuSE Labs.
   4  * Thanks to Ben LaHaise for precious feedback.
   5  */
   6 #include <linux/highmem.h>
   7 #include <linux/memblock.h>
   8 #include <linux/sched.h>
   9 #include <linux/mm.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/seq_file.h>
  12 #include <linux/debugfs.h>
  13 #include <linux/pfn.h>
  14 #include <linux/percpu.h>
  15 #include <linux/gfp.h>
  16 #include <linux/pci.h>
  17 #include <linux/vmalloc.h>
  18
  19 #include <asm/e820/api.h>
  20 #include <asm/processor.h>
  21 #include <asm/tlbflush.h>
  22 #include <asm/sections.h>
  23 #include <asm/setup.h>
  24 #include <linux/uaccess.h>
  25 #include <asm/pgalloc.h>
  26 #include <asm/proto.h>
  27 #include <asm/pat.h>
  28 #include <asm/set_memory.h>
  29
  30 #include "mm_internal.h"
  31
  32 /*
  33  * The current flushing context - we pass it instead of 5 arguments:
  34  */
  35 struct cpa_data {
  36         unsigned long   *vaddr;
  37         pgd_t           *pgd;
  38         pgprot_t        mask_set;
  39         pgprot_t        mask_clr;
  40         unsigned long   numpages;
  41         unsigned long   curpage;
  42         unsigned long   pfn;
  43         unsigned int    flags;
  44         unsigned int    force_split             : 1,
  45                         force_static_prot       : 1;
  46         struct page     **pages;
  47 };
  48
  49 enum cpa_warn {
  50         CPA_CONFLICT,
  51         CPA_PROTECT,
  52         CPA_DETECT,
  53 };
  54
  55 static const int cpa_warn_level = CPA_PROTECT;
  56
  57 /*
  58  * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
  59  * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
  60  * entries change the page attribute in parallel to some other cpu
  61  * splitting a large page entry along with changing the attribute.
  62  */
  63 static DEFINE_SPINLOCK(cpa_lock);
  64
  65 #define CPA_FLUSHTLB 1
  66 #define CPA_ARRAY 2
  67 #define CPA_PAGES_ARRAY 4
  68 #define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */
  69
  70 #ifdef CONFIG_PROC_FS
  71 static unsigned long direct_pages_count[PG_LEVEL_NUM];
  72
  73 void update_page_count(int level, unsigned long pages)
  74 {
  75         /* Protect against CPA */
  76         spin_lock(&pgd_lock);
  77         direct_pages_count[level] += pages;
  78         spin_unlock(&pgd_lock);
  79 }
  80
  81 static void split_page_count(int level)
  82 {
  83         if (direct_pages_count[level] == 0)
  84                 return;
  85
  86         direct_pages_count[level]--;
  87         direct_pages_count[level - 1] += PTRS_PER_PTE;
  88 }
  89
  90 void arch_report_meminfo(struct seq_file *m)
  91 {
  92         seq_printf(m, "DirectMap4k:    %8lu kB\n",
  93                         direct_pages_count[PG_LEVEL_4K] << 2);
  94 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
  95         seq_printf(m, "DirectMap2M:    %8lu kB\n",
  96                         direct_pages_count[PG_LEVEL_2M] << 11);
  97 #else
  98         seq_printf(m, "DirectMap4M:    %8lu kB\n",
  99                         direct_pages_count[PG_LEVEL_2M] << 12);
 100 #endif
 101         if (direct_gbpages)
 102                 seq_printf(m, "DirectMap1G:    %8lu kB\n",
 103                         direct_pages_count[PG_LEVEL_1G] << 20);
 104 }
 105 #else
 106 static inline void split_page_count(int level) { }
 107 #endif
 108
 109 #ifdef CONFIG_X86_CPA_STATISTICS
 110
 111 static unsigned long cpa_1g_checked;
 112 static unsigned long cpa_1g_sameprot;
 113 static unsigned long cpa_1g_preserved;
 114 static unsigned long cpa_2m_checked;
 115 static unsigned long cpa_2m_sameprot;
 116 static unsigned long cpa_2m_preserved;
 117 static unsigned long cpa_4k_install;
 118
 119 static inline void cpa_inc_1g_checked(void)
 120 {
 121         cpa_1g_checked++;
 122 }
 123
 124 static inline void cpa_inc_2m_checked(void)
 125 {
 126         cpa_2m_checked++;
 127 }
 128
 129 static inline void cpa_inc_4k_install(void)
 130 {
 131         cpa_4k_install++;
 132 }
 133
 134 static inline void cpa_inc_lp_sameprot(int level)
 135 {
 136         if (level == PG_LEVEL_1G)
 137                 cpa_1g_sameprot++;
 138         else
 139                 cpa_2m_sameprot++;
 140 }
 141
 142 static inline void cpa_inc_lp_preserved(int level)
 143 {
 144         if (level == PG_LEVEL_1G)
 145                 cpa_1g_preserved++;
 146         else
 147                 cpa_2m_preserved++;
 148 }
 149
 150 static int cpastats_show(struct seq_file *m, void *p)
 151 {
 152         seq_printf(m, "1G pages checked:     %16lu\n", cpa_1g_checked);
 153         seq_printf(m, "1G pages sameprot:    %16lu\n", cpa_1g_sameprot);
 154         seq_printf(m, "1G pages preserved:   %16lu\n", cpa_1g_preserved);
 155         seq_printf(m, "2M pages checked:     %16lu\n", cpa_2m_checked);
 156         seq_printf(m, "2M pages sameprot:    %16lu\n", cpa_2m_sameprot);
 157         seq_printf(m, "2M pages preserved:   %16lu\n", cpa_2m_preserved);
 158         seq_printf(m, "4K pages set-checked: %16lu\n", cpa_4k_install);
 159         return 0;
 160 }
 161
 162 static int cpastats_open(struct inode *inode, struct file *file)
 163 {
 164         return single_open(file, cpastats_show, NULL);
 165 }
 166
 167 static const struct file_operations cpastats_fops = {
 168         .open           = cpastats_open,
 169         .read           = seq_read,
 170         .llseek         = seq_lseek,
 171         .release        = single_release,
 172 };
 173
 174 static int __init cpa_stats_init(void)
 175 {
 176         debugfs_create_file("cpa_stats", S_IRUSR, arch_debugfs_dir, NULL,
 177                             &cpastats_fops);
 178         return 0;
 179 }
 180 late_initcall(cpa_stats_init);
 181 #else
 182 static inline void cpa_inc_1g_checked(void) { }
 183 static inline void cpa_inc_2m_checked(void) { }
 184 static inline void cpa_inc_4k_install(void) { }
 185 static inline void cpa_inc_lp_sameprot(int level) { }
 186 static inline void cpa_inc_lp_preserved(int level) { }
 187 #endif
 188
 189
 190 static inline int
 191 within(unsigned long addr, unsigned long start, unsigned long end)
 192 {
 193         return addr >= start && addr < end;
 194 }
 195
 196 static inline int
 197 within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
 198 {
 199         return addr >= start && addr <= end;
 200 }
 201
 202 #ifdef CONFIG_X86_64
 203
 204 static inline unsigned long highmap_start_pfn(void)
 205 {
 206         return __pa_symbol(_text) >> PAGE_SHIFT;
 207 }
 208
 209 static inline unsigned long highmap_end_pfn(void)
 210 {
 211         /* Do not reference physical address outside the kernel. */
 212         return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
 213 }
 214
 215 static bool __cpa_pfn_in_highmap(unsigned long pfn)
 216 {
 217         /*
 218          * Kernel text has an alias mapping at a high address, known
 219          * here as "highmap".
 220          */
 221         return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());
 222 }
 223
 224 #else
 225
 226 static bool __cpa_pfn_in_highmap(unsigned long pfn)
 227 {
 228         /* There is no highmap on 32-bit */
 229         return false;
 230 }
 231
 232 #endif
 233
 234 /*
 235  * See set_mce_nospec().
 236  *
 237  * Machine check recovery code needs to change cache mode of poisoned pages to
 238  * UC to avoid speculative access logging another error. But passing the
 239  * address of the 1:1 mapping to set_memory_uc() is a fine way to encourage a
 240  * speculative access. So we cheat and flip the top bit of the address. This
 241  * works fine for the code that updates the page tables. But at the end of the
 242  * process we need to flush the TLB and cache and the non-canonical address
 243  * causes a #GP fault when used by the INVLPG and CLFLUSH instructions.
 244  *
 245  * But in the common case we already have a canonical address. This code
 246  * will fix the top bit if needed and is a no-op otherwise.
 247  */
 248 static inline unsigned long fix_addr(unsigned long addr)
 249 {
 250 #ifdef CONFIG_X86_64
 251         return (long)(addr << 1) >> 1;
 252 #else
 253         return addr;
 254 #endif
 255 }
 256
 257 static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
 258 {
 259         if (cpa->flags & CPA_PAGES_ARRAY) {
 260                 struct page *page = cpa->pages[idx];
 261
 262                 if (unlikely(PageHighMem(page)))
 263                         return 0;
 264
 265                 return (unsigned long)page_address(page);
 266         }
 267
 268         if (cpa->flags & CPA_ARRAY)
 269                 return cpa->vaddr[idx];
 270
 271         return *cpa->vaddr + idx * PAGE_SIZE;
 272 }
 273
 274 /*
 275  * Flushing functions
 276  */
 277
 278 static void clflush_cache_range_opt(void *vaddr, unsigned int size)
 279 {
 280         const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
 281         void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
 282         void *vend = vaddr + size;
 283
 284         if (p >= vend)
 285                 return;
 286
 287         for (; p < vend; p += clflush_size)
 288                 clflushopt(p);
 289 }
 290
 291 /**
 292  * clflush_cache_range - flush a cache range with clflush
 293  * @vaddr:      virtual start address
 294  * @size:       number of bytes to flush
 295  *
 296  * CLFLUSHOPT is an unordered instruction which needs fencing with MFENCE or
 297  * SFENCE to avoid ordering issues.
 298  */
 299 void clflush_cache_range(void *vaddr, unsigned int size)
 300 {
 301         mb();
 302         clflush_cache_range_opt(vaddr, size);
 303         mb();
 304 }
 305 EXPORT_SYMBOL_GPL(clflush_cache_range);
 306
 307 void arch_invalidate_pmem(void *addr, size_t size)
 308 {
 309         clflush_cache_range(addr, size);
 310 }
 311 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
 312
 313 static void __cpa_flush_all(void *arg)
 314 {
 315         unsigned long cache = (unsigned long)arg;
 316
 317         /*
 318          * Flush all to work around Errata in early athlons regarding
 319          * large page flushing.
 320          */
 321         __flush_tlb_all();
 322
 323         if (cache && boot_cpu_data.x86 >= 4)
 324                 wbinvd();
 325 }
 326
 327 static void cpa_flush_all(unsigned long cache)
 328 {
 329         BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 330
 331         on_each_cpu(__cpa_flush_all, (void *) cache, 1);
 332 }
 333
 334 void __cpa_flush_tlb(void *data)
 335 {
 336         struct cpa_data *cpa = data;
 337         unsigned int i;
 338
 339         for (i = 0; i < cpa->numpages; i++)
 340                 __flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
 341 }
 342
 343 static void cpa_flush(struct cpa_data *data, int cache)
 344 {
 345         struct cpa_data *cpa = data;
 346         unsigned int i;
 347
 348         BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 349
 350         if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 351                 cpa_flush_all(cache);
 352                 return;
 353         }
 354
 355         if (cpa->numpages <= tlb_single_page_flush_ceiling)
 356                 on_each_cpu(__cpa_flush_tlb, cpa, 1);
 357         else
 358                 flush_tlb_all();
 359
 360         if (!cache)
 361                 return;
 362
 363         mb();
 364         for (i = 0; i < cpa->numpages; i++) {
 365                 unsigned long addr = __cpa_addr(cpa, i);
 366                 unsigned int level;
 367
 368                 pte_t *pte = lookup_address(addr, &level);
 369
 370                 /*
 371                  * Only flush present addresses:
 372                  */
 373                 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
 374                         clflush_cache_range_opt((void *)fix_addr(addr), PAGE_SIZE);
 375         }
 376         mb();
 377 }
 378
 379 static bool overlaps(unsigned long r1_start, unsigned long r1_end,
 380                      unsigned long r2_start, unsigned long r2_end)
 381 {
 382         return (r1_start <= r2_end && r1_end >= r2_start) ||
 383                 (r2_start <= r1_end && r2_end >= r1_start);
 384 }
 385
 386 #ifdef CONFIG_PCI_BIOS
 387 /*
 388  * The BIOS area between 640k and 1Mb needs to be executable for PCI BIOS
 389  * based config access (CONFIG_PCI_GOBIOS) support.
 390  */
 391 #define BIOS_PFN        PFN_DOWN(BIOS_BEGIN)
 392 #define BIOS_PFN_END    PFN_DOWN(BIOS_END - 1)
 393
 394 static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
 395 {
 396         if (pcibios_enabled && overlaps(spfn, epfn, BIOS_PFN, BIOS_PFN_END))
 397                 return _PAGE_NX;
 398         return 0;
 399 }
 400 #else
 401 static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
 402 {
 403         return 0;
 404 }
 405 #endif
 406
 407 /*
 408  * The .rodata section needs to be read-only. Using the pfn catches all
 409  * aliases.  This also includes __ro_after_init, so do not enforce until
 410  * kernel_set_to_readonly is true.
 411  */
 412 static pgprotval_t protect_rodata(unsigned long spfn, unsigned long epfn)
 413 {
 414         unsigned long epfn_ro, spfn_ro = PFN_DOWN(__pa_symbol(__start_rodata));
 415
 416         /*
 417          * Note: __end_rodata is at page aligned and not inclusive, so
 418          * subtract 1 to get the last enforced PFN in the rodata area.
 419          */
 420         epfn_ro = PFN_DOWN(__pa_symbol(__end_rodata)) - 1;
 421
 422         if (kernel_set_to_readonly && overlaps(spfn, epfn, spfn_ro, epfn_ro))
 423                 return _PAGE_RW;
 424         return 0;
 425 }
 426
 427 /*
 428  * Protect kernel text against becoming non executable by forbidding
 429  * _PAGE_NX.  This protects only the high kernel mapping (_text -> _etext)
 430  * out of which the kernel actually executes.  Do not protect the low
 431  * mapping.
 432  *
 433  * This does not cover __inittext since that is gone after boot.
 434  */
 435 static pgprotval_t protect_kernel_text(unsigned long start, unsigned long end)
 436 {
 437         unsigned long t_end = (unsigned long)_etext - 1;
 438         unsigned long t_start = (unsigned long)_text;
 439
 440         if (overlaps(start, end, t_start, t_end))
 441                 return _PAGE_NX;
 442         return 0;
 443 }
 444
 445 #if defined(CONFIG_X86_64)
 446 /*
 447  * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
 448  * kernel text mappings for the large page aligned text, rodata sections
 449  * will be always read-only. For the kernel identity mappings covering the
 450  * holes caused by this alignment can be anything that user asks.
 451  *
 452  * This will preserve the large page mappings for kernel text/data at no
 453  * extra cost.
 454  */
 455 static pgprotval_t protect_kernel_text_ro(unsigned long start,
 456                                           unsigned long end)
 457 {
 458         unsigned long t_end = (unsigned long)__end_rodata_hpage_align - 1;
 459         unsigned long t_start = (unsigned long)_text;
 460         unsigned int level;
 461
 462         if (!kernel_set_to_readonly || !overlaps(start, end, t_start, t_end))
 463                 return 0;
 464         /*
 465          * Don't enforce the !RW mapping for the kernel text mapping, if
 466          * the current mapping is already using small page mapping.  No
 467          * need to work hard to preserve large page mappings in this case.
 468          *
 469          * This also fixes the Linux Xen paravirt guest boot failure caused
 470          * by unexpected read-only mappings for kernel identity
 471          * mappings. In this paravirt guest case, the kernel text mapping
 472          * and the kernel identity mapping share the same page-table pages,
 473          * so the protections for kernel text and identity mappings have to
 474          * be the same.
 475          */
 476         if (lookup_address(start, &level) && (level != PG_LEVEL_4K))
 477                 return _PAGE_RW;
 478         return 0;
 479 }
 480 #else
 481 static pgprotval_t protect_kernel_text_ro(unsigned long start,
 482                                           unsigned long end)
 483 {
 484         return 0;
 485 }
 486 #endif
 487
 488 static inline bool conflicts(pgprot_t prot, pgprotval_t val)
 489 {
 490         return (pgprot_val(prot) & ~val) != pgprot_val(prot);
 491 }
 492
 493 static inline void check_conflict(int warnlvl, pgprot_t prot, pgprotval_t val,
 494                                   unsigned long start, unsigned long end,
 495                                   unsigned long pfn, const char *txt)
 496 {
 497         static const char *lvltxt[] = {
 498                 [CPA_CONFLICT]  = "conflict",
 499                 [CPA_PROTECT]   = "protect",
 500                 [CPA_DETECT]    = "detect",
 501         };
 502
 503         if (warnlvl > cpa_warn_level || !conflicts(prot, val))
 504                 return;
 505
 506         pr_warn("CPA %8s %10s: 0x%016lx - 0x%016lx PFN %lx req %016llx prevent %016llx\n",
 507                 lvltxt[warnlvl], txt, start, end, pfn, (unsigned long long)pgprot_val(prot),
 508                 (unsigned long long)val);
 509 }
 510
 511 /*
 512  * Certain areas of memory on x86 require very specific protection flags,
 513  * for example the BIOS area or kernel text. Callers don't always get this
 514  * right (again, ioremap() on BIOS memory is not uncommon) so this function
 515  * checks and fixes these known static required protection bits.
 516  */
 517 static inline pgprot_t static_protections(pgprot_t prot, unsigned long start,
 518                                           unsigned long pfn, unsigned long npg,
 519                                           int warnlvl)
 520 {
 521         pgprotval_t forbidden, res;
 522         unsigned long end;
 523
 524         /*
 525          * There is no point in checking RW/NX conflicts when the requested
 526          * mapping is setting the page !PRESENT.
 527          */
 528         if (!(pgprot_val(prot) & _PAGE_PRESENT))
 529                 return prot;
 530
 531         /* Operate on the virtual address */
 532         end = start + npg * PAGE_SIZE - 1;
 533
 534         res = protect_kernel_text(start, end);
 535         check_conflict(warnlvl, prot, res, start, end, pfn, "Text NX");
 536         forbidden = res;
 537
 538         res = protect_kernel_text_ro(start, end);
 539         check_conflict(warnlvl, prot, res, start, end, pfn, "Text RO");
 540         forbidden |= res;
 541
 542         /* Check the PFN directly */
 543         res = protect_pci_bios(pfn, pfn + npg - 1);
 544         check_conflict(warnlvl, prot, res, start, end, pfn, "PCIBIOS NX");
 545         forbidden |= res;
 546
 547         res = protect_rodata(pfn, pfn + npg - 1);
 548         check_conflict(warnlvl, prot, res, start, end, pfn, "Rodata RO");
 549         forbidden |= res;
 550
 551         return __pgprot(pgprot_val(prot) & ~forbidden);
 552 }
 553
 554 /*
 555  * Lookup the page table entry for a virtual address in a specific pgd.
 556  * Return a pointer to the entry and the level of the mapping.
 557  */
 558 pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
 559                              unsigned int *level)
 560 {
 561         p4d_t *p4d;
 562         pud_t *pud;
 563         pmd_t *pmd;
 564
 565         *level = PG_LEVEL_NONE;
 566
 567         if (pgd_none(*pgd))
 568                 return NULL;
 569
 570         p4d = p4d_offset(pgd, address);
 571         if (p4d_none(*p4d))
 572                 return NULL;
 573
 574         *level = PG_LEVEL_512G;
 575         if (p4d_large(*p4d) || !p4d_present(*p4d))
 576                 return (pte_t *)p4d;
 577
 578         pud = pud_offset(p4d, address);
 579         if (pud_none(*pud))
 580                 return NULL;
 581
 582         *level = PG_LEVEL_1G;
 583         if (pud_large(*pud) || !pud_present(*pud))
 584                 return (pte_t *)pud;
 585
 586         pmd = pmd_offset(pud, address);
 587         if (pmd_none(*pmd))
 588                 return NULL;
 589
 590         *level = PG_LEVEL_2M;
 591         if (pmd_large(*pmd) || !pmd_present(*pmd))
 592                 return (pte_t *)pmd;
 593
 594         *level = PG_LEVEL_4K;
 595
 596         return pte_offset_kernel(pmd, address);
 597 }
 598
 599 /*
 600  * Lookup the page table entry for a virtual address. Return a pointer
 601  * to the entry and the level of the mapping.
 602  *
 603  * Note: We return pud and pmd either when the entry is marked large
 604  * or when the present bit is not set. Otherwise we would return a
 605  * pointer to a nonexisting mapping.
 606  */
 607 pte_t *lookup_address(unsigned long address, unsigned int *level)
 608 {
 609         return lookup_address_in_pgd(pgd_offset_k(address), address, level);
 610 }
 611 EXPORT_SYMBOL_GPL(lookup_address);
 612
 613 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
 614                                   unsigned int *level)
 615 {
 616         if (cpa->pgd)
 617                 return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
 618                                                address, level);
 619
 620         return lookup_address(address, level);
 621 }
 622
 623 /*
 624  * Lookup the PMD entry for a virtual address. Return a pointer to the entry
 625  * or NULL if not present.
 626  */
 627 pmd_t *lookup_pmd_address(unsigned long address)
 628 {
 629         pgd_t *pgd;
 630         p4d_t *p4d;
 631         pud_t *pud;
 632
 633         pgd = pgd_offset_k(address);
 634         if (pgd_none(*pgd))
 635                 return NULL;
 636
 637         p4d = p4d_offset(pgd, address);
 638         if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d))
 639                 return NULL;
 640
 641         pud = pud_offset(p4d, address);
 642         if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
 643                 return NULL;
 644
 645         return pmd_offset(pud, address);
 646 }
 647
 648 /*
 649  * This is necessary because __pa() does not work on some
 650  * kinds of memory, like vmalloc() or the alloc_remap()
 651  * areas on 32-bit NUMA systems.  The percpu areas can
 652  * end up in this kind of memory, for instance.
 653  *
 654  * This could be optimized, but it is only intended to be
 655  * used at inititalization time, and keeping it
 656  * unoptimized should increase the testing coverage for
 657  * the more obscure platforms.
 658  */
 659 phys_addr_t slow_virt_to_phys(void *__virt_addr)
 660 {
 661         unsigned long virt_addr = (unsigned long)__virt_addr;
 662         phys_addr_t phys_addr;
 663         unsigned long offset;
 664         enum pg_level level;
 665         pte_t *pte;
 666
 667         pte = lookup_address(virt_addr, &level);
 668         BUG_ON(!pte);
 669
 670         /*
 671          * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
 672          * before being left-shifted PAGE_SHIFT bits -- this trick is to
 673          * make 32-PAE kernel work correctly.
 674          */
 675         switch (level) {
 676         case PG_LEVEL_1G:
 677                 phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
 678                 offset = virt_addr & ~PUD_PAGE_MASK;
 679                 break;
 680         case PG_LEVEL_2M:
 681                 phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
 682                 offset = virt_addr & ~PMD_PAGE_MASK;
 683                 break;
 684         default:
 685                 phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
 686                 offset = virt_addr & ~PAGE_MASK;
 687         }
 688
 689         return (phys_addr_t)(phys_addr | offset);
 690 }
 691 EXPORT_SYMBOL_GPL(slow_virt_to_phys);
 692
 693 /*
 694  * Set the new pmd in all the pgds we know about:
 695  */
 696 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 697 {
 698         /* change init_mm */
 699         set_pte_atomic(kpte, pte);
 700 #ifdef CONFIG_X86_32
 701         if (!SHARED_KERNEL_PMD) {
 702                 struct page *page;
 703
 704                 list_for_each_entry(page, &pgd_list, lru) {
 705                         pgd_t *pgd;
 706                         p4d_t *p4d;
 707                         pud_t *pud;
 708                         pmd_t *pmd;
 709
 710                         pgd = (pgd_t *)page_address(page) + pgd_index(address);
 711                         p4d = p4d_offset(pgd, address);
 712                         pud = pud_offset(p4d, address);
 713                         pmd = pmd_offset(pud, address);
 714                         set_pte_atomic((pte_t *)pmd, pte);
 715                 }
 716         }
 717 #endif
 718 }
 719
 720 static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)
 721 {
 722         /*
 723          * _PAGE_GLOBAL means "global page" for present PTEs.
 724          * But, it is also used to indicate _PAGE_PROTNONE
 725          * for non-present PTEs.
 726          *
 727          * This ensures that a _PAGE_GLOBAL PTE going from
 728          * present to non-present is not confused as
 729          * _PAGE_PROTNONE.
 730          */
 731         if (!(pgprot_val(prot) & _PAGE_PRESENT))
 732                 pgprot_val(prot) &= ~_PAGE_GLOBAL;
 733
 734         return prot;
 735 }
 736
 737 static int __should_split_large_page(pte_t *kpte, unsigned long address,
 738                                      struct cpa_data *cpa)
 739 {
 740         unsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn;
 741         pgprot_t old_prot, new_prot, req_prot, chk_prot;
 742         pte_t new_pte, *tmp;
 743         enum pg_level level;
 744
 745         /*
 746          * Check for races, another CPU might have split this page
 747          * up already:
 748          */
 749         tmp = _lookup_address_cpa(cpa, address, &level);
 750         if (tmp != kpte)
 751                 return 1;
 752
 753         switch (level) {
 754         case PG_LEVEL_2M:
 755                 old_prot = pmd_pgprot(*(pmd_t *)kpte);
 756                 old_pfn = pmd_pfn(*(pmd_t *)kpte);
 757                 cpa_inc_2m_checked();
 758                 break;
 759         case PG_LEVEL_1G:
 760                 old_prot = pud_pgprot(*(pud_t *)kpte);
 761                 old_pfn = pud_pfn(*(pud_t *)kpte);
 762                 cpa_inc_1g_checked();
 763                 break;
 764         default:
 765                 return -EINVAL;
 766         }
 767
 768         psize = page_level_size(level);
 769         pmask = page_level_mask(level);
 770
 771         /*
 772          * Calculate the number of pages, which fit into this large
 773          * page starting at address:
 774          */
 775         lpaddr = (address + psize) & pmask;
 776         numpages = (lpaddr - address) >> PAGE_SHIFT;
 777         if (numpages < cpa->numpages)
 778                 cpa->numpages = numpages;
 779
 780         /*
 781          * We are safe now. Check whether the new pgprot is the same:
 782          * Convert protection attributes to 4k-format, as cpa->mask* are set
 783          * up accordingly.
 784          */
 785
 786         /* Clear PSE (aka _PAGE_PAT) and move PAT bit to correct position */
 787         req_prot = pgprot_large_2_4k(old_prot);
 788
 789         pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
 790         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
 791
 792         /*
 793          * req_prot is in format of 4k pages. It must be converted to large
 794          * page format: the caching mode includes the PAT bit located at
 795          * different bit positions in the two formats.
 796          */
 797         req_prot = pgprot_4k_2_large(req_prot);
 798         req_prot = pgprot_clear_protnone_bits(req_prot);
 799         if (pgprot_val(req_prot) & _PAGE_PRESENT)
 800                 pgprot_val(req_prot) |= _PAGE_PSE;
 801
 802         /*
 803          * old_pfn points to the large page base pfn. So we need to add the
 804          * offset of the virtual address:
 805          */
 806         pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
 807         cpa->pfn = pfn;
 808
 809         /*
 810          * Calculate the large page base address and the number of 4K pages
 811          * in the large page
 812          */
 813         lpaddr = address & pmask;
 814         numpages = psize >> PAGE_SHIFT;
 815
 816         /*
 817          * Sanity check that the existing mapping is correct versus the static
 818          * protections. static_protections() guards against !PRESENT, so no
 819          * extra conditional required here.
 820          */
 821         chk_prot = static_protections(old_prot, lpaddr, old_pfn, numpages,
 822                                       CPA_CONFLICT);
 823
 824         if (WARN_ON_ONCE(pgprot_val(chk_prot) != pgprot_val(old_prot))) {
 825                 /*
 826                  * Split the large page and tell the split code to
 827                  * enforce static protections.
 828                  */
 829                 cpa->force_static_prot = 1;
 830                 return 1;
 831         }
 832
 833         /*
 834          * Optimization: If the requested pgprot is the same as the current
 835          * pgprot, then the large page can be preserved and no updates are
 836          * required independent of alignment and length of the requested
 837          * range. The above already established that the current pgprot is
 838          * correct, which in consequence makes the requested pgprot correct
 839          * as well if it is the same. The static protection scan below will
 840          * not come to a different conclusion.
 841          */
 842         if (pgprot_val(req_prot) == pgprot_val(old_prot)) {
 843                 cpa_inc_lp_sameprot(level);
 844                 return 0;
 845         }
 846
 847         /*
 848          * If the requested range does not cover the full page, split it up
 849          */
 850         if (address != lpaddr || cpa->numpages != numpages)
 851                 return 1;
 852
 853         /*
 854          * Check whether the requested pgprot is conflicting with a static
 855          * protection requirement in the large page.
 856          */
 857         new_prot = static_protections(req_prot, lpaddr, old_pfn, numpages,
 858                                       CPA_DETECT);
 859
 860         /*
 861          * If there is a conflict, split the large page.
 862          *
 863          * There used to be a 4k wise evaluation trying really hard to
 864          * preserve the large pages, but experimentation has shown, that this
 865          * does not help at all. There might be corner cases which would
 866          * preserve one large page occasionally, but it's really not worth the
 867          * extra code and cycles for the common case.
 868          */
 869         if (pgprot_val(req_prot) != pgprot_val(new_prot))
 870                 return 1;
 871
 872         /* All checks passed. Update the large page mapping. */
 873         new_pte = pfn_pte(old_pfn, new_prot);
 874         __set_pmd_pte(kpte, address, new_pte);
 875         cpa->flags |= CPA_FLUSHTLB;
 876         cpa_inc_lp_preserved(level);
 877         return 0;
 878 }
 879
 880 static int should_split_large_page(pte_t *kpte, unsigned long address,
 881                                    struct cpa_data *cpa)
 882 {
 883         int do_split;
 884
 885         if (cpa->force_split)
 886                 return 1;
 887
 888         spin_lock(&pgd_lock);
 889         do_split = __should_split_large_page(kpte, address, cpa);
 890         spin_unlock(&pgd_lock);
 891
 892         return do_split;
 893 }
 894
 895 static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
 896                           pgprot_t ref_prot, unsigned long address,
 897                           unsigned long size)
 898 {
 899         unsigned int npg = PFN_DOWN(size);
 900         pgprot_t prot;
 901
 902         /*
 903          * If should_split_large_page() discovered an inconsistent mapping,
 904          * remove the invalid protection in the split mapping.
 905          */
 906         if (!cpa->force_static_prot)
 907                 goto set;
 908
 909         prot = static_protections(ref_prot, address, pfn, npg, CPA_PROTECT);
 910
 911         if (pgprot_val(prot) == pgprot_val(ref_prot))
 912                 goto set;
 913
 914         /*
 915          * If this is splitting a PMD, fix it up. PUD splits cannot be
 916          * fixed trivially as that would require to rescan the newly
 917          * installed PMD mappings after returning from split_large_page()
 918          * so an eventual further split can allocate the necessary PTE
 919          * pages. Warn for now and revisit it in case this actually
 920          * happens.
 921          */
 922         if (size == PAGE_SIZE)
 923                 ref_prot = prot;
 924         else
 925                 pr_warn_once("CPA: Cannot fixup static protections for PUD split\n");
 926 set:
 927         set_pte(pte, pfn_pte(pfn, ref_prot));
 928 }
 929
 930 static int
 931 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 932                    struct page *base)
 933 {
 934         unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
 935         pte_t *pbase = (pte_t *)page_address(base);
 936         unsigned int i, level;
 937         pgprot_t ref_prot;
 938         pte_t *tmp;
 939
 940         spin_lock(&pgd_lock);
 941         /*
 942          * Check for races, another CPU might have split this page
 943          * up for us already:
 944          */
 945         tmp = _lookup_address_cpa(cpa, address, &level);
 946         if (tmp != kpte) {
 947                 spin_unlock(&pgd_lock);
 948                 return 1;
 949         }
 950
 951         paravirt_alloc_pte(&init_mm, page_to_pfn(base));
 952
 953         switch (level) {
 954         case PG_LEVEL_2M:
 955                 ref_prot = pmd_pgprot(*(pmd_t *)kpte);
 956                 /*
 957                  * Clear PSE (aka _PAGE_PAT) and move
 958                  * PAT bit to correct position.
 959                  */
 960                 ref_prot = pgprot_large_2_4k(ref_prot);
 961                 ref_pfn = pmd_pfn(*(pmd_t *)kpte);
 962                 lpaddr = address & PMD_MASK;
 963                 lpinc = PAGE_SIZE;
 964                 break;
 965
 966         case PG_LEVEL_1G:
 967                 ref_prot = pud_pgprot(*(pud_t *)kpte);
 968                 ref_pfn = pud_pfn(*(pud_t *)kpte);
 969                 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
 970                 lpaddr = address & PUD_MASK;
 971                 lpinc = PMD_SIZE;
 972                 /*
 973                  * Clear the PSE flags if the PRESENT flag is not set
 974                  * otherwise pmd_present/pmd_huge will return true
 975                  * even on a non present pmd.
 976                  */
 977                 if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
 978                         pgprot_val(ref_prot) &= ~_PAGE_PSE;
 979                 break;
 980
 981         default:
 982                 spin_unlock(&pgd_lock);
 983                 return 1;
 984         }
 985
 986         ref_prot = pgprot_clear_protnone_bits(ref_prot);
 987
 988         /*
 989          * Get the target pfn from the original entry:
 990          */
 991         pfn = ref_pfn;
 992         for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc, lpaddr += lpinc)
 993                 split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc);
 994
 995         if (virt_addr_valid(address)) {
 996                 unsigned long pfn = PFN_DOWN(__pa(address));
 997
 998                 if (pfn_range_is_mapped(pfn, pfn + 1))
 999                         split_page_count(level);
1000         }
1001
1002         /*
1003          * Install the new, split up pagetable.
1004          *
1005          * We use the standard kernel pagetable protections for the new
1006          * pagetable protections, the actual ptes set above control the
1007          * primary protection behavior:
1008          */
1009         __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
1010
1011         /*
1012          * Do a global flush tlb after splitting the large page
1013          * and before we do the actual change page attribute in the PTE.
1014          *
1015          * Without this, we violate the TLB application note, that says:
1016          * "The TLBs may contain both ordinary and large-page
1017          *  translations for a 4-KByte range of linear addresses. This
1018          *  may occur if software modifies the paging structures so that
1019          *  the page size used for the address range changes. If the two
1020          *  translations differ with respect to page frame or attributes
1021          *  (e.g., permissions), processor behavior is undefined and may
1022          *  be implementation-specific."
1023          *
1024          * We do this global tlb flush inside the cpa_lock, so that we
1025          * don't allow any other cpu, with stale tlb entries change the
1026          * page attribute in parallel, that also falls into the
1027          * just split large page entry.
1028          */
1029         flush_tlb_all();
1030         spin_unlock(&pgd_lock);
1031
1032         return 0;
1033 }
1034
1035 static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
1036                             unsigned long address)
1037 {
1038         struct page *base;
1039
1040         if (!debug_pagealloc_enabled())
1041                 spin_unlock(&cpa_lock);
1042         base = alloc_pages(GFP_KERNEL, 0);
1043         if (!debug_pagealloc_enabled())
1044                 spin_lock(&cpa_lock);
1045         if (!base)
1046                 return -ENOMEM;
1047
1048         if (__split_large_page(cpa, kpte, address, base))
1049                 __free_page(base);
1050
1051         return 0;
1052 }
1053
1054 static bool try_to_free_pte_page(pte_t *pte)
1055 {
1056         int i;
1057
1058         for (i = 0; i < PTRS_PER_PTE; i++)
1059                 if (!pte_none(pte[i]))
1060                         return false;
1061
1062         free_page((unsigned long)pte);
1063         return true;
1064 }
1065
1066 static bool try_to_free_pmd_page(pmd_t *pmd)
1067 {
1068         int i;
1069
1070         for (i = 0; i < PTRS_PER_PMD; i++)
1071                 if (!pmd_none(pmd[i]))
1072                         return false;
1073
1074         free_page((unsigned long)pmd);
1075         return true;
1076 }
1077
1078 static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
1079 {
1080         pte_t *pte = pte_offset_kernel(pmd, start);
1081
1082         while (start < end) {
1083                 set_pte(pte, __pte(0));
1084
1085                 start += PAGE_SIZE;
1086                 pte++;
1087         }
1088
1089         if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
1090                 pmd_clear(pmd);
1091                 return true;
1092         }
1093         return false;
1094 }
1095
1096 static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
1097                               unsigned long start, unsigned long end)
1098 {
1099         if (unmap_pte_range(pmd, start, end))
1100                 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
1101                         pud_clear(pud);
1102 }
1103
1104 static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
1105 {
1106         pmd_t *pmd = pmd_offset(pud, start);
1107
1108         /*
1109          * Not on a 2MB page boundary?
1110          */
1111         if (start & (PMD_SIZE - 1)) {
1112                 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
1113                 unsigned long pre_end = min_t(unsigned long, end, next_page);
1114
1115                 __unmap_pmd_range(pud, pmd, start, pre_end);
1116
1117                 start = pre_end;
1118                 pmd++;
1119         }
1120
1121         /*
1122          * Try to unmap in 2M chunks.
1123          */
1124         while (end - start >= PMD_SIZE) {
1125                 if (pmd_large(*pmd))
1126                         pmd_clear(pmd);
1127                 else
1128                         __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
1129
1130                 start += PMD_SIZE;
1131                 pmd++;
1132         }
1133
1134         /*
1135          * 4K leftovers?
1136          */
1137         if (start < end)
1138                 return __unmap_pmd_range(pud, pmd, start, end);
1139
1140         /*
1141          * Try again to free the PMD page if haven't succeeded above.
1142          */
1143         if (!pud_none(*pud))
1144                 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
1145                         pud_clear(pud);
1146 }
1147
1148 static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
1149 {
1150         pud_t *pud = pud_offset(p4d, start);
1151
1152         /*
1153          * Not on a GB page boundary?
1154          */
1155         if (start & (PUD_SIZE - 1)) {
1156                 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1157                 unsigned long pre_end   = min_t(unsigned long, end, next_page);
1158
1159                 unmap_pmd_range(pud, start, pre_end);
1160
1161                 start = pre_end;
1162                 pud++;
1163         }
1164
1165         /*
1166          * Try to unmap in 1G chunks?
1167          */
1168         while (end - start >= PUD_SIZE) {
1169
1170                 if (pud_large(*pud))
1171                         pud_clear(pud);
1172                 else
1173                         unmap_pmd_range(pud, start, start + PUD_SIZE);
1174
1175                 start += PUD_SIZE;
1176                 pud++;
1177         }
1178
1179         /*
1180          * 2M leftovers?
1181          */
1182         if (start < end)
1183                 unmap_pmd_range(pud, start, end);
1184
1185         /*
1186          * No need to try to free the PUD page because we'll free it in
1187          * populate_pgd's error path
1188          */
1189 }
1190
1191 static int alloc_pte_page(pmd_t *pmd)
1192 {
1193         pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
1194         if (!pte)
1195                 return -1;
1196
1197         set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
1198         return 0;
1199 }
1200
1201 static int alloc_pmd_page(pud_t *pud)
1202 {
1203         pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
1204         if (!pmd)
1205                 return -1;
1206
1207         set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
1208         return 0;
1209 }
1210
1211 static void populate_pte(struct cpa_data *cpa,
1212                          unsigned long start, unsigned long end,
1213                          unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
1214 {
1215         pte_t *pte;
1216
1217         pte = pte_offset_kernel(pmd, start);
1218
1219         pgprot = pgprot_clear_protnone_bits(pgprot);
1220
1221         while (num_pages-- && start < end) {
1222                 set_pte(pte, pfn_pte(cpa->pfn, pgprot));
1223
1224                 start    += PAGE_SIZE;
1225                 cpa->pfn++;
1226                 pte++;
1227         }
1228 }
1229
1230 static long populate_pmd(struct cpa_data *cpa,
1231                          unsigned long start, unsigned long end,
1232                          unsigned num_pages, pud_t *pud, pgprot_t pgprot)
1233 {
1234         long cur_pages = 0;
1235         pmd_t *pmd;
1236         pgprot_t pmd_pgprot;
1237
1238         /*
1239          * Not on a 2M boundary?
1240          */
1241         if (start & (PMD_SIZE - 1)) {
1242                 unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
1243                 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
1244
1245                 pre_end   = min_t(unsigned long, pre_end, next_page);
1246                 cur_pages = (pre_end - start) >> PAGE_SHIFT;
1247                 cur_pages = min_t(unsigned int, num_pages, cur_pages);
1248
1249                 /*
1250                  * Need a PTE page?
1251                  */
1252                 pmd = pmd_offset(pud, start);
1253                 if (pmd_none(*pmd))
1254                         if (alloc_pte_page(pmd))
1255                                 return -1;
1256
1257                 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
1258
1259                 start = pre_end;
1260         }
1261
1262         /*
1263          * We mapped them all?
1264          */
1265         if (num_pages == cur_pages)
1266                 return cur_pages;
1267
1268         pmd_pgprot = pgprot_4k_2_large(pgprot);
1269
1270         while (end - start >= PMD_SIZE) {
1271
1272                 /*
1273                  * We cannot use a 1G page so allocate a PMD page if needed.
1274                  */
1275                 if (pud_none(*pud))
1276                         if (alloc_pmd_page(pud))
1277                                 return -1;
1278
1279                 pmd = pmd_offset(pud, start);
1280
1281                 set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn,
1282                                         canon_pgprot(pmd_pgprot))));
1283
1284                 start     += PMD_SIZE;
1285                 cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
1286                 cur_pages += PMD_SIZE >> PAGE_SHIFT;
1287         }
1288
1289         /*
1290          * Map trailing 4K pages.
1291          */
1292         if (start < end) {
1293                 pmd = pmd_offset(pud, start);
1294                 if (pmd_none(*pmd))
1295                         if (alloc_pte_page(pmd))
1296                                 return -1;
1297
1298                 populate_pte(cpa, start, end, num_pages - cur_pages,
1299                              pmd, pgprot);
1300         }
1301         return num_pages;
1302 }
1303
1304 static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
1305                         pgprot_t pgprot)
1306 {
1307         pud_t *pud;
1308         unsigned long end;
1309         long cur_pages = 0;
1310         pgprot_t pud_pgprot;
1311
1312         end = start + (cpa->numpages << PAGE_SHIFT);
1313
1314         /*
1315          * Not on a Gb page boundary? => map everything up to it with
1316          * smaller pages.
1317          */
1318         if (start & (PUD_SIZE - 1)) {
1319                 unsigned long pre_end;
1320                 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1321
1322                 pre_end   = min_t(unsigned long, end, next_page);
1323                 cur_pages = (pre_end - start) >> PAGE_SHIFT;
1324                 cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
1325
1326                 pud = pud_offset(p4d, start);
1327
1328                 /*
1329                  * Need a PMD page?
1330                  */
1331                 if (pud_none(*pud))
1332                         if (alloc_pmd_page(pud))
1333                                 return -1;
1334
1335                 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
1336                                          pud, pgprot);
1337                 if (cur_pages < 0)
1338                         return cur_pages;
1339
1340                 start = pre_end;
1341         }
1342
1343         /* We mapped them all? */
1344         if (cpa->numpages == cur_pages)
1345                 return cur_pages;
1346
1347         pud = pud_offset(p4d, start);
1348         pud_pgprot = pgprot_4k_2_large(pgprot);
1349
1350         /*
1351          * Map everything starting from the Gb boundary, possibly with 1G pages
1352          */
1353         while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) {
1354                 set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn,
1355                                    canon_pgprot(pud_pgprot))));
1356
1357                 start     += PUD_SIZE;
1358                 cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
1359                 cur_pages += PUD_SIZE >> PAGE_SHIFT;
1360                 pud++;
1361         }
1362
1363         /* Map trailing leftover */
1364         if (start < end) {
1365                 long tmp;
1366
1367                 pud = pud_offset(p4d, start);
1368                 if (pud_none(*pud))
1369                         if (alloc_pmd_page(pud))
1370                                 return -1;
1371
1372                 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
1373                                    pud, pgprot);
1374                 if (tmp < 0)
1375                         return cur_pages;
1376
1377                 cur_pages += tmp;
1378         }
1379         return cur_pages;
1380 }
1381
1382 /*
1383  * Restrictions for kernel page table do not necessarily apply when mapping in
1384  * an alternate PGD.
1385  */
1386 static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
1387 {
1388         pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
1389         pud_t *pud = NULL;      /* shut up gcc */
1390         p4d_t *p4d;
1391         pgd_t *pgd_entry;
1392         long ret;
1393
1394         pgd_entry = cpa->pgd + pgd_index(addr);
1395
1396         if (pgd_none(*pgd_entry)) {
1397                 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
1398                 if (!p4d)
1399                         return -1;
1400
1401                 set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE));
1402         }
1403
1404         /*
1405          * Allocate a PUD page and hand it down for mapping.
1406          */
1407         p4d = p4d_offset(pgd_entry, addr);
1408         if (p4d_none(*p4d)) {
1409                 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
1410                 if (!pud)
1411                         return -1;
1412
1413                 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
1414         }
1415
1416         pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
1417         pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);
1418
1419         ret = populate_pud(cpa, addr, p4d, pgprot);
1420         if (ret < 0) {
1421                 /*
1422                  * Leave the PUD page in place in case some other CPU or thread
1423                  * already found it, but remove any useless entries we just
1424                  * added to it.
1425                  */
1426                 unmap_pud_range(p4d, addr,
1427                                 addr + (cpa->numpages << PAGE_SHIFT));
1428                 return ret;
1429         }
1430
1431         cpa->numpages = ret;
1432         return 0;
1433 }
1434
1435 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
1436                                int primary)
1437 {
1438         if (cpa->pgd) {
1439                 /*
1440                  * Right now, we only execute this code path when mapping
1441                  * the EFI virtual memory map regions, no other users
1442                  * provide a ->pgd value. This may change in the future.
1443                  */
1444                 return populate_pgd(cpa, vaddr);
1445         }
1446
1447         /*
1448          * Ignore all non primary paths.
1449          */
1450         if (!primary) {
1451                 cpa->numpages = 1;
1452                 return 0;
1453         }
1454
1455         /*
1456          * Ignore the NULL PTE for kernel identity mapping, as it is expected
1457          * to have holes.
1458          * Also set numpages to '1' indicating that we processed cpa req for
1459          * one virtual address page and its pfn. TBD: numpages can be set based
1460          * on the initial value and the level returned by lookup_address().
1461          */
1462         if (within(vaddr, PAGE_OFFSET,
1463                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
1464                 cpa->numpages = 1;
1465                 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
1466                 return 0;
1467
1468         } else if (__cpa_pfn_in_highmap(cpa->pfn)) {
1469                 /* Faults in the highmap are OK, so do not warn: */
1470                 return -EFAULT;
1471         } else {
1472                 WARN(1, KERN_WARNING "CPA: called for zero pte. "
1473                         "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
1474                         *cpa->vaddr);
1475
1476                 return -EFAULT;
1477         }
1478 }
1479
1480 static int __change_page_attr(struct cpa_data *cpa, int primary)
1481 {
1482         unsigned long address;
1483         int do_split, err;
1484         unsigned int level;
1485         pte_t *kpte, old_pte;
1486
1487         address = __cpa_addr(cpa, cpa->curpage);
1488 repeat:
1489         kpte = _lookup_address_cpa(cpa, address, &level);
1490         if (!kpte)
1491                 return __cpa_process_fault(cpa, address, primary);
1492
1493         old_pte = *kpte;
1494         if (pte_none(old_pte))
1495                 return __cpa_process_fault(cpa, address, primary);
1496
1497         if (level == PG_LEVEL_4K) {
1498                 pte_t new_pte;
1499                 pgprot_t new_prot = pte_pgprot(old_pte);
1500                 unsigned long pfn = pte_pfn(old_pte);
1501
1502                 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
1503                 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
1504
1505                 cpa_inc_4k_install();
1506                 new_prot = static_protections(new_prot, address, pfn, 1,
1507                                               CPA_PROTECT);
1508
1509                 new_prot = pgprot_clear_protnone_bits(new_prot);
1510
1511                 /*
1512                  * We need to keep the pfn from the existing PTE,
1513                  * after all we're only going to change it's attributes
1514                  * not the memory it points to
1515                  */
1516                 new_pte = pfn_pte(pfn, new_prot);
1517                 cpa->pfn = pfn;
1518                 /*
1519                  * Do we really change anything ?
1520                  */
1521                 if (pte_val(old_pte) != pte_val(new_pte)) {
1522                         set_pte_atomic(kpte, new_pte);
1523                         cpa->flags |= CPA_FLUSHTLB;
1524                 }
1525                 cpa->numpages = 1;
1526                 return 0;
1527         }
1528
1529         /*
1530          * Check, whether we can keep the large page intact
1531          * and just change the pte:
1532          */
1533         do_split = should_split_large_page(kpte, address, cpa);
1534         /*
1535          * When the range fits into the existing large page,
1536          * return. cp->numpages and cpa->tlbflush have been updated in
1537          * try_large_page:
1538          */
1539         if (do_split <= 0)
1540                 return do_split;
1541
1542         /*
1543          * We have to split the large page:
1544          */
1545         err = split_large_page(cpa, kpte, address);
1546         if (!err)
1547                 goto repeat;
1548
1549         return err;
1550 }
1551
1552 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1553
1554 static int cpa_process_alias(struct cpa_data *cpa)
1555 {
1556         struct cpa_data alias_cpa;
1557         unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
1558         unsigned long vaddr;
1559         int ret;
1560
1561         if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
1562                 return 0;
1563
1564         /*
1565          * No need to redo, when the primary call touched the direct
1566          * mapping already:
1567          */
1568         vaddr = __cpa_addr(cpa, cpa->curpage);
1569         if (!(within(vaddr, PAGE_OFFSET,
1570                     PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
1571
1572                 alias_cpa = *cpa;
1573                 alias_cpa.vaddr = &laddr;
1574                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1575                 alias_cpa.curpage = 0;
1576
1577                 ret = __change_page_attr_set_clr(&alias_cpa, 0);
1578                 if (ret)
1579                         return ret;
1580         }
1581
1582 #ifdef CONFIG_X86_64
1583         /*
1584          * If the primary call didn't touch the high mapping already
1585          * and the physical address is inside the kernel map, we need
1586          * to touch the high mapped kernel as well:
1587          */
1588         if (!within(vaddr, (unsigned long)_text, _brk_end) &&
1589             __cpa_pfn_in_highmap(cpa->pfn)) {
1590                 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
1591                                                __START_KERNEL_map - phys_base;
1592                 alias_cpa = *cpa;
1593                 alias_cpa.vaddr = &temp_cpa_vaddr;
1594                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1595                 alias_cpa.curpage = 0;
1596
1597                 /*
1598                  * The high mapping range is imprecise, so ignore the
1599                  * return value.
1600                  */
1601                 __change_page_attr_set_clr(&alias_cpa, 0);
1602         }
1603 #endif
1604
1605         return 0;
1606 }
1607
1608 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
1609 {
1610         unsigned long numpages = cpa->numpages;
1611         unsigned long rempages = numpages;
1612         int ret = 0;
1613
1614         while (rempages) {
1615                 /*
1616                  * Store the remaining nr of pages for the large page
1617                  * preservation check.
1618                  */
1619                 cpa->numpages = rempages;
1620                 /* for array changes, we can't use large page */
1621                 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
1622                         cpa->numpages = 1;
1623
1624                 if (!debug_pagealloc_enabled())
1625                         spin_lock(&cpa_lock);
1626                 ret = __change_page_attr(cpa, checkalias);
1627                 if (!debug_pagealloc_enabled())
1628                         spin_unlock(&cpa_lock);
1629                 if (ret)
1630                         goto out;
1631
1632                 if (checkalias) {
1633                         ret = cpa_process_alias(cpa);
1634                         if (ret)
1635                                 goto out;
1636                 }
1637
1638                 /*
1639                  * Adjust the number of pages with the result of the
1640                  * CPA operation. Either a large page has been
1641                  * preserved or a single page update happened.
1642                  */
1643                 BUG_ON(cpa->numpages > rempages || !cpa->numpages);
1644                 rempages -= cpa->numpages;
1645                 cpa->curpage += cpa->numpages;
1646         }
1647
1648 out:
1649         /* Restore the original numpages */
1650         cpa->numpages = numpages;
1651         return ret;
1652 }
1653
1654 static int change_page_attr_set_clr(unsigned long *addr, int numpages,
1655                                     pgprot_t mask_set, pgprot_t mask_clr,
1656                                     int force_split, int in_flag,
1657                                     struct page **pages)
1658 {
1659         struct cpa_data cpa;
1660         int ret, cache, checkalias;
1661
1662         memset(&cpa, 0, sizeof(cpa));
1663
1664         /*
1665          * Check, if we are requested to set a not supported
1666          * feature.  Clearing non-supported features is OK.
1667          */
1668         mask_set = canon_pgprot(mask_set);
1669
1670         if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
1671                 return 0;
1672
1673         /* Ensure we are PAGE_SIZE aligned */
1674         if (in_flag & CPA_ARRAY) {
1675                 int i;
1676                 for (i = 0; i < numpages; i++) {
1677                         if (addr[i] & ~PAGE_MASK) {
1678                                 addr[i] &= PAGE_MASK;
1679                                 WARN_ON_ONCE(1);
1680                         }
1681                 }
1682         } else if (!(in_flag & CPA_PAGES_ARRAY)) {
1683                 /*
1684                  * in_flag of CPA_PAGES_ARRAY implies it is aligned.
1685                  * No need to check in that case
1686                  */
1687                 if (*addr & ~PAGE_MASK) {
1688                         *addr &= PAGE_MASK;
1689                         /*
1690                          * People should not be passing in unaligned addresses:
1691                          */
1692                         WARN_ON_ONCE(1);
1693                 }
1694         }
1695
1696         /* Must avoid aliasing mappings in the highmem code */
1697         kmap_flush_unused();
1698
1699         vm_unmap_aliases();
1700
1701         cpa.vaddr = addr;
1702         cpa.pages = pages;
1703         cpa.numpages = numpages;
1704         cpa.mask_set = mask_set;
1705         cpa.mask_clr = mask_clr;
1706         cpa.flags = 0;
1707         cpa.curpage = 0;
1708         cpa.force_split = force_split;
1709
1710         if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
1711                 cpa.flags |= in_flag;
1712
1713         /* No alias checking for _NX bit modifications */
1714         checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
1715         /* Has caller explicitly disabled alias checking? */
1716         if (in_flag & CPA_NO_CHECK_ALIAS)
1717                 checkalias = 0;
1718
1719         ret = __change_page_attr_set_clr(&cpa, checkalias);
1720
1721         /*
1722          * Check whether we really changed something:
1723          */
1724         if (!(cpa.flags & CPA_FLUSHTLB))
1725                 goto out;
1726
1727         /*
1728          * No need to flush, when we did not set any of the caching
1729          * attributes:
1730          */
1731         cache = !!pgprot2cachemode(mask_set);
1732
1733         /*
1734          * On error; flush everything to be sure.
1735          */
1736         if (ret) {
1737                 cpa_flush_all(cache);
1738                 goto out;
1739         }
1740
1741         cpa_flush(&cpa, cache);
1742 out:
1743         return ret;
1744 }
1745
1746 static inline int change_page_attr_set(unsigned long *addr, int numpages,
1747                                        pgprot_t mask, int array)
1748 {
1749         return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
1750                 (array ? CPA_ARRAY : 0), NULL);
1751 }
1752
1753 static inline int change_page_attr_clear(unsigned long *addr, int numpages,
1754                                          pgprot_t mask, int array)
1755 {
1756         return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
1757                 (array ? CPA_ARRAY : 0), NULL);
1758 }
1759
1760 static inline int cpa_set_pages_array(struct page **pages, int numpages,
1761                                        pgprot_t mask)
1762 {
1763         return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
1764                 CPA_PAGES_ARRAY, pages);
1765 }
1766
1767 static inline int cpa_clear_pages_array(struct page **pages, int numpages,
1768                                          pgprot_t mask)
1769 {
1770         return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
1771                 CPA_PAGES_ARRAY, pages);
1772 }
1773
1774 int _set_memory_uc(unsigned long addr, int numpages)
1775 {
1776         /*
1777          * for now UC MINUS. see comments in ioremap_nocache()
1778          * If you really need strong UC use ioremap_uc(), but note
1779          * that you cannot override IO areas with set_memory_*() as
1780          * these helpers cannot work with IO memory.
1781          */
1782         return change_page_attr_set(&addr, numpages,
1783                                     cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1784                                     0);
1785 }
1786
1787 int set_memory_uc(unsigned long addr, int numpages)
1788 {
1789         int ret;
1790
1791         /*
1792          * for now UC MINUS. see comments in ioremap_nocache()
1793          */
1794         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1795                               _PAGE_CACHE_MODE_UC_MINUS, NULL);
1796         if (ret)
1797                 goto out_err;
1798
1799         ret = _set_memory_uc(addr, numpages);
1800         if (ret)
1801                 goto out_free;
1802
1803         return 0;
1804
1805 out_free:
1806         free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1807 out_err:
1808         return ret;
1809 }
1810 EXPORT_SYMBOL(set_memory_uc);
1811
1812 static int _set_memory_array(unsigned long *addr, int numpages,
1813                 enum page_cache_mode new_type)
1814 {
1815         enum page_cache_mode set_type;
1816         int i, j;
1817         int ret;
1818
1819         for (i = 0; i < numpages; i++) {
1820                 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
1821                                         new_type, NULL);
1822                 if (ret)
1823                         goto out_free;
1824         }
1825
1826         /* If WC, set to UC- first and then WC */
1827         set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
1828                                 _PAGE_CACHE_MODE_UC_MINUS : new_type;
1829
1830         ret = change_page_attr_set(addr, numpages,
1831                                    cachemode2pgprot(set_type), 1);
1832
1833         if (!ret && new_type == _PAGE_CACHE_MODE_WC)
1834                 ret = change_page_attr_set_clr(addr, numpages,
1835                                                cachemode2pgprot(
1836                                                 _PAGE_CACHE_MODE_WC),
1837                                                __pgprot(_PAGE_CACHE_MASK),
1838                                                0, CPA_ARRAY, NULL);
1839         if (ret)
1840                 goto out_free;
1841
1842         return 0;
1843
1844 out_free:
1845         for (j = 0; j < i; j++)
1846                 free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);
1847
1848         return ret;
1849 }
1850
1851 int set_memory_array_uc(unsigned long *addr, int numpages)
1852 {
1853         return _set_memory_array(addr, numpages, _PAGE_CACHE_MODE_UC_MINUS);
1854 }
1855 EXPORT_SYMBOL(set_memory_array_uc);
1856
1857 int set_memory_array_wc(unsigned long *addr, int numpages)
1858 {
1859         return _set_memory_array(addr, numpages, _PAGE_CACHE_MODE_WC);
1860 }
1861 EXPORT_SYMBOL(set_memory_array_wc);
1862
1863 int set_memory_array_wt(unsigned long *addr, int numpages)
1864 {
1865         return _set_memory_array(addr, numpages, _PAGE_CACHE_MODE_WT);
1866 }
1867 EXPORT_SYMBOL_GPL(set_memory_array_wt);
1868
1869 int _set_memory_wc(unsigned long addr, int numpages)
1870 {
1871         int ret;
1872
1873         ret = change_page_attr_set(&addr, numpages,
1874                                    cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1875                                    0);
1876         if (!ret) {
1877                 ret = change_page_attr_set_clr(&addr, numpages,
1878                                                cachemode2pgprot(_PAGE_CACHE_MODE_WC),
1879                                                __pgprot(_PAGE_CACHE_MASK),
1880                                                0, 0, NULL);
1881         }
1882         return ret;
1883 }
1884
1885 int set_memory_wc(unsigned long addr, int numpages)
1886 {
1887         int ret;
1888
1889         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1890                 _PAGE_CACHE_MODE_WC, NULL);
1891         if (ret)
1892                 return ret;
1893
1894         ret = _set_memory_wc(addr, numpages);
1895         if (ret)
1896                 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1897
1898         return ret;
1899 }
1900 EXPORT_SYMBOL(set_memory_wc);
1901
1902 int _set_memory_wt(unsigned long addr, int numpages)
1903 {
1904         return change_page_attr_set(&addr, numpages,
1905                                     cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
1906 }
1907
1908 int set_memory_wt(unsigned long addr, int numpages)
1909 {
1910         int ret;
1911
1912         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1913                               _PAGE_CACHE_MODE_WT, NULL);
1914         if (ret)
1915                 return ret;
1916
1917         ret = _set_memory_wt(addr, numpages);
1918         if (ret)
1919                 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1920
1921         return ret;
1922 }
1923 EXPORT_SYMBOL_GPL(set_memory_wt);
1924
1925 int _set_memory_wb(unsigned long addr, int numpages)
1926 {
1927         /* WB cache mode is hard wired to all cache attribute bits being 0 */
1928         return change_page_attr_clear(&addr, numpages,
1929                                       __pgprot(_PAGE_CACHE_MASK), 0);
1930 }
1931
1932 int set_memory_wb(unsigned long addr, int numpages)
1933 {
1934         int ret;
1935
1936         ret = _set_memory_wb(addr, numpages);
1937         if (ret)
1938                 return ret;
1939
1940         free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1941         return 0;
1942 }
1943 EXPORT_SYMBOL(set_memory_wb);
1944
1945 int set_memory_array_wb(unsigned long *addr, int numpages)
1946 {
1947         int i;
1948         int ret;
1949
1950         /* WB cache mode is hard wired to all cache attribute bits being 0 */
1951         ret = change_page_attr_clear(addr, numpages,
1952                                       __pgprot(_PAGE_CACHE_MASK), 1);
1953         if (ret)
1954                 return ret;
1955
1956         for (i = 0; i < numpages; i++)
1957                 free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
1958
1959         return 0;
1960 }
1961 EXPORT_SYMBOL(set_memory_array_wb);
1962
1963 int set_memory_x(unsigned long addr, int numpages)
1964 {
1965         if (!(__supported_pte_mask & _PAGE_NX))
1966                 return 0;
1967
1968         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
1969 }
1970 EXPORT_SYMBOL(set_memory_x);
1971
1972 int set_memory_nx(unsigned long addr, int numpages)
1973 {
1974         if (!(__supported_pte_mask & _PAGE_NX))
1975                 return 0;
1976
1977         return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
1978 }
1979 EXPORT_SYMBOL(set_memory_nx);
1980
1981 int set_memory_ro(unsigned long addr, int numpages)
1982 {
1983         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
1984 }
1985
1986 int set_memory_rw(unsigned long addr, int numpages)
1987 {
1988         return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
1989 }
1990
1991 int set_memory_np(unsigned long addr, int numpages)
1992 {
1993         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
1994 }
1995
1996 int set_memory_np_noalias(unsigned long addr, int numpages)
1997 {
1998         int cpa_flags = CPA_NO_CHECK_ALIAS;
1999
2000         return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
2001                                         __pgprot(_PAGE_PRESENT), 0,
2002                                         cpa_flags, NULL);
2003 }
2004
2005 int set_memory_4k(unsigned long addr, int numpages)
2006 {
2007         return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
2008                                         __pgprot(0), 1, 0, NULL);
2009 }
2010
2011 int set_memory_nonglobal(unsigned long addr, int numpages)
2012 {
2013         return change_page_attr_clear(&addr, numpages,
2014                                       __pgprot(_PAGE_GLOBAL), 0);
2015 }
2016
2017 int set_memory_global(unsigned long addr, int numpages)
2018 {
2019         return change_page_attr_set(&addr, numpages,
2020                                     __pgprot(_PAGE_GLOBAL), 0);
2021 }
2022
2023 static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
2024 {
2025         struct cpa_data cpa;
2026         int ret;
2027
2028         /* Nothing to do if memory encryption is not active */
2029         if (!mem_encrypt_active())
2030                 return 0;
2031
2032         /* Should not be working on unaligned addresses */
2033         if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
2034                 addr &= PAGE_MASK;
2035
2036         memset(&cpa, 0, sizeof(cpa));
2037         cpa.vaddr = &addr;
2038         cpa.numpages = numpages;
2039         cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
2040         cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
2041         cpa.pgd = init_mm.pgd;
2042
2043         /* Must avoid aliasing mappings in the highmem code */
2044         kmap_flush_unused();
2045         vm_unmap_aliases();
2046
2047         /*
2048          * Before changing the encryption attribute, we need to flush caches.
2049          */
2050         cpa_flush(&cpa, 1);
2051
2052         ret = __change_page_attr_set_clr(&cpa, 1);
2053
2054         /*
2055          * After changing the encryption attribute, we need to flush TLBs again
2056          * in case any speculative TLB caching occurred (but no need to flush
2057          * caches again).  We could just use cpa_flush_all(), but in case TLB
2058          * flushing gets optimized in the cpa_flush() path use the same logic
2059          * as above.
2060          */
2061         cpa_flush(&cpa, 0);
2062
2063         return ret;
2064 }
2065
2066 int set_memory_encrypted(unsigned long addr, int numpages)
2067 {
2068         return __set_memory_enc_dec(addr, numpages, true);
2069 }
2070 EXPORT_SYMBOL_GPL(set_memory_encrypted);
2071
2072 int set_memory_decrypted(unsigned long addr, int numpages)
2073 {
2074         return __set_memory_enc_dec(addr, numpages, false);
2075 }
2076 EXPORT_SYMBOL_GPL(set_memory_decrypted);
2077
2078 int set_pages_uc(struct page *page, int numpages)
2079 {
2080         unsigned long addr = (unsigned long)page_address(page);
2081
2082         return set_memory_uc(addr, numpages);
2083 }
2084 EXPORT_SYMBOL(set_pages_uc);
2085
2086 static int _set_pages_array(struct page **pages, int numpages,
2087                 enum page_cache_mode new_type)
2088 {
2089         unsigned long start;
2090         unsigned long end;
2091         enum page_cache_mode set_type;
2092         int i;
2093         int free_idx;
2094         int ret;
2095
2096         for (i = 0; i < numpages; i++) {
2097                 if (PageHighMem(pages[i]))
2098                         continue;
2099                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2100                 end = start + PAGE_SIZE;
2101                 if (reserve_memtype(start, end, new_type, NULL))
2102                         goto err_out;
2103         }
2104
2105         /* If WC, set to UC- first and then WC */
2106         set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
2107                                 _PAGE_CACHE_MODE_UC_MINUS : new_type;
2108
2109         ret = cpa_set_pages_array(pages, numpages,
2110                                   cachemode2pgprot(set_type));
2111         if (!ret && new_type == _PAGE_CACHE_MODE_WC)
2112                 ret = change_page_attr_set_clr(NULL, numpages,
2113                                                cachemode2pgprot(
2114                                                 _PAGE_CACHE_MODE_WC),
2115                                                __pgprot(_PAGE_CACHE_MASK),
2116                                                0, CPA_PAGES_ARRAY, pages);
2117         if (ret)
2118                 goto err_out;
2119         return 0; /* Success */
2120 err_out:
2121         free_idx = i;
2122         for (i = 0; i < free_idx; i++) {
2123                 if (PageHighMem(pages[i]))
2124                         continue;
2125                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2126                 end = start + PAGE_SIZE;
2127                 free_memtype(start, end);
2128         }
2129         return -EINVAL;
2130 }
2131
2132 int set_pages_array_uc(struct page **pages, int numpages)
2133 {
2134         return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_UC_MINUS);
2135 }
2136 EXPORT_SYMBOL(set_pages_array_uc);
2137
2138 int set_pages_array_wc(struct page **pages, int numpages)
2139 {
2140         return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WC);
2141 }
2142 EXPORT_SYMBOL(set_pages_array_wc);
2143
2144 int set_pages_array_wt(struct page **pages, int numpages)
2145 {
2146         return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WT);
2147 }
2148 EXPORT_SYMBOL_GPL(set_pages_array_wt);
2149
2150 int set_pages_wb(struct page *page, int numpages)
2151 {
2152         unsigned long addr = (unsigned long)page_address(page);
2153
2154         return set_memory_wb(addr, numpages);
2155 }
2156 EXPORT_SYMBOL(set_pages_wb);
2157
2158 int set_pages_array_wb(struct page **pages, int numpages)
2159 {
2160         int retval;
2161         unsigned long start;
2162         unsigned long end;
2163         int i;
2164
2165         /* WB cache mode is hard wired to all cache attribute bits being 0 */
2166         retval = cpa_clear_pages_array(pages, numpages,
2167                         __pgprot(_PAGE_CACHE_MASK));
2168         if (retval)
2169                 return retval;
2170
2171         for (i = 0; i < numpages; i++) {
2172                 if (PageHighMem(pages[i]))
2173                         continue;
2174                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2175                 end = start + PAGE_SIZE;
2176                 free_memtype(start, end);
2177         }
2178
2179         return 0;
2180 }
2181 EXPORT_SYMBOL(set_pages_array_wb);
2182
2183 int set_pages_x(struct page *page, int numpages)
2184 {
2185         unsigned long addr = (unsigned long)page_address(page);
2186
2187         return set_memory_x(addr, numpages);
2188 }
2189 EXPORT_SYMBOL(set_pages_x);
2190
2191 int set_pages_nx(struct page *page, int numpages)
2192 {
2193         unsigned long addr = (unsigned long)page_address(page);
2194
2195         return set_memory_nx(addr, numpages);
2196 }
2197 EXPORT_SYMBOL(set_pages_nx);
2198
2199 int set_pages_ro(struct page *page, int numpages)
2200 {
2201         unsigned long addr = (unsigned long)page_address(page);
2202
2203         return set_memory_ro(addr, numpages);
2204 }
2205
2206 int set_pages_rw(struct page *page, int numpages)
2207 {
2208         unsigned long addr = (unsigned long)page_address(page);
2209
2210         return set_memory_rw(addr, numpages);
2211 }
2212
2213 static int __set_pages_p(struct page *page, int numpages)
2214 {
2215         unsigned long tempaddr = (unsigned long) page_address(page);
2216         struct cpa_data cpa = { .vaddr = &tempaddr,
2217                                 .pgd = NULL,
2218                                 .numpages = numpages,
2219                                 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2220                                 .mask_clr = __pgprot(0),
2221                                 .flags = 0};
2222
2223         /*
2224          * No alias checking needed for setting present flag. otherwise,
2225          * we may need to break large pages for 64-bit kernel text
2226          * mappings (this adds to complexity if we want to do this from
2227          * atomic context especially). Let's keep it simple!
2228          */
2229         return __change_page_attr_set_clr(&cpa, 0);
2230 }
2231
2232 static int __set_pages_np(struct page *page, int numpages)
2233 {
2234         unsigned long tempaddr = (unsigned long) page_address(page);
2235         struct cpa_data cpa = { .vaddr = &tempaddr,
2236                                 .pgd = NULL,
2237                                 .numpages = numpages,
2238                                 .mask_set = __pgprot(0),
2239                                 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2240                                 .flags = 0};
2241
2242         /*
2243          * No alias checking needed for setting not present flag. otherwise,
2244          * we may need to break large pages for 64-bit kernel text
2245          * mappings (this adds to complexity if we want to do this from
2246          * atomic context especially). Let's keep it simple!
2247          */
2248         return __change_page_attr_set_clr(&cpa, 0);
2249 }
2250
2251 int set_direct_map_invalid_noflush(struct page *page)
2252 {
2253         return __set_pages_np(page, 1);
2254 }
2255
2256 int set_direct_map_default_noflush(struct page *page)
2257 {
2258         return __set_pages_p(page, 1);
2259 }
2260
2261 void __kernel_map_pages(struct page *page, int numpages, int enable)
2262 {
2263         if (PageHighMem(page))
2264                 return;
2265         if (!enable) {
2266                 debug_check_no_locks_freed(page_address(page),
2267                                            numpages * PAGE_SIZE);
2268         }
2269
2270         /*
2271          * The return value is ignored as the calls cannot fail.
2272          * Large pages for identity mappings are not used at boot time
2273          * and hence no memory allocations during large page split.
2274          */
2275         if (enable)
2276                 __set_pages_p(page, numpages);
2277         else
2278                 __set_pages_np(page, numpages);
2279
2280         /*
2281          * We should perform an IPI and flush all tlbs,
2282          * but that can deadlock->flush only current cpu.
2283          * Preemption needs to be disabled around __flush_tlb_all() due to
2284          * CR3 reload in __native_flush_tlb().
2285          */
2286         preempt_disable();
2287         __flush_tlb_all();
2288         preempt_enable();
2289
2290         arch_flush_lazy_mmu_mode();
2291 }
2292
2293 #ifdef CONFIG_HIBERNATION
2294 bool kernel_page_present(struct page *page)
2295 {
2296         unsigned int level;
2297         pte_t *pte;
2298
2299         if (PageHighMem(page))
2300                 return false;
2301
2302         pte = lookup_address((unsigned long)page_address(page), &level);
2303         return (pte_val(*pte) & _PAGE_PRESENT);
2304 }
2305 #endif /* CONFIG_HIBERNATION */
2306
2307 int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
2308                                    unsigned numpages, unsigned long page_flags)
2309 {
2310         int retval = -EINVAL;
2311
2312         struct cpa_data cpa = {
2313                 .vaddr = &address,
2314                 .pfn = pfn,
2315                 .pgd = pgd,
2316                 .numpages = numpages,
2317                 .mask_set = __pgprot(0),
2318                 .mask_clr = __pgprot(0),
2319                 .flags = 0,
2320         };
2321
2322         WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP");
2323
2324         if (!(__supported_pte_mask & _PAGE_NX))
2325                 goto out;
2326
2327         if (!(page_flags & _PAGE_NX))
2328                 cpa.mask_clr = __pgprot(_PAGE_NX);
2329
2330         if (!(page_flags & _PAGE_RW))
2331                 cpa.mask_clr = __pgprot(_PAGE_RW);
2332
2333         if (!(page_flags & _PAGE_ENC))
2334                 cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);
2335
2336         cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
2337
2338         retval = __change_page_attr_set_clr(&cpa, 0);
2339         __flush_tlb_all();
2340
2341 out:
2342         return retval;
2343 }
2344
2345 /*
2346  * __flush_tlb_all() flushes mappings only on current CPU and hence this
2347  * function shouldn't be used in an SMP environment. Presently, it's used only
2348  * during boot (way before smp_init()) by EFI subsystem and hence is ok.
2349  */
2350 int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
2351                                      unsigned long numpages)
2352 {
2353         int retval;
2354
2355         /*
2356          * The typical sequence for unmapping is to find a pte through
2357          * lookup_address_in_pgd() (ideally, it should never return NULL because
2358          * the address is already mapped) and change it's protections. As pfn is
2359          * the *target* of a mapping, it's not useful while unmapping.
2360          */
2361         struct cpa_data cpa = {
2362                 .vaddr          = &address,
2363                 .pfn            = 0,
2364                 .pgd            = pgd,
2365                 .numpages       = numpages,
2366                 .mask_set       = __pgprot(0),
2367                 .mask_clr       = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2368                 .flags          = 0,
2369         };
2370
2371         WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP");
2372
2373         retval = __change_page_attr_set_clr(&cpa, 0);
2374         __flush_tlb_all();
2375
2376         return retval;
2377 }
2378
2379 /*
2380  * The testcases use internal knowledge of the implementation that shouldn't
2381  * be exposed to the rest of the kernel. Include these directly here.
2382  */
2383 #ifdef CONFIG_CPA_DEBUG
2384 #include "pageattr-test.c"
2385 #endif