kmemcheck: stop using GFP_NOTRACK and SLAB_NOTRACK
[sfrench/cifs-2.6.git] / arch / x86 / mm / pageattr.c
1 /*
2  * Copyright 2002 Andi Kleen, SuSE Labs.
3  * Thanks to Ben LaHaise for precious feedback.
4  */
5 #include <linux/highmem.h>
6 #include <linux/bootmem.h>
7 #include <linux/sched.h>
8 #include <linux/mm.h>
9 #include <linux/interrupt.h>
10 #include <linux/seq_file.h>
11 #include <linux/debugfs.h>
12 #include <linux/pfn.h>
13 #include <linux/percpu.h>
14 #include <linux/gfp.h>
15 #include <linux/pci.h>
16 #include <linux/vmalloc.h>
17
18 #include <asm/e820/api.h>
19 #include <asm/processor.h>
20 #include <asm/tlbflush.h>
21 #include <asm/sections.h>
22 #include <asm/setup.h>
23 #include <linux/uaccess.h>
24 #include <asm/pgalloc.h>
25 #include <asm/proto.h>
26 #include <asm/pat.h>
27 #include <asm/set_memory.h>
28
29 /*
30  * The current flushing context - we pass it instead of 5 arguments:
31  */
32 struct cpa_data {
33         unsigned long   *vaddr;
34         pgd_t           *pgd;
35         pgprot_t        mask_set;
36         pgprot_t        mask_clr;
37         unsigned long   numpages;
38         int             flags;
39         unsigned long   pfn;
40         unsigned        force_split : 1;
41         int             curpage;
42         struct page     **pages;
43 };
44
45 /*
46  * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
47  * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
48  * entries change the page attribute in parallel to some other cpu
49  * splitting a large page entry along with changing the attribute.
50  */
51 static DEFINE_SPINLOCK(cpa_lock);
52
53 #define CPA_FLUSHTLB 1
54 #define CPA_ARRAY 2
55 #define CPA_PAGES_ARRAY 4
56
57 #ifdef CONFIG_PROC_FS
58 static unsigned long direct_pages_count[PG_LEVEL_NUM];
59
60 void update_page_count(int level, unsigned long pages)
61 {
62         /* Protect against CPA */
63         spin_lock(&pgd_lock);
64         direct_pages_count[level] += pages;
65         spin_unlock(&pgd_lock);
66 }
67
68 static void split_page_count(int level)
69 {
70         if (direct_pages_count[level] == 0)
71                 return;
72
73         direct_pages_count[level]--;
74         direct_pages_count[level - 1] += PTRS_PER_PTE;
75 }
76
77 void arch_report_meminfo(struct seq_file *m)
78 {
79         seq_printf(m, "DirectMap4k:    %8lu kB\n",
80                         direct_pages_count[PG_LEVEL_4K] << 2);
81 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
82         seq_printf(m, "DirectMap2M:    %8lu kB\n",
83                         direct_pages_count[PG_LEVEL_2M] << 11);
84 #else
85         seq_printf(m, "DirectMap4M:    %8lu kB\n",
86                         direct_pages_count[PG_LEVEL_2M] << 12);
87 #endif
88         if (direct_gbpages)
89                 seq_printf(m, "DirectMap1G:    %8lu kB\n",
90                         direct_pages_count[PG_LEVEL_1G] << 20);
91 }
92 #else
93 static inline void split_page_count(int level) { }
94 #endif
95
96 #ifdef CONFIG_X86_64
97
98 static inline unsigned long highmap_start_pfn(void)
99 {
100         return __pa_symbol(_text) >> PAGE_SHIFT;
101 }
102
103 static inline unsigned long highmap_end_pfn(void)
104 {
105         /* Do not reference physical address outside the kernel. */
106         return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
107 }
108
109 #endif
110
111 static inline int
112 within(unsigned long addr, unsigned long start, unsigned long end)
113 {
114         return addr >= start && addr < end;
115 }
116
117 static inline int
118 within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
119 {
120         return addr >= start && addr <= end;
121 }
122
123 /*
124  * Flushing functions
125  */
126
127 /**
128  * clflush_cache_range - flush a cache range with clflush
129  * @vaddr:      virtual start address
130  * @size:       number of bytes to flush
131  *
132  * clflushopt is an unordered instruction which needs fencing with mfence or
133  * sfence to avoid ordering issues.
134  */
135 void clflush_cache_range(void *vaddr, unsigned int size)
136 {
137         const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
138         void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
139         void *vend = vaddr + size;
140
141         if (p >= vend)
142                 return;
143
144         mb();
145
146         for (; p < vend; p += clflush_size)
147                 clflushopt(p);
148
149         mb();
150 }
151 EXPORT_SYMBOL_GPL(clflush_cache_range);
152
153 void arch_invalidate_pmem(void *addr, size_t size)
154 {
155         clflush_cache_range(addr, size);
156 }
157 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
158
159 static void __cpa_flush_all(void *arg)
160 {
161         unsigned long cache = (unsigned long)arg;
162
163         /*
164          * Flush all to work around Errata in early athlons regarding
165          * large page flushing.
166          */
167         __flush_tlb_all();
168
169         if (cache && boot_cpu_data.x86 >= 4)
170                 wbinvd();
171 }
172
173 static void cpa_flush_all(unsigned long cache)
174 {
175         BUG_ON(irqs_disabled());
176
177         on_each_cpu(__cpa_flush_all, (void *) cache, 1);
178 }
179
180 static void __cpa_flush_range(void *arg)
181 {
182         /*
183          * We could optimize that further and do individual per page
184          * tlb invalidates for a low number of pages. Caveat: we must
185          * flush the high aliases on 64bit as well.
186          */
187         __flush_tlb_all();
188 }
189
190 static void cpa_flush_range(unsigned long start, int numpages, int cache)
191 {
192         unsigned int i, level;
193         unsigned long addr;
194
195         BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
196         WARN_ON(PAGE_ALIGN(start) != start);
197
198         on_each_cpu(__cpa_flush_range, NULL, 1);
199
200         if (!cache)
201                 return;
202
203         /*
204          * We only need to flush on one CPU,
205          * clflush is a MESI-coherent instruction that
206          * will cause all other CPUs to flush the same
207          * cachelines:
208          */
209         for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
210                 pte_t *pte = lookup_address(addr, &level);
211
212                 /*
213                  * Only flush present addresses:
214                  */
215                 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
216                         clflush_cache_range((void *) addr, PAGE_SIZE);
217         }
218 }
219
220 static void cpa_flush_array(unsigned long *start, int numpages, int cache,
221                             int in_flags, struct page **pages)
222 {
223         unsigned int i, level;
224 #ifdef CONFIG_PREEMPT
225         /*
226          * Avoid wbinvd() because it causes latencies on all CPUs,
227          * regardless of any CPU isolation that may be in effect.
228          *
229          * This should be extended for CAT enabled systems independent of
230          * PREEMPT because wbinvd() does not respect the CAT partitions and
231          * this is exposed to unpriviledged users through the graphics
232          * subsystem.
233          */
234         unsigned long do_wbinvd = 0;
235 #else
236         unsigned long do_wbinvd = cache && numpages >= 1024; /* 4M threshold */
237 #endif
238
239         BUG_ON(irqs_disabled());
240
241         on_each_cpu(__cpa_flush_all, (void *) do_wbinvd, 1);
242
243         if (!cache || do_wbinvd)
244                 return;
245
246         /*
247          * We only need to flush on one CPU,
248          * clflush is a MESI-coherent instruction that
249          * will cause all other CPUs to flush the same
250          * cachelines:
251          */
252         for (i = 0; i < numpages; i++) {
253                 unsigned long addr;
254                 pte_t *pte;
255
256                 if (in_flags & CPA_PAGES_ARRAY)
257                         addr = (unsigned long)page_address(pages[i]);
258                 else
259                         addr = start[i];
260
261                 pte = lookup_address(addr, &level);
262
263                 /*
264                  * Only flush present addresses:
265                  */
266                 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
267                         clflush_cache_range((void *)addr, PAGE_SIZE);
268         }
269 }
270
271 /*
272  * Certain areas of memory on x86 require very specific protection flags,
273  * for example the BIOS area or kernel text. Callers don't always get this
274  * right (again, ioremap() on BIOS memory is not uncommon) so this function
275  * checks and fixes these known static required protection bits.
276  */
277 static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
278                                    unsigned long pfn)
279 {
280         pgprot_t forbidden = __pgprot(0);
281
282         /*
283          * The BIOS area between 640k and 1Mb needs to be executable for
284          * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
285          */
286 #ifdef CONFIG_PCI_BIOS
287         if (pcibios_enabled && within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
288                 pgprot_val(forbidden) |= _PAGE_NX;
289 #endif
290
291         /*
292          * The kernel text needs to be executable for obvious reasons
293          * Does not cover __inittext since that is gone later on. On
294          * 64bit we do not enforce !NX on the low mapping
295          */
296         if (within(address, (unsigned long)_text, (unsigned long)_etext))
297                 pgprot_val(forbidden) |= _PAGE_NX;
298
299         /*
300          * The .rodata section needs to be read-only. Using the pfn
301          * catches all aliases.
302          */
303         if (within(pfn, __pa_symbol(__start_rodata) >> PAGE_SHIFT,
304                    __pa_symbol(__end_rodata) >> PAGE_SHIFT))
305                 pgprot_val(forbidden) |= _PAGE_RW;
306
307 #if defined(CONFIG_X86_64)
308         /*
309          * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
310          * kernel text mappings for the large page aligned text, rodata sections
311          * will be always read-only. For the kernel identity mappings covering
312          * the holes caused by this alignment can be anything that user asks.
313          *
314          * This will preserve the large page mappings for kernel text/data
315          * at no extra cost.
316          */
317         if (kernel_set_to_readonly &&
318             within(address, (unsigned long)_text,
319                    (unsigned long)__end_rodata_hpage_align)) {
320                 unsigned int level;
321
322                 /*
323                  * Don't enforce the !RW mapping for the kernel text mapping,
324                  * if the current mapping is already using small page mapping.
325                  * No need to work hard to preserve large page mappings in this
326                  * case.
327                  *
328                  * This also fixes the Linux Xen paravirt guest boot failure
329                  * (because of unexpected read-only mappings for kernel identity
330                  * mappings). In this paravirt guest case, the kernel text
331                  * mapping and the kernel identity mapping share the same
332                  * page-table pages. Thus we can't really use different
333                  * protections for the kernel text and identity mappings. Also,
334                  * these shared mappings are made of small page mappings.
335                  * Thus this don't enforce !RW mapping for small page kernel
336                  * text mapping logic will help Linux Xen parvirt guest boot
337                  * as well.
338                  */
339                 if (lookup_address(address, &level) && (level != PG_LEVEL_4K))
340                         pgprot_val(forbidden) |= _PAGE_RW;
341         }
342 #endif
343
344         prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
345
346         return prot;
347 }
348
349 /*
350  * Lookup the page table entry for a virtual address in a specific pgd.
351  * Return a pointer to the entry and the level of the mapping.
352  */
353 pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
354                              unsigned int *level)
355 {
356         p4d_t *p4d;
357         pud_t *pud;
358         pmd_t *pmd;
359
360         *level = PG_LEVEL_NONE;
361
362         if (pgd_none(*pgd))
363                 return NULL;
364
365         p4d = p4d_offset(pgd, address);
366         if (p4d_none(*p4d))
367                 return NULL;
368
369         *level = PG_LEVEL_512G;
370         if (p4d_large(*p4d) || !p4d_present(*p4d))
371                 return (pte_t *)p4d;
372
373         pud = pud_offset(p4d, address);
374         if (pud_none(*pud))
375                 return NULL;
376
377         *level = PG_LEVEL_1G;
378         if (pud_large(*pud) || !pud_present(*pud))
379                 return (pte_t *)pud;
380
381         pmd = pmd_offset(pud, address);
382         if (pmd_none(*pmd))
383                 return NULL;
384
385         *level = PG_LEVEL_2M;
386         if (pmd_large(*pmd) || !pmd_present(*pmd))
387                 return (pte_t *)pmd;
388
389         *level = PG_LEVEL_4K;
390
391         return pte_offset_kernel(pmd, address);
392 }
393
394 /*
395  * Lookup the page table entry for a virtual address. Return a pointer
396  * to the entry and the level of the mapping.
397  *
398  * Note: We return pud and pmd either when the entry is marked large
399  * or when the present bit is not set. Otherwise we would return a
400  * pointer to a nonexisting mapping.
401  */
402 pte_t *lookup_address(unsigned long address, unsigned int *level)
403 {
404         return lookup_address_in_pgd(pgd_offset_k(address), address, level);
405 }
406 EXPORT_SYMBOL_GPL(lookup_address);
407
408 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
409                                   unsigned int *level)
410 {
411         if (cpa->pgd)
412                 return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
413                                                address, level);
414
415         return lookup_address(address, level);
416 }
417
418 /*
419  * Lookup the PMD entry for a virtual address. Return a pointer to the entry
420  * or NULL if not present.
421  */
422 pmd_t *lookup_pmd_address(unsigned long address)
423 {
424         pgd_t *pgd;
425         p4d_t *p4d;
426         pud_t *pud;
427
428         pgd = pgd_offset_k(address);
429         if (pgd_none(*pgd))
430                 return NULL;
431
432         p4d = p4d_offset(pgd, address);
433         if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d))
434                 return NULL;
435
436         pud = pud_offset(p4d, address);
437         if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
438                 return NULL;
439
440         return pmd_offset(pud, address);
441 }
442
443 /*
444  * This is necessary because __pa() does not work on some
445  * kinds of memory, like vmalloc() or the alloc_remap()
446  * areas on 32-bit NUMA systems.  The percpu areas can
447  * end up in this kind of memory, for instance.
448  *
449  * This could be optimized, but it is only intended to be
450  * used at inititalization time, and keeping it
451  * unoptimized should increase the testing coverage for
452  * the more obscure platforms.
453  */
454 phys_addr_t slow_virt_to_phys(void *__virt_addr)
455 {
456         unsigned long virt_addr = (unsigned long)__virt_addr;
457         phys_addr_t phys_addr;
458         unsigned long offset;
459         enum pg_level level;
460         pte_t *pte;
461
462         pte = lookup_address(virt_addr, &level);
463         BUG_ON(!pte);
464
465         /*
466          * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
467          * before being left-shifted PAGE_SHIFT bits -- this trick is to
468          * make 32-PAE kernel work correctly.
469          */
470         switch (level) {
471         case PG_LEVEL_1G:
472                 phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
473                 offset = virt_addr & ~PUD_PAGE_MASK;
474                 break;
475         case PG_LEVEL_2M:
476                 phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
477                 offset = virt_addr & ~PMD_PAGE_MASK;
478                 break;
479         default:
480                 phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
481                 offset = virt_addr & ~PAGE_MASK;
482         }
483
484         return (phys_addr_t)(phys_addr | offset);
485 }
486 EXPORT_SYMBOL_GPL(slow_virt_to_phys);
487
488 /*
489  * Set the new pmd in all the pgds we know about:
490  */
491 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
492 {
493         /* change init_mm */
494         set_pte_atomic(kpte, pte);
495 #ifdef CONFIG_X86_32
496         if (!SHARED_KERNEL_PMD) {
497                 struct page *page;
498
499                 list_for_each_entry(page, &pgd_list, lru) {
500                         pgd_t *pgd;
501                         p4d_t *p4d;
502                         pud_t *pud;
503                         pmd_t *pmd;
504
505                         pgd = (pgd_t *)page_address(page) + pgd_index(address);
506                         p4d = p4d_offset(pgd, address);
507                         pud = pud_offset(p4d, address);
508                         pmd = pmd_offset(pud, address);
509                         set_pte_atomic((pte_t *)pmd, pte);
510                 }
511         }
512 #endif
513 }
514
515 static int
516 try_preserve_large_page(pte_t *kpte, unsigned long address,
517                         struct cpa_data *cpa)
518 {
519         unsigned long nextpage_addr, numpages, pmask, psize, addr, pfn, old_pfn;
520         pte_t new_pte, old_pte, *tmp;
521         pgprot_t old_prot, new_prot, req_prot;
522         int i, do_split = 1;
523         enum pg_level level;
524
525         if (cpa->force_split)
526                 return 1;
527
528         spin_lock(&pgd_lock);
529         /*
530          * Check for races, another CPU might have split this page
531          * up already:
532          */
533         tmp = _lookup_address_cpa(cpa, address, &level);
534         if (tmp != kpte)
535                 goto out_unlock;
536
537         switch (level) {
538         case PG_LEVEL_2M:
539                 old_prot = pmd_pgprot(*(pmd_t *)kpte);
540                 old_pfn = pmd_pfn(*(pmd_t *)kpte);
541                 break;
542         case PG_LEVEL_1G:
543                 old_prot = pud_pgprot(*(pud_t *)kpte);
544                 old_pfn = pud_pfn(*(pud_t *)kpte);
545                 break;
546         default:
547                 do_split = -EINVAL;
548                 goto out_unlock;
549         }
550
551         psize = page_level_size(level);
552         pmask = page_level_mask(level);
553
554         /*
555          * Calculate the number of pages, which fit into this large
556          * page starting at address:
557          */
558         nextpage_addr = (address + psize) & pmask;
559         numpages = (nextpage_addr - address) >> PAGE_SHIFT;
560         if (numpages < cpa->numpages)
561                 cpa->numpages = numpages;
562
563         /*
564          * We are safe now. Check whether the new pgprot is the same:
565          * Convert protection attributes to 4k-format, as cpa->mask* are set
566          * up accordingly.
567          */
568         old_pte = *kpte;
569         req_prot = pgprot_large_2_4k(old_prot);
570
571         pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
572         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
573
574         /*
575          * req_prot is in format of 4k pages. It must be converted to large
576          * page format: the caching mode includes the PAT bit located at
577          * different bit positions in the two formats.
578          */
579         req_prot = pgprot_4k_2_large(req_prot);
580
581         /*
582          * Set the PSE and GLOBAL flags only if the PRESENT flag is
583          * set otherwise pmd_present/pmd_huge will return true even on
584          * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
585          * for the ancient hardware that doesn't support it.
586          */
587         if (pgprot_val(req_prot) & _PAGE_PRESENT)
588                 pgprot_val(req_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
589         else
590                 pgprot_val(req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
591
592         req_prot = canon_pgprot(req_prot);
593
594         /*
595          * old_pfn points to the large page base pfn. So we need
596          * to add the offset of the virtual address:
597          */
598         pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
599         cpa->pfn = pfn;
600
601         new_prot = static_protections(req_prot, address, pfn);
602
603         /*
604          * We need to check the full range, whether
605          * static_protection() requires a different pgprot for one of
606          * the pages in the range we try to preserve:
607          */
608         addr = address & pmask;
609         pfn = old_pfn;
610         for (i = 0; i < (psize >> PAGE_SHIFT); i++, addr += PAGE_SIZE, pfn++) {
611                 pgprot_t chk_prot = static_protections(req_prot, addr, pfn);
612
613                 if (pgprot_val(chk_prot) != pgprot_val(new_prot))
614                         goto out_unlock;
615         }
616
617         /*
618          * If there are no changes, return. maxpages has been updated
619          * above:
620          */
621         if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
622                 do_split = 0;
623                 goto out_unlock;
624         }
625
626         /*
627          * We need to change the attributes. Check, whether we can
628          * change the large page in one go. We request a split, when
629          * the address is not aligned and the number of pages is
630          * smaller than the number of pages in the large page. Note
631          * that we limited the number of possible pages already to
632          * the number of pages in the large page.
633          */
634         if (address == (address & pmask) && cpa->numpages == (psize >> PAGE_SHIFT)) {
635                 /*
636                  * The address is aligned and the number of pages
637                  * covers the full page.
638                  */
639                 new_pte = pfn_pte(old_pfn, new_prot);
640                 __set_pmd_pte(kpte, address, new_pte);
641                 cpa->flags |= CPA_FLUSHTLB;
642                 do_split = 0;
643         }
644
645 out_unlock:
646         spin_unlock(&pgd_lock);
647
648         return do_split;
649 }
650
651 static int
652 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
653                    struct page *base)
654 {
655         pte_t *pbase = (pte_t *)page_address(base);
656         unsigned long ref_pfn, pfn, pfninc = 1;
657         unsigned int i, level;
658         pte_t *tmp;
659         pgprot_t ref_prot;
660
661         spin_lock(&pgd_lock);
662         /*
663          * Check for races, another CPU might have split this page
664          * up for us already:
665          */
666         tmp = _lookup_address_cpa(cpa, address, &level);
667         if (tmp != kpte) {
668                 spin_unlock(&pgd_lock);
669                 return 1;
670         }
671
672         paravirt_alloc_pte(&init_mm, page_to_pfn(base));
673
674         switch (level) {
675         case PG_LEVEL_2M:
676                 ref_prot = pmd_pgprot(*(pmd_t *)kpte);
677                 /* clear PSE and promote PAT bit to correct position */
678                 ref_prot = pgprot_large_2_4k(ref_prot);
679                 ref_pfn = pmd_pfn(*(pmd_t *)kpte);
680                 break;
681
682         case PG_LEVEL_1G:
683                 ref_prot = pud_pgprot(*(pud_t *)kpte);
684                 ref_pfn = pud_pfn(*(pud_t *)kpte);
685                 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
686
687                 /*
688                  * Clear the PSE flags if the PRESENT flag is not set
689                  * otherwise pmd_present/pmd_huge will return true
690                  * even on a non present pmd.
691                  */
692                 if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
693                         pgprot_val(ref_prot) &= ~_PAGE_PSE;
694                 break;
695
696         default:
697                 spin_unlock(&pgd_lock);
698                 return 1;
699         }
700
701         /*
702          * Set the GLOBAL flags only if the PRESENT flag is set
703          * otherwise pmd/pte_present will return true even on a non
704          * present pmd/pte. The canon_pgprot will clear _PAGE_GLOBAL
705          * for the ancient hardware that doesn't support it.
706          */
707         if (pgprot_val(ref_prot) & _PAGE_PRESENT)
708                 pgprot_val(ref_prot) |= _PAGE_GLOBAL;
709         else
710                 pgprot_val(ref_prot) &= ~_PAGE_GLOBAL;
711
712         /*
713          * Get the target pfn from the original entry:
714          */
715         pfn = ref_pfn;
716         for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
717                 set_pte(&pbase[i], pfn_pte(pfn, canon_pgprot(ref_prot)));
718
719         if (virt_addr_valid(address)) {
720                 unsigned long pfn = PFN_DOWN(__pa(address));
721
722                 if (pfn_range_is_mapped(pfn, pfn + 1))
723                         split_page_count(level);
724         }
725
726         /*
727          * Install the new, split up pagetable.
728          *
729          * We use the standard kernel pagetable protections for the new
730          * pagetable protections, the actual ptes set above control the
731          * primary protection behavior:
732          */
733         __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
734
735         /*
736          * Intel Atom errata AAH41 workaround.
737          *
738          * The real fix should be in hw or in a microcode update, but
739          * we also probabilistically try to reduce the window of having
740          * a large TLB mixed with 4K TLBs while instruction fetches are
741          * going on.
742          */
743         __flush_tlb_all();
744         spin_unlock(&pgd_lock);
745
746         return 0;
747 }
748
749 static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
750                             unsigned long address)
751 {
752         struct page *base;
753
754         if (!debug_pagealloc_enabled())
755                 spin_unlock(&cpa_lock);
756         base = alloc_pages(GFP_KERNEL, 0);
757         if (!debug_pagealloc_enabled())
758                 spin_lock(&cpa_lock);
759         if (!base)
760                 return -ENOMEM;
761
762         if (__split_large_page(cpa, kpte, address, base))
763                 __free_page(base);
764
765         return 0;
766 }
767
768 static bool try_to_free_pte_page(pte_t *pte)
769 {
770         int i;
771
772         for (i = 0; i < PTRS_PER_PTE; i++)
773                 if (!pte_none(pte[i]))
774                         return false;
775
776         free_page((unsigned long)pte);
777         return true;
778 }
779
780 static bool try_to_free_pmd_page(pmd_t *pmd)
781 {
782         int i;
783
784         for (i = 0; i < PTRS_PER_PMD; i++)
785                 if (!pmd_none(pmd[i]))
786                         return false;
787
788         free_page((unsigned long)pmd);
789         return true;
790 }
791
792 static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
793 {
794         pte_t *pte = pte_offset_kernel(pmd, start);
795
796         while (start < end) {
797                 set_pte(pte, __pte(0));
798
799                 start += PAGE_SIZE;
800                 pte++;
801         }
802
803         if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
804                 pmd_clear(pmd);
805                 return true;
806         }
807         return false;
808 }
809
810 static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
811                               unsigned long start, unsigned long end)
812 {
813         if (unmap_pte_range(pmd, start, end))
814                 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
815                         pud_clear(pud);
816 }
817
818 static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
819 {
820         pmd_t *pmd = pmd_offset(pud, start);
821
822         /*
823          * Not on a 2MB page boundary?
824          */
825         if (start & (PMD_SIZE - 1)) {
826                 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
827                 unsigned long pre_end = min_t(unsigned long, end, next_page);
828
829                 __unmap_pmd_range(pud, pmd, start, pre_end);
830
831                 start = pre_end;
832                 pmd++;
833         }
834
835         /*
836          * Try to unmap in 2M chunks.
837          */
838         while (end - start >= PMD_SIZE) {
839                 if (pmd_large(*pmd))
840                         pmd_clear(pmd);
841                 else
842                         __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
843
844                 start += PMD_SIZE;
845                 pmd++;
846         }
847
848         /*
849          * 4K leftovers?
850          */
851         if (start < end)
852                 return __unmap_pmd_range(pud, pmd, start, end);
853
854         /*
855          * Try again to free the PMD page if haven't succeeded above.
856          */
857         if (!pud_none(*pud))
858                 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
859                         pud_clear(pud);
860 }
861
862 static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
863 {
864         pud_t *pud = pud_offset(p4d, start);
865
866         /*
867          * Not on a GB page boundary?
868          */
869         if (start & (PUD_SIZE - 1)) {
870                 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
871                 unsigned long pre_end   = min_t(unsigned long, end, next_page);
872
873                 unmap_pmd_range(pud, start, pre_end);
874
875                 start = pre_end;
876                 pud++;
877         }
878
879         /*
880          * Try to unmap in 1G chunks?
881          */
882         while (end - start >= PUD_SIZE) {
883
884                 if (pud_large(*pud))
885                         pud_clear(pud);
886                 else
887                         unmap_pmd_range(pud, start, start + PUD_SIZE);
888
889                 start += PUD_SIZE;
890                 pud++;
891         }
892
893         /*
894          * 2M leftovers?
895          */
896         if (start < end)
897                 unmap_pmd_range(pud, start, end);
898
899         /*
900          * No need to try to free the PUD page because we'll free it in
901          * populate_pgd's error path
902          */
903 }
904
905 static int alloc_pte_page(pmd_t *pmd)
906 {
907         pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
908         if (!pte)
909                 return -1;
910
911         set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
912         return 0;
913 }
914
915 static int alloc_pmd_page(pud_t *pud)
916 {
917         pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
918         if (!pmd)
919                 return -1;
920
921         set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
922         return 0;
923 }
924
925 static void populate_pte(struct cpa_data *cpa,
926                          unsigned long start, unsigned long end,
927                          unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
928 {
929         pte_t *pte;
930
931         pte = pte_offset_kernel(pmd, start);
932
933         /*
934          * Set the GLOBAL flags only if the PRESENT flag is
935          * set otherwise pte_present will return true even on
936          * a non present pte. The canon_pgprot will clear
937          * _PAGE_GLOBAL for the ancient hardware that doesn't
938          * support it.
939          */
940         if (pgprot_val(pgprot) & _PAGE_PRESENT)
941                 pgprot_val(pgprot) |= _PAGE_GLOBAL;
942         else
943                 pgprot_val(pgprot) &= ~_PAGE_GLOBAL;
944
945         pgprot = canon_pgprot(pgprot);
946
947         while (num_pages-- && start < end) {
948                 set_pte(pte, pfn_pte(cpa->pfn, pgprot));
949
950                 start    += PAGE_SIZE;
951                 cpa->pfn++;
952                 pte++;
953         }
954 }
955
956 static long populate_pmd(struct cpa_data *cpa,
957                          unsigned long start, unsigned long end,
958                          unsigned num_pages, pud_t *pud, pgprot_t pgprot)
959 {
960         long cur_pages = 0;
961         pmd_t *pmd;
962         pgprot_t pmd_pgprot;
963
964         /*
965          * Not on a 2M boundary?
966          */
967         if (start & (PMD_SIZE - 1)) {
968                 unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
969                 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
970
971                 pre_end   = min_t(unsigned long, pre_end, next_page);
972                 cur_pages = (pre_end - start) >> PAGE_SHIFT;
973                 cur_pages = min_t(unsigned int, num_pages, cur_pages);
974
975                 /*
976                  * Need a PTE page?
977                  */
978                 pmd = pmd_offset(pud, start);
979                 if (pmd_none(*pmd))
980                         if (alloc_pte_page(pmd))
981                                 return -1;
982
983                 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
984
985                 start = pre_end;
986         }
987
988         /*
989          * We mapped them all?
990          */
991         if (num_pages == cur_pages)
992                 return cur_pages;
993
994         pmd_pgprot = pgprot_4k_2_large(pgprot);
995
996         while (end - start >= PMD_SIZE) {
997
998                 /*
999                  * We cannot use a 1G page so allocate a PMD page if needed.
1000                  */
1001                 if (pud_none(*pud))
1002                         if (alloc_pmd_page(pud))
1003                                 return -1;
1004
1005                 pmd = pmd_offset(pud, start);
1006
1007                 set_pmd(pmd, __pmd(cpa->pfn << PAGE_SHIFT | _PAGE_PSE |
1008                                    massage_pgprot(pmd_pgprot)));
1009
1010                 start     += PMD_SIZE;
1011                 cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
1012                 cur_pages += PMD_SIZE >> PAGE_SHIFT;
1013         }
1014
1015         /*
1016          * Map trailing 4K pages.
1017          */
1018         if (start < end) {
1019                 pmd = pmd_offset(pud, start);
1020                 if (pmd_none(*pmd))
1021                         if (alloc_pte_page(pmd))
1022                                 return -1;
1023
1024                 populate_pte(cpa, start, end, num_pages - cur_pages,
1025                              pmd, pgprot);
1026         }
1027         return num_pages;
1028 }
1029
1030 static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
1031                         pgprot_t pgprot)
1032 {
1033         pud_t *pud;
1034         unsigned long end;
1035         long cur_pages = 0;
1036         pgprot_t pud_pgprot;
1037
1038         end = start + (cpa->numpages << PAGE_SHIFT);
1039
1040         /*
1041          * Not on a Gb page boundary? => map everything up to it with
1042          * smaller pages.
1043          */
1044         if (start & (PUD_SIZE - 1)) {
1045                 unsigned long pre_end;
1046                 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1047
1048                 pre_end   = min_t(unsigned long, end, next_page);
1049                 cur_pages = (pre_end - start) >> PAGE_SHIFT;
1050                 cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
1051
1052                 pud = pud_offset(p4d, start);
1053
1054                 /*
1055                  * Need a PMD page?
1056                  */
1057                 if (pud_none(*pud))
1058                         if (alloc_pmd_page(pud))
1059                                 return -1;
1060
1061                 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
1062                                          pud, pgprot);
1063                 if (cur_pages < 0)
1064                         return cur_pages;
1065
1066                 start = pre_end;
1067         }
1068
1069         /* We mapped them all? */
1070         if (cpa->numpages == cur_pages)
1071                 return cur_pages;
1072
1073         pud = pud_offset(p4d, start);
1074         pud_pgprot = pgprot_4k_2_large(pgprot);
1075
1076         /*
1077          * Map everything starting from the Gb boundary, possibly with 1G pages
1078          */
1079         while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) {
1080                 set_pud(pud, __pud(cpa->pfn << PAGE_SHIFT | _PAGE_PSE |
1081                                    massage_pgprot(pud_pgprot)));
1082
1083                 start     += PUD_SIZE;
1084                 cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
1085                 cur_pages += PUD_SIZE >> PAGE_SHIFT;
1086                 pud++;
1087         }
1088
1089         /* Map trailing leftover */
1090         if (start < end) {
1091                 long tmp;
1092
1093                 pud = pud_offset(p4d, start);
1094                 if (pud_none(*pud))
1095                         if (alloc_pmd_page(pud))
1096                                 return -1;
1097
1098                 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
1099                                    pud, pgprot);
1100                 if (tmp < 0)
1101                         return cur_pages;
1102
1103                 cur_pages += tmp;
1104         }
1105         return cur_pages;
1106 }
1107
1108 /*
1109  * Restrictions for kernel page table do not necessarily apply when mapping in
1110  * an alternate PGD.
1111  */
1112 static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
1113 {
1114         pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
1115         pud_t *pud = NULL;      /* shut up gcc */
1116         p4d_t *p4d;
1117         pgd_t *pgd_entry;
1118         long ret;
1119
1120         pgd_entry = cpa->pgd + pgd_index(addr);
1121
1122         if (pgd_none(*pgd_entry)) {
1123                 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
1124                 if (!p4d)
1125                         return -1;
1126
1127                 set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE));
1128         }
1129
1130         /*
1131          * Allocate a PUD page and hand it down for mapping.
1132          */
1133         p4d = p4d_offset(pgd_entry, addr);
1134         if (p4d_none(*p4d)) {
1135                 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
1136                 if (!pud)
1137                         return -1;
1138
1139                 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
1140         }
1141
1142         pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
1143         pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);
1144
1145         ret = populate_pud(cpa, addr, p4d, pgprot);
1146         if (ret < 0) {
1147                 /*
1148                  * Leave the PUD page in place in case some other CPU or thread
1149                  * already found it, but remove any useless entries we just
1150                  * added to it.
1151                  */
1152                 unmap_pud_range(p4d, addr,
1153                                 addr + (cpa->numpages << PAGE_SHIFT));
1154                 return ret;
1155         }
1156
1157         cpa->numpages = ret;
1158         return 0;
1159 }
1160
1161 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
1162                                int primary)
1163 {
1164         if (cpa->pgd) {
1165                 /*
1166                  * Right now, we only execute this code path when mapping
1167                  * the EFI virtual memory map regions, no other users
1168                  * provide a ->pgd value. This may change in the future.
1169                  */
1170                 return populate_pgd(cpa, vaddr);
1171         }
1172
1173         /*
1174          * Ignore all non primary paths.
1175          */
1176         if (!primary) {
1177                 cpa->numpages = 1;
1178                 return 0;
1179         }
1180
1181         /*
1182          * Ignore the NULL PTE for kernel identity mapping, as it is expected
1183          * to have holes.
1184          * Also set numpages to '1' indicating that we processed cpa req for
1185          * one virtual address page and its pfn. TBD: numpages can be set based
1186          * on the initial value and the level returned by lookup_address().
1187          */
1188         if (within(vaddr, PAGE_OFFSET,
1189                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
1190                 cpa->numpages = 1;
1191                 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
1192                 return 0;
1193         } else {
1194                 WARN(1, KERN_WARNING "CPA: called for zero pte. "
1195                         "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
1196                         *cpa->vaddr);
1197
1198                 return -EFAULT;
1199         }
1200 }
1201
1202 static int __change_page_attr(struct cpa_data *cpa, int primary)
1203 {
1204         unsigned long address;
1205         int do_split, err;
1206         unsigned int level;
1207         pte_t *kpte, old_pte;
1208
1209         if (cpa->flags & CPA_PAGES_ARRAY) {
1210                 struct page *page = cpa->pages[cpa->curpage];
1211                 if (unlikely(PageHighMem(page)))
1212                         return 0;
1213                 address = (unsigned long)page_address(page);
1214         } else if (cpa->flags & CPA_ARRAY)
1215                 address = cpa->vaddr[cpa->curpage];
1216         else
1217                 address = *cpa->vaddr;
1218 repeat:
1219         kpte = _lookup_address_cpa(cpa, address, &level);
1220         if (!kpte)
1221                 return __cpa_process_fault(cpa, address, primary);
1222
1223         old_pte = *kpte;
1224         if (pte_none(old_pte))
1225                 return __cpa_process_fault(cpa, address, primary);
1226
1227         if (level == PG_LEVEL_4K) {
1228                 pte_t new_pte;
1229                 pgprot_t new_prot = pte_pgprot(old_pte);
1230                 unsigned long pfn = pte_pfn(old_pte);
1231
1232                 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
1233                 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
1234
1235                 new_prot = static_protections(new_prot, address, pfn);
1236
1237                 /*
1238                  * Set the GLOBAL flags only if the PRESENT flag is
1239                  * set otherwise pte_present will return true even on
1240                  * a non present pte. The canon_pgprot will clear
1241                  * _PAGE_GLOBAL for the ancient hardware that doesn't
1242                  * support it.
1243                  */
1244                 if (pgprot_val(new_prot) & _PAGE_PRESENT)
1245                         pgprot_val(new_prot) |= _PAGE_GLOBAL;
1246                 else
1247                         pgprot_val(new_prot) &= ~_PAGE_GLOBAL;
1248
1249                 /*
1250                  * We need to keep the pfn from the existing PTE,
1251                  * after all we're only going to change it's attributes
1252                  * not the memory it points to
1253                  */
1254                 new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
1255                 cpa->pfn = pfn;
1256                 /*
1257                  * Do we really change anything ?
1258                  */
1259                 if (pte_val(old_pte) != pte_val(new_pte)) {
1260                         set_pte_atomic(kpte, new_pte);
1261                         cpa->flags |= CPA_FLUSHTLB;
1262                 }
1263                 cpa->numpages = 1;
1264                 return 0;
1265         }
1266
1267         /*
1268          * Check, whether we can keep the large page intact
1269          * and just change the pte:
1270          */
1271         do_split = try_preserve_large_page(kpte, address, cpa);
1272         /*
1273          * When the range fits into the existing large page,
1274          * return. cp->numpages and cpa->tlbflush have been updated in
1275          * try_large_page:
1276          */
1277         if (do_split <= 0)
1278                 return do_split;
1279
1280         /*
1281          * We have to split the large page:
1282          */
1283         err = split_large_page(cpa, kpte, address);
1284         if (!err) {
1285                 /*
1286                  * Do a global flush tlb after splitting the large page
1287                  * and before we do the actual change page attribute in the PTE.
1288                  *
1289                  * With out this, we violate the TLB application note, that says
1290                  * "The TLBs may contain both ordinary and large-page
1291                  *  translations for a 4-KByte range of linear addresses. This
1292                  *  may occur if software modifies the paging structures so that
1293                  *  the page size used for the address range changes. If the two
1294                  *  translations differ with respect to page frame or attributes
1295                  *  (e.g., permissions), processor behavior is undefined and may
1296                  *  be implementation-specific."
1297                  *
1298                  * We do this global tlb flush inside the cpa_lock, so that we
1299                  * don't allow any other cpu, with stale tlb entries change the
1300                  * page attribute in parallel, that also falls into the
1301                  * just split large page entry.
1302                  */
1303                 flush_tlb_all();
1304                 goto repeat;
1305         }
1306
1307         return err;
1308 }
1309
1310 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1311
1312 static int cpa_process_alias(struct cpa_data *cpa)
1313 {
1314         struct cpa_data alias_cpa;
1315         unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
1316         unsigned long vaddr;
1317         int ret;
1318
1319         if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
1320                 return 0;
1321
1322         /*
1323          * No need to redo, when the primary call touched the direct
1324          * mapping already:
1325          */
1326         if (cpa->flags & CPA_PAGES_ARRAY) {
1327                 struct page *page = cpa->pages[cpa->curpage];
1328                 if (unlikely(PageHighMem(page)))
1329                         return 0;
1330                 vaddr = (unsigned long)page_address(page);
1331         } else if (cpa->flags & CPA_ARRAY)
1332                 vaddr = cpa->vaddr[cpa->curpage];
1333         else
1334                 vaddr = *cpa->vaddr;
1335
1336         if (!(within(vaddr, PAGE_OFFSET,
1337                     PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
1338
1339                 alias_cpa = *cpa;
1340                 alias_cpa.vaddr = &laddr;
1341                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1342
1343                 ret = __change_page_attr_set_clr(&alias_cpa, 0);
1344                 if (ret)
1345                         return ret;
1346         }
1347
1348 #ifdef CONFIG_X86_64
1349         /*
1350          * If the primary call didn't touch the high mapping already
1351          * and the physical address is inside the kernel map, we need
1352          * to touch the high mapped kernel as well:
1353          */
1354         if (!within(vaddr, (unsigned long)_text, _brk_end) &&
1355             within_inclusive(cpa->pfn, highmap_start_pfn(),
1356                              highmap_end_pfn())) {
1357                 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
1358                                                __START_KERNEL_map - phys_base;
1359                 alias_cpa = *cpa;
1360                 alias_cpa.vaddr = &temp_cpa_vaddr;
1361                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1362
1363                 /*
1364                  * The high mapping range is imprecise, so ignore the
1365                  * return value.
1366                  */
1367                 __change_page_attr_set_clr(&alias_cpa, 0);
1368         }
1369 #endif
1370
1371         return 0;
1372 }
1373
1374 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
1375 {
1376         unsigned long numpages = cpa->numpages;
1377         int ret;
1378
1379         while (numpages) {
1380                 /*
1381                  * Store the remaining nr of pages for the large page
1382                  * preservation check.
1383                  */
1384                 cpa->numpages = numpages;
1385                 /* for array changes, we can't use large page */
1386                 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
1387                         cpa->numpages = 1;
1388
1389                 if (!debug_pagealloc_enabled())
1390                         spin_lock(&cpa_lock);
1391                 ret = __change_page_attr(cpa, checkalias);
1392                 if (!debug_pagealloc_enabled())
1393                         spin_unlock(&cpa_lock);
1394                 if (ret)
1395                         return ret;
1396
1397                 if (checkalias) {
1398                         ret = cpa_process_alias(cpa);
1399                         if (ret)
1400                                 return ret;
1401                 }
1402
1403                 /*
1404                  * Adjust the number of pages with the result of the
1405                  * CPA operation. Either a large page has been
1406                  * preserved or a single page update happened.
1407                  */
1408                 BUG_ON(cpa->numpages > numpages || !cpa->numpages);
1409                 numpages -= cpa->numpages;
1410                 if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
1411                         cpa->curpage++;
1412                 else
1413                         *cpa->vaddr += cpa->numpages * PAGE_SIZE;
1414
1415         }
1416         return 0;
1417 }
1418
1419 static int change_page_attr_set_clr(unsigned long *addr, int numpages,
1420                                     pgprot_t mask_set, pgprot_t mask_clr,
1421                                     int force_split, int in_flag,
1422                                     struct page **pages)
1423 {
1424         struct cpa_data cpa;
1425         int ret, cache, checkalias;
1426         unsigned long baddr = 0;
1427
1428         memset(&cpa, 0, sizeof(cpa));
1429
1430         /*
1431          * Check, if we are requested to change a not supported
1432          * feature:
1433          */
1434         mask_set = canon_pgprot(mask_set);
1435         mask_clr = canon_pgprot(mask_clr);
1436         if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
1437                 return 0;
1438
1439         /* Ensure we are PAGE_SIZE aligned */
1440         if (in_flag & CPA_ARRAY) {
1441                 int i;
1442                 for (i = 0; i < numpages; i++) {
1443                         if (addr[i] & ~PAGE_MASK) {
1444                                 addr[i] &= PAGE_MASK;
1445                                 WARN_ON_ONCE(1);
1446                         }
1447                 }
1448         } else if (!(in_flag & CPA_PAGES_ARRAY)) {
1449                 /*
1450                  * in_flag of CPA_PAGES_ARRAY implies it is aligned.
1451                  * No need to cehck in that case
1452                  */
1453                 if (*addr & ~PAGE_MASK) {
1454                         *addr &= PAGE_MASK;
1455                         /*
1456                          * People should not be passing in unaligned addresses:
1457                          */
1458                         WARN_ON_ONCE(1);
1459                 }
1460                 /*
1461                  * Save address for cache flush. *addr is modified in the call
1462                  * to __change_page_attr_set_clr() below.
1463                  */
1464                 baddr = *addr;
1465         }
1466
1467         /* Must avoid aliasing mappings in the highmem code */
1468         kmap_flush_unused();
1469
1470         vm_unmap_aliases();
1471
1472         cpa.vaddr = addr;
1473         cpa.pages = pages;
1474         cpa.numpages = numpages;
1475         cpa.mask_set = mask_set;
1476         cpa.mask_clr = mask_clr;
1477         cpa.flags = 0;
1478         cpa.curpage = 0;
1479         cpa.force_split = force_split;
1480
1481         if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
1482                 cpa.flags |= in_flag;
1483
1484         /* No alias checking for _NX bit modifications */
1485         checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
1486
1487         ret = __change_page_attr_set_clr(&cpa, checkalias);
1488
1489         /*
1490          * Check whether we really changed something:
1491          */
1492         if (!(cpa.flags & CPA_FLUSHTLB))
1493                 goto out;
1494
1495         /*
1496          * No need to flush, when we did not set any of the caching
1497          * attributes:
1498          */
1499         cache = !!pgprot2cachemode(mask_set);
1500
1501         /*
1502          * On success we use CLFLUSH, when the CPU supports it to
1503          * avoid the WBINVD. If the CPU does not support it and in the
1504          * error case we fall back to cpa_flush_all (which uses
1505          * WBINVD):
1506          */
1507         if (!ret && boot_cpu_has(X86_FEATURE_CLFLUSH)) {
1508                 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
1509                         cpa_flush_array(addr, numpages, cache,
1510                                         cpa.flags, pages);
1511                 } else
1512                         cpa_flush_range(baddr, numpages, cache);
1513         } else
1514                 cpa_flush_all(cache);
1515
1516 out:
1517         return ret;
1518 }
1519
1520 static inline int change_page_attr_set(unsigned long *addr, int numpages,
1521                                        pgprot_t mask, int array)
1522 {
1523         return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
1524                 (array ? CPA_ARRAY : 0), NULL);
1525 }
1526
1527 static inline int change_page_attr_clear(unsigned long *addr, int numpages,
1528                                          pgprot_t mask, int array)
1529 {
1530         return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
1531                 (array ? CPA_ARRAY : 0), NULL);
1532 }
1533
1534 static inline int cpa_set_pages_array(struct page **pages, int numpages,
1535                                        pgprot_t mask)
1536 {
1537         return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
1538                 CPA_PAGES_ARRAY, pages);
1539 }
1540
1541 static inline int cpa_clear_pages_array(struct page **pages, int numpages,
1542                                          pgprot_t mask)
1543 {
1544         return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
1545                 CPA_PAGES_ARRAY, pages);
1546 }
1547
1548 int _set_memory_uc(unsigned long addr, int numpages)
1549 {
1550         /*
1551          * for now UC MINUS. see comments in ioremap_nocache()
1552          * If you really need strong UC use ioremap_uc(), but note
1553          * that you cannot override IO areas with set_memory_*() as
1554          * these helpers cannot work with IO memory.
1555          */
1556         return change_page_attr_set(&addr, numpages,
1557                                     cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1558                                     0);
1559 }
1560
1561 int set_memory_uc(unsigned long addr, int numpages)
1562 {
1563         int ret;
1564
1565         /*
1566          * for now UC MINUS. see comments in ioremap_nocache()
1567          */
1568         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1569                               _PAGE_CACHE_MODE_UC_MINUS, NULL);
1570         if (ret)
1571                 goto out_err;
1572
1573         ret = _set_memory_uc(addr, numpages);
1574         if (ret)
1575                 goto out_free;
1576
1577         return 0;
1578
1579 out_free:
1580         free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1581 out_err:
1582         return ret;
1583 }
1584 EXPORT_SYMBOL(set_memory_uc);
1585
1586 static int _set_memory_array(unsigned long *addr, int addrinarray,
1587                 enum page_cache_mode new_type)
1588 {
1589         enum page_cache_mode set_type;
1590         int i, j;
1591         int ret;
1592
1593         for (i = 0; i < addrinarray; i++) {
1594                 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
1595                                         new_type, NULL);
1596                 if (ret)
1597                         goto out_free;
1598         }
1599
1600         /* If WC, set to UC- first and then WC */
1601         set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
1602                                 _PAGE_CACHE_MODE_UC_MINUS : new_type;
1603
1604         ret = change_page_attr_set(addr, addrinarray,
1605                                    cachemode2pgprot(set_type), 1);
1606
1607         if (!ret && new_type == _PAGE_CACHE_MODE_WC)
1608                 ret = change_page_attr_set_clr(addr, addrinarray,
1609                                                cachemode2pgprot(
1610                                                 _PAGE_CACHE_MODE_WC),
1611                                                __pgprot(_PAGE_CACHE_MASK),
1612                                                0, CPA_ARRAY, NULL);
1613         if (ret)
1614                 goto out_free;
1615
1616         return 0;
1617
1618 out_free:
1619         for (j = 0; j < i; j++)
1620                 free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);
1621
1622         return ret;
1623 }
1624
1625 int set_memory_array_uc(unsigned long *addr, int addrinarray)
1626 {
1627         return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
1628 }
1629 EXPORT_SYMBOL(set_memory_array_uc);
1630
1631 int set_memory_array_wc(unsigned long *addr, int addrinarray)
1632 {
1633         return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WC);
1634 }
1635 EXPORT_SYMBOL(set_memory_array_wc);
1636
1637 int set_memory_array_wt(unsigned long *addr, int addrinarray)
1638 {
1639         return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WT);
1640 }
1641 EXPORT_SYMBOL_GPL(set_memory_array_wt);
1642
1643 int _set_memory_wc(unsigned long addr, int numpages)
1644 {
1645         int ret;
1646         unsigned long addr_copy = addr;
1647
1648         ret = change_page_attr_set(&addr, numpages,
1649                                    cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1650                                    0);
1651         if (!ret) {
1652                 ret = change_page_attr_set_clr(&addr_copy, numpages,
1653                                                cachemode2pgprot(
1654                                                 _PAGE_CACHE_MODE_WC),
1655                                                __pgprot(_PAGE_CACHE_MASK),
1656                                                0, 0, NULL);
1657         }
1658         return ret;
1659 }
1660
1661 int set_memory_wc(unsigned long addr, int numpages)
1662 {
1663         int ret;
1664
1665         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1666                 _PAGE_CACHE_MODE_WC, NULL);
1667         if (ret)
1668                 return ret;
1669
1670         ret = _set_memory_wc(addr, numpages);
1671         if (ret)
1672                 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1673
1674         return ret;
1675 }
1676 EXPORT_SYMBOL(set_memory_wc);
1677
1678 int _set_memory_wt(unsigned long addr, int numpages)
1679 {
1680         return change_page_attr_set(&addr, numpages,
1681                                     cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
1682 }
1683
1684 int set_memory_wt(unsigned long addr, int numpages)
1685 {
1686         int ret;
1687
1688         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1689                               _PAGE_CACHE_MODE_WT, NULL);
1690         if (ret)
1691                 return ret;
1692
1693         ret = _set_memory_wt(addr, numpages);
1694         if (ret)
1695                 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1696
1697         return ret;
1698 }
1699 EXPORT_SYMBOL_GPL(set_memory_wt);
1700
1701 int _set_memory_wb(unsigned long addr, int numpages)
1702 {
1703         /* WB cache mode is hard wired to all cache attribute bits being 0 */
1704         return change_page_attr_clear(&addr, numpages,
1705                                       __pgprot(_PAGE_CACHE_MASK), 0);
1706 }
1707
1708 int set_memory_wb(unsigned long addr, int numpages)
1709 {
1710         int ret;
1711
1712         ret = _set_memory_wb(addr, numpages);
1713         if (ret)
1714                 return ret;
1715
1716         free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1717         return 0;
1718 }
1719 EXPORT_SYMBOL(set_memory_wb);
1720
1721 int set_memory_array_wb(unsigned long *addr, int addrinarray)
1722 {
1723         int i;
1724         int ret;
1725
1726         /* WB cache mode is hard wired to all cache attribute bits being 0 */
1727         ret = change_page_attr_clear(addr, addrinarray,
1728                                       __pgprot(_PAGE_CACHE_MASK), 1);
1729         if (ret)
1730                 return ret;
1731
1732         for (i = 0; i < addrinarray; i++)
1733                 free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
1734
1735         return 0;
1736 }
1737 EXPORT_SYMBOL(set_memory_array_wb);
1738
1739 int set_memory_x(unsigned long addr, int numpages)
1740 {
1741         if (!(__supported_pte_mask & _PAGE_NX))
1742                 return 0;
1743
1744         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
1745 }
1746 EXPORT_SYMBOL(set_memory_x);
1747
1748 int set_memory_nx(unsigned long addr, int numpages)
1749 {
1750         if (!(__supported_pte_mask & _PAGE_NX))
1751                 return 0;
1752
1753         return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
1754 }
1755 EXPORT_SYMBOL(set_memory_nx);
1756
1757 int set_memory_ro(unsigned long addr, int numpages)
1758 {
1759         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
1760 }
1761
1762 int set_memory_rw(unsigned long addr, int numpages)
1763 {
1764         return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
1765 }
1766
1767 int set_memory_np(unsigned long addr, int numpages)
1768 {
1769         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
1770 }
1771
1772 int set_memory_4k(unsigned long addr, int numpages)
1773 {
1774         return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
1775                                         __pgprot(0), 1, 0, NULL);
1776 }
1777
1778 static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
1779 {
1780         struct cpa_data cpa;
1781         unsigned long start;
1782         int ret;
1783
1784         /* Nothing to do if memory encryption is not active */
1785         if (!mem_encrypt_active())
1786                 return 0;
1787
1788         /* Should not be working on unaligned addresses */
1789         if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
1790                 addr &= PAGE_MASK;
1791
1792         start = addr;
1793
1794         memset(&cpa, 0, sizeof(cpa));
1795         cpa.vaddr = &addr;
1796         cpa.numpages = numpages;
1797         cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
1798         cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
1799         cpa.pgd = init_mm.pgd;
1800
1801         /* Must avoid aliasing mappings in the highmem code */
1802         kmap_flush_unused();
1803         vm_unmap_aliases();
1804
1805         /*
1806          * Before changing the encryption attribute, we need to flush caches.
1807          */
1808         if (static_cpu_has(X86_FEATURE_CLFLUSH))
1809                 cpa_flush_range(start, numpages, 1);
1810         else
1811                 cpa_flush_all(1);
1812
1813         ret = __change_page_attr_set_clr(&cpa, 1);
1814
1815         /*
1816          * After changing the encryption attribute, we need to flush TLBs
1817          * again in case any speculative TLB caching occurred (but no need
1818          * to flush caches again).  We could just use cpa_flush_all(), but
1819          * in case TLB flushing gets optimized in the cpa_flush_range()
1820          * path use the same logic as above.
1821          */
1822         if (static_cpu_has(X86_FEATURE_CLFLUSH))
1823                 cpa_flush_range(start, numpages, 0);
1824         else
1825                 cpa_flush_all(0);
1826
1827         return ret;
1828 }
1829
1830 int set_memory_encrypted(unsigned long addr, int numpages)
1831 {
1832         return __set_memory_enc_dec(addr, numpages, true);
1833 }
1834 EXPORT_SYMBOL_GPL(set_memory_encrypted);
1835
1836 int set_memory_decrypted(unsigned long addr, int numpages)
1837 {
1838         return __set_memory_enc_dec(addr, numpages, false);
1839 }
1840 EXPORT_SYMBOL_GPL(set_memory_decrypted);
1841
1842 int set_pages_uc(struct page *page, int numpages)
1843 {
1844         unsigned long addr = (unsigned long)page_address(page);
1845
1846         return set_memory_uc(addr, numpages);
1847 }
1848 EXPORT_SYMBOL(set_pages_uc);
1849
1850 static int _set_pages_array(struct page **pages, int addrinarray,
1851                 enum page_cache_mode new_type)
1852 {
1853         unsigned long start;
1854         unsigned long end;
1855         enum page_cache_mode set_type;
1856         int i;
1857         int free_idx;
1858         int ret;
1859
1860         for (i = 0; i < addrinarray; i++) {
1861                 if (PageHighMem(pages[i]))
1862                         continue;
1863                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1864                 end = start + PAGE_SIZE;
1865                 if (reserve_memtype(start, end, new_type, NULL))
1866                         goto err_out;
1867         }
1868
1869         /* If WC, set to UC- first and then WC */
1870         set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
1871                                 _PAGE_CACHE_MODE_UC_MINUS : new_type;
1872
1873         ret = cpa_set_pages_array(pages, addrinarray,
1874                                   cachemode2pgprot(set_type));
1875         if (!ret && new_type == _PAGE_CACHE_MODE_WC)
1876                 ret = change_page_attr_set_clr(NULL, addrinarray,
1877                                                cachemode2pgprot(
1878                                                 _PAGE_CACHE_MODE_WC),
1879                                                __pgprot(_PAGE_CACHE_MASK),
1880                                                0, CPA_PAGES_ARRAY, pages);
1881         if (ret)
1882                 goto err_out;
1883         return 0; /* Success */
1884 err_out:
1885         free_idx = i;
1886         for (i = 0; i < free_idx; i++) {
1887                 if (PageHighMem(pages[i]))
1888                         continue;
1889                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1890                 end = start + PAGE_SIZE;
1891                 free_memtype(start, end);
1892         }
1893         return -EINVAL;
1894 }
1895
1896 int set_pages_array_uc(struct page **pages, int addrinarray)
1897 {
1898         return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
1899 }
1900 EXPORT_SYMBOL(set_pages_array_uc);
1901
1902 int set_pages_array_wc(struct page **pages, int addrinarray)
1903 {
1904         return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WC);
1905 }
1906 EXPORT_SYMBOL(set_pages_array_wc);
1907
1908 int set_pages_array_wt(struct page **pages, int addrinarray)
1909 {
1910         return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WT);
1911 }
1912 EXPORT_SYMBOL_GPL(set_pages_array_wt);
1913
1914 int set_pages_wb(struct page *page, int numpages)
1915 {
1916         unsigned long addr = (unsigned long)page_address(page);
1917
1918         return set_memory_wb(addr, numpages);
1919 }
1920 EXPORT_SYMBOL(set_pages_wb);
1921
1922 int set_pages_array_wb(struct page **pages, int addrinarray)
1923 {
1924         int retval;
1925         unsigned long start;
1926         unsigned long end;
1927         int i;
1928
1929         /* WB cache mode is hard wired to all cache attribute bits being 0 */
1930         retval = cpa_clear_pages_array(pages, addrinarray,
1931                         __pgprot(_PAGE_CACHE_MASK));
1932         if (retval)
1933                 return retval;
1934
1935         for (i = 0; i < addrinarray; i++) {
1936                 if (PageHighMem(pages[i]))
1937                         continue;
1938                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1939                 end = start + PAGE_SIZE;
1940                 free_memtype(start, end);
1941         }
1942
1943         return 0;
1944 }
1945 EXPORT_SYMBOL(set_pages_array_wb);
1946
1947 int set_pages_x(struct page *page, int numpages)
1948 {
1949         unsigned long addr = (unsigned long)page_address(page);
1950
1951         return set_memory_x(addr, numpages);
1952 }
1953 EXPORT_SYMBOL(set_pages_x);
1954
1955 int set_pages_nx(struct page *page, int numpages)
1956 {
1957         unsigned long addr = (unsigned long)page_address(page);
1958
1959         return set_memory_nx(addr, numpages);
1960 }
1961 EXPORT_SYMBOL(set_pages_nx);
1962
1963 int set_pages_ro(struct page *page, int numpages)
1964 {
1965         unsigned long addr = (unsigned long)page_address(page);
1966
1967         return set_memory_ro(addr, numpages);
1968 }
1969
1970 int set_pages_rw(struct page *page, int numpages)
1971 {
1972         unsigned long addr = (unsigned long)page_address(page);
1973
1974         return set_memory_rw(addr, numpages);
1975 }
1976
1977 #ifdef CONFIG_DEBUG_PAGEALLOC
1978
1979 static int __set_pages_p(struct page *page, int numpages)
1980 {
1981         unsigned long tempaddr = (unsigned long) page_address(page);
1982         struct cpa_data cpa = { .vaddr = &tempaddr,
1983                                 .pgd = NULL,
1984                                 .numpages = numpages,
1985                                 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1986                                 .mask_clr = __pgprot(0),
1987                                 .flags = 0};
1988
1989         /*
1990          * No alias checking needed for setting present flag. otherwise,
1991          * we may need to break large pages for 64-bit kernel text
1992          * mappings (this adds to complexity if we want to do this from
1993          * atomic context especially). Let's keep it simple!
1994          */
1995         return __change_page_attr_set_clr(&cpa, 0);
1996 }
1997
1998 static int __set_pages_np(struct page *page, int numpages)
1999 {
2000         unsigned long tempaddr = (unsigned long) page_address(page);
2001         struct cpa_data cpa = { .vaddr = &tempaddr,
2002                                 .pgd = NULL,
2003                                 .numpages = numpages,
2004                                 .mask_set = __pgprot(0),
2005                                 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2006                                 .flags = 0};
2007
2008         /*
2009          * No alias checking needed for setting not present flag. otherwise,
2010          * we may need to break large pages for 64-bit kernel text
2011          * mappings (this adds to complexity if we want to do this from
2012          * atomic context especially). Let's keep it simple!
2013          */
2014         return __change_page_attr_set_clr(&cpa, 0);
2015 }
2016
2017 void __kernel_map_pages(struct page *page, int numpages, int enable)
2018 {
2019         if (PageHighMem(page))
2020                 return;
2021         if (!enable) {
2022                 debug_check_no_locks_freed(page_address(page),
2023                                            numpages * PAGE_SIZE);
2024         }
2025
2026         /*
2027          * The return value is ignored as the calls cannot fail.
2028          * Large pages for identity mappings are not used at boot time
2029          * and hence no memory allocations during large page split.
2030          */
2031         if (enable)
2032                 __set_pages_p(page, numpages);
2033         else
2034                 __set_pages_np(page, numpages);
2035
2036         /*
2037          * We should perform an IPI and flush all tlbs,
2038          * but that can deadlock->flush only current cpu:
2039          */
2040         __flush_tlb_all();
2041
2042         arch_flush_lazy_mmu_mode();
2043 }
2044
2045 #ifdef CONFIG_HIBERNATION
2046
2047 bool kernel_page_present(struct page *page)
2048 {
2049         unsigned int level;
2050         pte_t *pte;
2051
2052         if (PageHighMem(page))
2053                 return false;
2054
2055         pte = lookup_address((unsigned long)page_address(page), &level);
2056         return (pte_val(*pte) & _PAGE_PRESENT);
2057 }
2058
2059 #endif /* CONFIG_HIBERNATION */
2060
2061 #endif /* CONFIG_DEBUG_PAGEALLOC */
2062
2063 int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
2064                             unsigned numpages, unsigned long page_flags)
2065 {
2066         int retval = -EINVAL;
2067
2068         struct cpa_data cpa = {
2069                 .vaddr = &address,
2070                 .pfn = pfn,
2071                 .pgd = pgd,
2072                 .numpages = numpages,
2073                 .mask_set = __pgprot(0),
2074                 .mask_clr = __pgprot(0),
2075                 .flags = 0,
2076         };
2077
2078         if (!(__supported_pte_mask & _PAGE_NX))
2079                 goto out;
2080
2081         if (!(page_flags & _PAGE_NX))
2082                 cpa.mask_clr = __pgprot(_PAGE_NX);
2083
2084         if (!(page_flags & _PAGE_RW))
2085                 cpa.mask_clr = __pgprot(_PAGE_RW);
2086
2087         if (!(page_flags & _PAGE_ENC))
2088                 cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);
2089
2090         cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
2091
2092         retval = __change_page_attr_set_clr(&cpa, 0);
2093         __flush_tlb_all();
2094
2095 out:
2096         return retval;
2097 }
2098
2099 /*
2100  * The testcases use internal knowledge of the implementation that shouldn't
2101  * be exposed to the rest of the kernel. Include these directly here.
2102  */
2103 #ifdef CONFIG_CPA_DEBUG
2104 #include "pageattr-test.c"
2105 #endif