x86: fix clflush_page_range logic
[sfrench/cifs-2.6.git] / arch / x86 / mm / pageattr.c
1 /*
2  * Copyright 2002 Andi Kleen, SuSE Labs.
3  * Thanks to Ben LaHaise for precious feedback.
4  */
5 #include <linux/highmem.h>
6 #include <linux/bootmem.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/slab.h>
10 #include <linux/mm.h>
11
12 #include <asm/e820.h>
13 #include <asm/processor.h>
14 #include <asm/tlbflush.h>
15 #include <asm/sections.h>
16 #include <asm/uaccess.h>
17 #include <asm/pgalloc.h>
18
19 static inline int
20 within(unsigned long addr, unsigned long start, unsigned long end)
21 {
22         return addr >= start && addr < end;
23 }
24
25 /*
26  * Flushing functions
27  */
28
29 /**
30  * clflush_cache_range - flush a cache range with clflush
31  * @addr:       virtual start address
32  * @size:       number of bytes to flush
33  *
34  * clflush is an unordered instruction which needs fencing with mfence
35  * to avoid ordering issues.
36  */
37 void clflush_cache_range(void *vaddr, unsigned int size)
38 {
39         void *vend = vaddr + size - 1;
40
41         mb();
42
43         for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size)
44                 clflush(vaddr);
45         /*
46          * Flush any possible final partial cacheline:
47          */
48         clflush(vend);
49
50         mb();
51 }
52
53 static void __cpa_flush_all(void *arg)
54 {
55         /*
56          * Flush all to work around Errata in early athlons regarding
57          * large page flushing.
58          */
59         __flush_tlb_all();
60
61         if (boot_cpu_data.x86_model >= 4)
62                 wbinvd();
63 }
64
65 static void cpa_flush_all(void)
66 {
67         BUG_ON(irqs_disabled());
68
69         on_each_cpu(__cpa_flush_all, NULL, 1, 1);
70 }
71
72 static void __cpa_flush_range(void *arg)
73 {
74         /*
75          * We could optimize that further and do individual per page
76          * tlb invalidates for a low number of pages. Caveat: we must
77          * flush the high aliases on 64bit as well.
78          */
79         __flush_tlb_all();
80 }
81
82 static void cpa_flush_range(unsigned long start, int numpages)
83 {
84         unsigned int i, level;
85         unsigned long addr;
86
87         BUG_ON(irqs_disabled());
88         WARN_ON(PAGE_ALIGN(start) != start);
89
90         on_each_cpu(__cpa_flush_range, NULL, 1, 1);
91
92         /*
93          * We only need to flush on one CPU,
94          * clflush is a MESI-coherent instruction that
95          * will cause all other CPUs to flush the same
96          * cachelines:
97          */
98         for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
99                 pte_t *pte = lookup_address(addr, &level);
100
101                 /*
102                  * Only flush present addresses:
103                  */
104                 if (pte && pte_present(*pte))
105                         clflush_cache_range((void *) addr, PAGE_SIZE);
106         }
107 }
108
109 /*
110  * Certain areas of memory on x86 require very specific protection flags,
111  * for example the BIOS area or kernel text. Callers don't always get this
112  * right (again, ioremap() on BIOS memory is not uncommon) so this function
113  * checks and fixes these known static required protection bits.
114  */
115 static inline pgprot_t static_protections(pgprot_t prot, unsigned long address)
116 {
117         pgprot_t forbidden = __pgprot(0);
118
119         /*
120          * The BIOS area between 640k and 1Mb needs to be executable for
121          * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
122          */
123         if (within(__pa(address), BIOS_BEGIN, BIOS_END))
124                 pgprot_val(forbidden) |= _PAGE_NX;
125
126         /*
127          * The kernel text needs to be executable for obvious reasons
128          * Does not cover __inittext since that is gone later on
129          */
130         if (within(address, (unsigned long)_text, (unsigned long)_etext))
131                 pgprot_val(forbidden) |= _PAGE_NX;
132
133 #ifdef CONFIG_DEBUG_RODATA
134         /* The .rodata section needs to be read-only */
135         if (within(address, (unsigned long)__start_rodata,
136                                 (unsigned long)__end_rodata))
137                 pgprot_val(forbidden) |= _PAGE_RW;
138 #endif
139
140         prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
141
142         return prot;
143 }
144
145 pte_t *lookup_address(unsigned long address, int *level)
146 {
147         pgd_t *pgd = pgd_offset_k(address);
148         pud_t *pud;
149         pmd_t *pmd;
150
151         *level = PG_LEVEL_NONE;
152
153         if (pgd_none(*pgd))
154                 return NULL;
155         pud = pud_offset(pgd, address);
156         if (pud_none(*pud))
157                 return NULL;
158         pmd = pmd_offset(pud, address);
159         if (pmd_none(*pmd))
160                 return NULL;
161
162         *level = PG_LEVEL_2M;
163         if (pmd_large(*pmd))
164                 return (pte_t *)pmd;
165
166         *level = PG_LEVEL_4K;
167         return pte_offset_kernel(pmd, address);
168 }
169
170 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
171 {
172         /* change init_mm */
173         set_pte_atomic(kpte, pte);
174 #ifdef CONFIG_X86_32
175         if (!SHARED_KERNEL_PMD) {
176                 struct page *page;
177
178                 for (page = pgd_list; page; page = (struct page *)page->index) {
179                         pgd_t *pgd;
180                         pud_t *pud;
181                         pmd_t *pmd;
182
183                         pgd = (pgd_t *)page_address(page) + pgd_index(address);
184                         pud = pud_offset(pgd, address);
185                         pmd = pmd_offset(pud, address);
186                         set_pte_atomic((pte_t *)pmd, pte);
187                 }
188         }
189 #endif
190 }
191
192 static int split_large_page(pte_t *kpte, unsigned long address)
193 {
194         pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
195         gfp_t gfp_flags = GFP_KERNEL;
196         unsigned long flags;
197         unsigned long addr;
198         pte_t *pbase, *tmp;
199         struct page *base;
200         int i, level;
201
202 #ifdef CONFIG_DEBUG_PAGEALLOC
203         gfp_flags = GFP_ATOMIC;
204 #endif
205         base = alloc_pages(gfp_flags, 0);
206         if (!base)
207                 return -ENOMEM;
208
209         spin_lock_irqsave(&pgd_lock, flags);
210         /*
211          * Check for races, another CPU might have split this page
212          * up for us already:
213          */
214         tmp = lookup_address(address, &level);
215         if (tmp != kpte) {
216                 WARN_ON_ONCE(1);
217                 goto out_unlock;
218         }
219
220         address = __pa(address);
221         addr = address & LARGE_PAGE_MASK;
222         pbase = (pte_t *)page_address(base);
223 #ifdef CONFIG_X86_32
224         paravirt_alloc_pt(&init_mm, page_to_pfn(base));
225 #endif
226
227         for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE)
228                 set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot));
229
230         /*
231          * Install the new, split up pagetable. Important detail here:
232          *
233          * On Intel the NX bit of all levels must be cleared to make a
234          * page executable. See section 4.13.2 of Intel 64 and IA-32
235          * Architectures Software Developer's Manual).
236          */
237         ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte)));
238         __set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
239         base = NULL;
240
241 out_unlock:
242         spin_unlock_irqrestore(&pgd_lock, flags);
243
244         if (base)
245                 __free_pages(base, 0);
246
247         return 0;
248 }
249
250 static int
251 __change_page_attr(unsigned long address, unsigned long pfn, pgprot_t prot)
252 {
253         struct page *kpte_page;
254         int level, err = 0;
255         pte_t *kpte;
256
257 #ifdef CONFIG_X86_32
258         BUG_ON(pfn > max_low_pfn);
259 #endif
260
261 repeat:
262         kpte = lookup_address(address, &level);
263         if (!kpte)
264                 return -EINVAL;
265
266         kpte_page = virt_to_page(kpte);
267         BUG_ON(PageLRU(kpte_page));
268         BUG_ON(PageCompound(kpte_page));
269
270         prot = static_protections(prot, address);
271
272         if (level == PG_LEVEL_4K) {
273                 WARN_ON_ONCE(pgprot_val(prot) & _PAGE_PSE);
274                 set_pte_atomic(kpte, pfn_pte(pfn, canon_pgprot(prot)));
275         } else {
276                 /* Clear the PSE bit for the 4k level pages ! */
277                 pgprot_val(prot) = pgprot_val(prot) & ~_PAGE_PSE;
278
279                 err = split_large_page(kpte, address);
280                 if (!err)
281                         goto repeat;
282         }
283         return err;
284 }
285
286 /**
287  * change_page_attr_addr - Change page table attributes in linear mapping
288  * @address: Virtual address in linear mapping.
289  * @prot:    New page table attribute (PAGE_*)
290  *
291  * Change page attributes of a page in the direct mapping. This is a variant
292  * of change_page_attr() that also works on memory holes that do not have
293  * mem_map entry (pfn_valid() is false).
294  *
295  * See change_page_attr() documentation for more details.
296  *
297  * Modules and drivers should use the set_memory_* APIs instead.
298  */
299
300 static int change_page_attr_addr(unsigned long address, pgprot_t prot)
301 {
302         int err = 0, kernel_map = 0;
303         unsigned long pfn = __pa(address) >> PAGE_SHIFT;
304
305 #ifdef CONFIG_X86_64
306         if (address >= __START_KERNEL_map &&
307                         address < __START_KERNEL_map + KERNEL_TEXT_SIZE) {
308
309                 address = (unsigned long)__va(__pa(address));
310                 kernel_map = 1;
311         }
312 #endif
313
314         if (!kernel_map || pte_present(pfn_pte(0, prot))) {
315                 err = __change_page_attr(address, pfn, prot);
316                 if (err)
317                         return err;
318         }
319
320 #ifdef CONFIG_X86_64
321         /*
322          * Handle kernel mapping too which aliases part of
323          * lowmem:
324          */
325         if (__pa(address) < KERNEL_TEXT_SIZE) {
326                 unsigned long addr2;
327                 pgprot_t prot2;
328
329                 addr2 = __START_KERNEL_map + __pa(address);
330                 /* Make sure the kernel mappings stay executable */
331                 prot2 = pte_pgprot(pte_mkexec(pfn_pte(0, prot)));
332                 err = __change_page_attr(addr2, pfn, prot2);
333         }
334 #endif
335
336         return err;
337 }
338
339 static int __change_page_attr_set_clr(unsigned long addr, int numpages,
340                                       pgprot_t mask_set, pgprot_t mask_clr)
341 {
342         pgprot_t new_prot;
343         int level;
344         pte_t *pte;
345         int i, ret;
346
347         for (i = 0; i < numpages ; i++) {
348
349                 pte = lookup_address(addr, &level);
350                 if (!pte)
351                         return -EINVAL;
352
353                 new_prot = pte_pgprot(*pte);
354
355                 pgprot_val(new_prot) &= ~pgprot_val(mask_clr);
356                 pgprot_val(new_prot) |= pgprot_val(mask_set);
357
358                 ret = change_page_attr_addr(addr, new_prot);
359                 if (ret)
360                         return ret;
361                 addr += PAGE_SIZE;
362         }
363
364         return 0;
365 }
366
367 static int change_page_attr_set_clr(unsigned long addr, int numpages,
368                                     pgprot_t mask_set, pgprot_t mask_clr)
369 {
370         int ret = __change_page_attr_set_clr(addr, numpages, mask_set,
371                                              mask_clr);
372
373         /*
374          * On success we use clflush, when the CPU supports it to
375          * avoid the wbindv. If the CPU does not support it and in the
376          * error case we fall back to cpa_flush_all (which uses
377          * wbindv):
378          */
379         if (!ret && cpu_has_clflush)
380                 cpa_flush_range(addr, numpages);
381         else
382                 cpa_flush_all();
383
384         return ret;
385 }
386
387 static inline int change_page_attr_set(unsigned long addr, int numpages,
388                                        pgprot_t mask)
389 {
390         return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0));
391 }
392
393 static inline int change_page_attr_clear(unsigned long addr, int numpages,
394                                          pgprot_t mask)
395 {
396         return __change_page_attr_set_clr(addr, numpages, __pgprot(0), mask);
397
398 }
399
400 int set_memory_uc(unsigned long addr, int numpages)
401 {
402         return change_page_attr_set(addr, numpages,
403                                     __pgprot(_PAGE_PCD | _PAGE_PWT));
404 }
405 EXPORT_SYMBOL(set_memory_uc);
406
407 int set_memory_wb(unsigned long addr, int numpages)
408 {
409         return change_page_attr_clear(addr, numpages,
410                                       __pgprot(_PAGE_PCD | _PAGE_PWT));
411 }
412 EXPORT_SYMBOL(set_memory_wb);
413
414 int set_memory_x(unsigned long addr, int numpages)
415 {
416         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_NX));
417 }
418 EXPORT_SYMBOL(set_memory_x);
419
420 int set_memory_nx(unsigned long addr, int numpages)
421 {
422         return change_page_attr_set(addr, numpages, __pgprot(_PAGE_NX));
423 }
424 EXPORT_SYMBOL(set_memory_nx);
425
426 int set_memory_ro(unsigned long addr, int numpages)
427 {
428         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_RW));
429 }
430
431 int set_memory_rw(unsigned long addr, int numpages)
432 {
433         return change_page_attr_set(addr, numpages, __pgprot(_PAGE_RW));
434 }
435
436 int set_memory_np(unsigned long addr, int numpages)
437 {
438         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_PRESENT));
439 }
440
441 int set_pages_uc(struct page *page, int numpages)
442 {
443         unsigned long addr = (unsigned long)page_address(page);
444
445         return set_memory_uc(addr, numpages);
446 }
447 EXPORT_SYMBOL(set_pages_uc);
448
449 int set_pages_wb(struct page *page, int numpages)
450 {
451         unsigned long addr = (unsigned long)page_address(page);
452
453         return set_memory_wb(addr, numpages);
454 }
455 EXPORT_SYMBOL(set_pages_wb);
456
457 int set_pages_x(struct page *page, int numpages)
458 {
459         unsigned long addr = (unsigned long)page_address(page);
460
461         return set_memory_x(addr, numpages);
462 }
463 EXPORT_SYMBOL(set_pages_x);
464
465 int set_pages_nx(struct page *page, int numpages)
466 {
467         unsigned long addr = (unsigned long)page_address(page);
468
469         return set_memory_nx(addr, numpages);
470 }
471 EXPORT_SYMBOL(set_pages_nx);
472
473 int set_pages_ro(struct page *page, int numpages)
474 {
475         unsigned long addr = (unsigned long)page_address(page);
476
477         return set_memory_ro(addr, numpages);
478 }
479
480 int set_pages_rw(struct page *page, int numpages)
481 {
482         unsigned long addr = (unsigned long)page_address(page);
483
484         return set_memory_rw(addr, numpages);
485 }
486
487
488 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_CPA_DEBUG)
489 static inline int __change_page_attr_set(unsigned long addr, int numpages,
490                                          pgprot_t mask)
491 {
492         return __change_page_attr_set_clr(addr, numpages, mask, __pgprot(0));
493 }
494
495 static inline int __change_page_attr_clear(unsigned long addr, int numpages,
496                                            pgprot_t mask)
497 {
498         return __change_page_attr_set_clr(addr, numpages, __pgprot(0), mask);
499 }
500 #endif
501
502 #ifdef CONFIG_DEBUG_PAGEALLOC
503
504 static int __set_pages_p(struct page *page, int numpages)
505 {
506         unsigned long addr = (unsigned long)page_address(page);
507
508         return __change_page_attr_set(addr, numpages,
509                                       __pgprot(_PAGE_PRESENT | _PAGE_RW));
510 }
511
512 static int __set_pages_np(struct page *page, int numpages)
513 {
514         unsigned long addr = (unsigned long)page_address(page);
515
516         return __change_page_attr_clear(addr, numpages,
517                                         __pgprot(_PAGE_PRESENT));
518 }
519
520 void kernel_map_pages(struct page *page, int numpages, int enable)
521 {
522         if (PageHighMem(page))
523                 return;
524         if (!enable) {
525                 debug_check_no_locks_freed(page_address(page),
526                                            numpages * PAGE_SIZE);
527         }
528
529         /*
530          * If page allocator is not up yet then do not call c_p_a():
531          */
532         if (!debug_pagealloc_enabled)
533                 return;
534
535         /*
536          * The return value is ignored - the calls cannot fail,
537          * large pages are disabled at boot time:
538          */
539         if (enable)
540                 __set_pages_p(page, numpages);
541         else
542                 __set_pages_np(page, numpages);
543
544         /*
545          * We should perform an IPI and flush all tlbs,
546          * but that can deadlock->flush only current cpu:
547          */
548         __flush_tlb_all();
549 }
550 #endif
551
552 /*
553  * The testcases use internal knowledge of the implementation that shouldn't
554  * be exposed to the rest of the kernel. Include these directly here.
555  */
556 #ifdef CONFIG_CPA_DEBUG
557 #include "pageattr-test.c"
558 #endif