x86: cpa: simplify locking
[sfrench/cifs-2.6.git] / arch / x86 / mm / pageattr_32.c
1 /*
2  * Copyright 2002 Andi Kleen, SuSE Labs.
3  * Thanks to Ben LaHaise for precious feedback.
4  */
5
6 #include <linux/highmem.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/slab.h>
10 #include <linux/mm.h>
11
12 #include <asm/processor.h>
13 #include <asm/tlbflush.h>
14 #include <asm/sections.h>
15 #include <asm/uaccess.h>
16 #include <asm/pgalloc.h>
17
18 pte_t *lookup_address(unsigned long address, int *level)
19 {
20         pgd_t *pgd = pgd_offset_k(address);
21         pud_t *pud;
22         pmd_t *pmd;
23
24         if (pgd_none(*pgd))
25                 return NULL;
26         pud = pud_offset(pgd, address);
27         if (pud_none(*pud))
28                 return NULL;
29         pmd = pmd_offset(pud, address);
30         if (pmd_none(*pmd))
31                 return NULL;
32         *level = 2;
33         if (pmd_large(*pmd))
34                 return (pte_t *)pmd;
35         *level = 3;
36
37         return pte_offset_kernel(pmd, address);
38 }
39
40 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
41 {
42         struct page *page;
43
44         /* change init_mm */
45         set_pte_atomic(kpte, pte);
46         if (SHARED_KERNEL_PMD)
47                 return;
48
49         for (page = pgd_list; page; page = (struct page *)page->index) {
50                 pgd_t *pgd;
51                 pud_t *pud;
52                 pmd_t *pmd;
53
54                 pgd = (pgd_t *)page_address(page) + pgd_index(address);
55                 pud = pud_offset(pgd, address);
56                 pmd = pmd_offset(pud, address);
57                 set_pte_atomic((pte_t *)pmd, pte);
58         }
59 }
60
61 static int split_large_page(pte_t *kpte, unsigned long address)
62 {
63         pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
64         unsigned long flags;
65         unsigned long addr;
66         pte_t *pbase, *tmp;
67         struct page *base;
68         int i, level;
69
70         base = alloc_pages(GFP_KERNEL, 0);
71         if (!base)
72                 return -ENOMEM;
73
74         spin_lock_irqsave(&pgd_lock, flags);
75         /*
76          * Check for races, another CPU might have split this page
77          * up for us already:
78          */
79         tmp = lookup_address(address, &level);
80         if (tmp != kpte) {
81                 WARN_ON_ONCE(1);
82                 goto out_unlock;
83         }
84
85         address = __pa(address);
86         addr = address & LARGE_PAGE_MASK;
87         pbase = (pte_t *)page_address(base);
88         paravirt_alloc_pt(&init_mm, page_to_pfn(base));
89
90         for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE)
91                 set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot));
92
93         /*
94          * Install the new, split up pagetable:
95          */
96         __set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
97         base = NULL;
98
99 out_unlock:
100         spin_unlock_irqrestore(&pgd_lock, flags);
101
102         if (base)
103                 __free_pages(base, 0);
104
105         return 0;
106 }
107
108 static int __change_page_attr(struct page *page, pgprot_t prot)
109 {
110         struct page *kpte_page;
111         unsigned long address;
112         int level, err = 0;
113         pte_t *kpte;
114
115         BUG_ON(PageHighMem(page));
116         address = (unsigned long)page_address(page);
117
118 repeat:
119         kpte = lookup_address(address, &level);
120         if (!kpte)
121                 return -EINVAL;
122
123         kpte_page = virt_to_page(kpte);
124         BUG_ON(PageLRU(kpte_page));
125         BUG_ON(PageCompound(kpte_page));
126
127         /*
128          * Better fail early if someone sets the kernel text to NX.
129          * Does not cover __inittext
130          */
131         BUG_ON(address >= (unsigned long)&_text &&
132                 address < (unsigned long)&_etext &&
133                (pgprot_val(prot) & _PAGE_NX));
134
135         if (level == 3) {
136                 set_pte_atomic(kpte, mk_pte(page, canon_pgprot(prot)));
137         } else {
138                 err = split_large_page(kpte, address);
139                 if (!err)
140                         goto repeat;
141         }
142         return err;
143 }
144
145 /*
146  * Change the page attributes of an page in the linear mapping.
147  *
148  * This should be used when a page is mapped with a different caching policy
149  * than write-back somewhere - some CPUs do not like it when mappings with
150  * different caching policies exist. This changes the page attributes of the
151  * in kernel linear mapping too.
152  *
153  * The caller needs to ensure that there are no conflicting mappings elsewhere.
154  * This function only deals with the kernel linear map.
155  *
156  * Caller must call global_flush_tlb() after this.
157  */
158 int change_page_attr(struct page *page, int numpages, pgprot_t prot)
159 {
160         int err = 0, i;
161
162         for (i = 0; i < numpages; i++, page++) {
163                 err = __change_page_attr(page, prot);
164                 if (err)
165                         break;
166         }
167
168         return err;
169 }
170 EXPORT_SYMBOL(change_page_attr);
171
172 int change_page_attr_addr(unsigned long addr, int numpages, pgprot_t prot)
173 {
174         int i;
175         unsigned long pfn = (__pa(addr) >> PAGE_SHIFT);
176
177         for (i = 0; i < numpages; i++) {
178                 if (!pfn_valid(pfn + i)) {
179                         WARN_ON_ONCE(1);
180                         break;
181                 } else {
182                         int level;
183                         pte_t *pte = lookup_address(addr + i*PAGE_SIZE, &level);
184                         BUG_ON(pte && pte_none(*pte));
185                 }
186         }
187
188         return change_page_attr(virt_to_page(addr), i, prot);
189 }
190
191 static void flush_kernel_map(void *arg)
192 {
193         /*
194          * Flush all to work around Errata in early athlons regarding
195          * large page flushing.
196          */
197         __flush_tlb_all();
198
199         if (boot_cpu_data.x86_model >= 4)
200                 wbinvd();
201 }
202
203 void global_flush_tlb(void)
204 {
205         BUG_ON(irqs_disabled());
206
207         on_each_cpu(flush_kernel_map, NULL, 1, 1);
208 }
209 EXPORT_SYMBOL(global_flush_tlb);
210
211 #ifdef CONFIG_DEBUG_PAGEALLOC
212 void kernel_map_pages(struct page *page, int numpages, int enable)
213 {
214         if (PageHighMem(page))
215                 return;
216         if (!enable) {
217                 debug_check_no_locks_freed(page_address(page),
218                                            numpages * PAGE_SIZE);
219         }
220
221         /*
222          * the return value is ignored - the calls cannot fail,
223          * large pages are disabled at boot time.
224          */
225         change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0));
226
227         /*
228          * we should perform an IPI and flush all tlbs,
229          * but that can deadlock->flush only current cpu.
230          */
231         __flush_tlb_all();
232 }
233 #endif