Merge branches 'arm/rockchip', 'arm/exynos', 'arm/smmu', 'x86/vt-d', 'x86/amd', ...
[sfrench/cifs-2.6.git] / arch / x86 / kvm / mmu.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * MMU support
8  *
9  * Copyright (C) 2006 Qumranet, Inc.
10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11  *
12  * Authors:
13  *   Yaniv Kamay  <yaniv@qumranet.com>
14  *   Avi Kivity   <avi@qumranet.com>
15  *
16  * This work is licensed under the terms of the GNU GPL, version 2.  See
17  * the COPYING file in the top-level directory.
18  *
19  */
20
21 #include "irq.h"
22 #include "mmu.h"
23 #include "x86.h"
24 #include "kvm_cache_regs.h"
25 #include "cpuid.h"
26
27 #include <linux/kvm_host.h>
28 #include <linux/types.h>
29 #include <linux/string.h>
30 #include <linux/mm.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/swap.h>
34 #include <linux/hugetlb.h>
35 #include <linux/compiler.h>
36 #include <linux/srcu.h>
37 #include <linux/slab.h>
38 #include <linux/uaccess.h>
39
40 #include <asm/page.h>
41 #include <asm/cmpxchg.h>
42 #include <asm/io.h>
43 #include <asm/vmx.h>
44
45 /*
46  * When setting this variable to true it enables Two-Dimensional-Paging
47  * where the hardware walks 2 page tables:
48  * 1. the guest-virtual to guest-physical
49  * 2. while doing 1. it walks guest-physical to host-physical
50  * If the hardware supports that we don't need to do shadow paging.
51  */
52 bool tdp_enabled = false;
53
54 enum {
55         AUDIT_PRE_PAGE_FAULT,
56         AUDIT_POST_PAGE_FAULT,
57         AUDIT_PRE_PTE_WRITE,
58         AUDIT_POST_PTE_WRITE,
59         AUDIT_PRE_SYNC,
60         AUDIT_POST_SYNC
61 };
62
63 #undef MMU_DEBUG
64
65 #ifdef MMU_DEBUG
66 static bool dbg = 0;
67 module_param(dbg, bool, 0644);
68
69 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
70 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
71 #define MMU_WARN_ON(x) WARN_ON(x)
72 #else
73 #define pgprintk(x...) do { } while (0)
74 #define rmap_printk(x...) do { } while (0)
75 #define MMU_WARN_ON(x) do { } while (0)
76 #endif
77
78 #define PTE_PREFETCH_NUM                8
79
80 #define PT_FIRST_AVAIL_BITS_SHIFT 10
81 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
82
83 #define PT64_LEVEL_BITS 9
84
85 #define PT64_LEVEL_SHIFT(level) \
86                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
87
88 #define PT64_INDEX(address, level)\
89         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
90
91
92 #define PT32_LEVEL_BITS 10
93
94 #define PT32_LEVEL_SHIFT(level) \
95                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
96
97 #define PT32_LVL_OFFSET_MASK(level) \
98         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
99                                                 * PT32_LEVEL_BITS))) - 1))
100
101 #define PT32_INDEX(address, level)\
102         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
103
104
105 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
106 #define PT64_DIR_BASE_ADDR_MASK \
107         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
108 #define PT64_LVL_ADDR_MASK(level) \
109         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
110                                                 * PT64_LEVEL_BITS))) - 1))
111 #define PT64_LVL_OFFSET_MASK(level) \
112         (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
113                                                 * PT64_LEVEL_BITS))) - 1))
114
115 #define PT32_BASE_ADDR_MASK PAGE_MASK
116 #define PT32_DIR_BASE_ADDR_MASK \
117         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
118 #define PT32_LVL_ADDR_MASK(level) \
119         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
120                                             * PT32_LEVEL_BITS))) - 1))
121
122 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
123                         | shadow_x_mask | shadow_nx_mask)
124
125 #define ACC_EXEC_MASK    1
126 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
127 #define ACC_USER_MASK    PT_USER_MASK
128 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
129
130 #include <trace/events/kvm.h>
131
132 #define CREATE_TRACE_POINTS
133 #include "mmutrace.h"
134
135 #define SPTE_HOST_WRITEABLE     (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
136 #define SPTE_MMU_WRITEABLE      (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
137
138 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
139
140 /* make pte_list_desc fit well in cache line */
141 #define PTE_LIST_EXT 3
142
143 struct pte_list_desc {
144         u64 *sptes[PTE_LIST_EXT];
145         struct pte_list_desc *more;
146 };
147
148 struct kvm_shadow_walk_iterator {
149         u64 addr;
150         hpa_t shadow_addr;
151         u64 *sptep;
152         int level;
153         unsigned index;
154 };
155
156 #define for_each_shadow_entry(_vcpu, _addr, _walker)    \
157         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
158              shadow_walk_okay(&(_walker));                      \
159              shadow_walk_next(&(_walker)))
160
161 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)     \
162         for (shadow_walk_init(&(_walker), _vcpu, _addr);                \
163              shadow_walk_okay(&(_walker)) &&                            \
164                 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
165              __shadow_walk_next(&(_walker), spte))
166
167 static struct kmem_cache *pte_list_desc_cache;
168 static struct kmem_cache *mmu_page_header_cache;
169 static struct percpu_counter kvm_total_used_mmu_pages;
170
171 static u64 __read_mostly shadow_nx_mask;
172 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
173 static u64 __read_mostly shadow_user_mask;
174 static u64 __read_mostly shadow_accessed_mask;
175 static u64 __read_mostly shadow_dirty_mask;
176 static u64 __read_mostly shadow_mmio_mask;
177
178 static void mmu_spte_set(u64 *sptep, u64 spte);
179 static void mmu_free_roots(struct kvm_vcpu *vcpu);
180
181 void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
182 {
183         shadow_mmio_mask = mmio_mask;
184 }
185 EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
186
187 /*
188  * the low bit of the generation number is always presumed to be zero.
189  * This disables mmio caching during memslot updates.  The concept is
190  * similar to a seqcount but instead of retrying the access we just punt
191  * and ignore the cache.
192  *
193  * spte bits 3-11 are used as bits 1-9 of the generation number,
194  * the bits 52-61 are used as bits 10-19 of the generation number.
195  */
196 #define MMIO_SPTE_GEN_LOW_SHIFT         2
197 #define MMIO_SPTE_GEN_HIGH_SHIFT        52
198
199 #define MMIO_GEN_SHIFT                  20
200 #define MMIO_GEN_LOW_SHIFT              10
201 #define MMIO_GEN_LOW_MASK               ((1 << MMIO_GEN_LOW_SHIFT) - 2)
202 #define MMIO_GEN_MASK                   ((1 << MMIO_GEN_SHIFT) - 1)
203
204 static u64 generation_mmio_spte_mask(unsigned int gen)
205 {
206         u64 mask;
207
208         WARN_ON(gen & ~MMIO_GEN_MASK);
209
210         mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
211         mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
212         return mask;
213 }
214
215 static unsigned int get_mmio_spte_generation(u64 spte)
216 {
217         unsigned int gen;
218
219         spte &= ~shadow_mmio_mask;
220
221         gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
222         gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
223         return gen;
224 }
225
226 static unsigned int kvm_current_mmio_generation(struct kvm *kvm)
227 {
228         return kvm_memslots(kvm)->generation & MMIO_GEN_MASK;
229 }
230
231 static void mark_mmio_spte(struct kvm *kvm, u64 *sptep, u64 gfn,
232                            unsigned access)
233 {
234         unsigned int gen = kvm_current_mmio_generation(kvm);
235         u64 mask = generation_mmio_spte_mask(gen);
236
237         access &= ACC_WRITE_MASK | ACC_USER_MASK;
238         mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
239
240         trace_mark_mmio_spte(sptep, gfn, access, gen);
241         mmu_spte_set(sptep, mask);
242 }
243
244 static bool is_mmio_spte(u64 spte)
245 {
246         return (spte & shadow_mmio_mask) == shadow_mmio_mask;
247 }
248
249 static gfn_t get_mmio_spte_gfn(u64 spte)
250 {
251         u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
252         return (spte & ~mask) >> PAGE_SHIFT;
253 }
254
255 static unsigned get_mmio_spte_access(u64 spte)
256 {
257         u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
258         return (spte & ~mask) & ~PAGE_MASK;
259 }
260
261 static bool set_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
262                           pfn_t pfn, unsigned access)
263 {
264         if (unlikely(is_noslot_pfn(pfn))) {
265                 mark_mmio_spte(kvm, sptep, gfn, access);
266                 return true;
267         }
268
269         return false;
270 }
271
272 static bool check_mmio_spte(struct kvm *kvm, u64 spte)
273 {
274         unsigned int kvm_gen, spte_gen;
275
276         kvm_gen = kvm_current_mmio_generation(kvm);
277         spte_gen = get_mmio_spte_generation(spte);
278
279         trace_check_mmio_spte(spte, kvm_gen, spte_gen);
280         return likely(kvm_gen == spte_gen);
281 }
282
283 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
284                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
285 {
286         shadow_user_mask = user_mask;
287         shadow_accessed_mask = accessed_mask;
288         shadow_dirty_mask = dirty_mask;
289         shadow_nx_mask = nx_mask;
290         shadow_x_mask = x_mask;
291 }
292 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
293
294 static int is_cpuid_PSE36(void)
295 {
296         return 1;
297 }
298
299 static int is_nx(struct kvm_vcpu *vcpu)
300 {
301         return vcpu->arch.efer & EFER_NX;
302 }
303
304 static int is_shadow_present_pte(u64 pte)
305 {
306         return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
307 }
308
309 static int is_large_pte(u64 pte)
310 {
311         return pte & PT_PAGE_SIZE_MASK;
312 }
313
314 static int is_rmap_spte(u64 pte)
315 {
316         return is_shadow_present_pte(pte);
317 }
318
319 static int is_last_spte(u64 pte, int level)
320 {
321         if (level == PT_PAGE_TABLE_LEVEL)
322                 return 1;
323         if (is_large_pte(pte))
324                 return 1;
325         return 0;
326 }
327
328 static pfn_t spte_to_pfn(u64 pte)
329 {
330         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
331 }
332
333 static gfn_t pse36_gfn_delta(u32 gpte)
334 {
335         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
336
337         return (gpte & PT32_DIR_PSE36_MASK) << shift;
338 }
339
340 #ifdef CONFIG_X86_64
341 static void __set_spte(u64 *sptep, u64 spte)
342 {
343         *sptep = spte;
344 }
345
346 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
347 {
348         *sptep = spte;
349 }
350
351 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
352 {
353         return xchg(sptep, spte);
354 }
355
356 static u64 __get_spte_lockless(u64 *sptep)
357 {
358         return ACCESS_ONCE(*sptep);
359 }
360
361 static bool __check_direct_spte_mmio_pf(u64 spte)
362 {
363         /* It is valid if the spte is zapped. */
364         return spte == 0ull;
365 }
366 #else
367 union split_spte {
368         struct {
369                 u32 spte_low;
370                 u32 spte_high;
371         };
372         u64 spte;
373 };
374
375 static void count_spte_clear(u64 *sptep, u64 spte)
376 {
377         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
378
379         if (is_shadow_present_pte(spte))
380                 return;
381
382         /* Ensure the spte is completely set before we increase the count */
383         smp_wmb();
384         sp->clear_spte_count++;
385 }
386
387 static void __set_spte(u64 *sptep, u64 spte)
388 {
389         union split_spte *ssptep, sspte;
390
391         ssptep = (union split_spte *)sptep;
392         sspte = (union split_spte)spte;
393
394         ssptep->spte_high = sspte.spte_high;
395
396         /*
397          * If we map the spte from nonpresent to present, We should store
398          * the high bits firstly, then set present bit, so cpu can not
399          * fetch this spte while we are setting the spte.
400          */
401         smp_wmb();
402
403         ssptep->spte_low = sspte.spte_low;
404 }
405
406 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
407 {
408         union split_spte *ssptep, sspte;
409
410         ssptep = (union split_spte *)sptep;
411         sspte = (union split_spte)spte;
412
413         ssptep->spte_low = sspte.spte_low;
414
415         /*
416          * If we map the spte from present to nonpresent, we should clear
417          * present bit firstly to avoid vcpu fetch the old high bits.
418          */
419         smp_wmb();
420
421         ssptep->spte_high = sspte.spte_high;
422         count_spte_clear(sptep, spte);
423 }
424
425 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
426 {
427         union split_spte *ssptep, sspte, orig;
428
429         ssptep = (union split_spte *)sptep;
430         sspte = (union split_spte)spte;
431
432         /* xchg acts as a barrier before the setting of the high bits */
433         orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
434         orig.spte_high = ssptep->spte_high;
435         ssptep->spte_high = sspte.spte_high;
436         count_spte_clear(sptep, spte);
437
438         return orig.spte;
439 }
440
441 /*
442  * The idea using the light way get the spte on x86_32 guest is from
443  * gup_get_pte(arch/x86/mm/gup.c).
444  *
445  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
446  * coalesces them and we are running out of the MMU lock.  Therefore
447  * we need to protect against in-progress updates of the spte.
448  *
449  * Reading the spte while an update is in progress may get the old value
450  * for the high part of the spte.  The race is fine for a present->non-present
451  * change (because the high part of the spte is ignored for non-present spte),
452  * but for a present->present change we must reread the spte.
453  *
454  * All such changes are done in two steps (present->non-present and
455  * non-present->present), hence it is enough to count the number of
456  * present->non-present updates: if it changed while reading the spte,
457  * we might have hit the race.  This is done using clear_spte_count.
458  */
459 static u64 __get_spte_lockless(u64 *sptep)
460 {
461         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
462         union split_spte spte, *orig = (union split_spte *)sptep;
463         int count;
464
465 retry:
466         count = sp->clear_spte_count;
467         smp_rmb();
468
469         spte.spte_low = orig->spte_low;
470         smp_rmb();
471
472         spte.spte_high = orig->spte_high;
473         smp_rmb();
474
475         if (unlikely(spte.spte_low != orig->spte_low ||
476               count != sp->clear_spte_count))
477                 goto retry;
478
479         return spte.spte;
480 }
481
482 static bool __check_direct_spte_mmio_pf(u64 spte)
483 {
484         union split_spte sspte = (union split_spte)spte;
485         u32 high_mmio_mask = shadow_mmio_mask >> 32;
486
487         /* It is valid if the spte is zapped. */
488         if (spte == 0ull)
489                 return true;
490
491         /* It is valid if the spte is being zapped. */
492         if (sspte.spte_low == 0ull &&
493             (sspte.spte_high & high_mmio_mask) == high_mmio_mask)
494                 return true;
495
496         return false;
497 }
498 #endif
499
500 static bool spte_is_locklessly_modifiable(u64 spte)
501 {
502         return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
503                 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
504 }
505
506 static bool spte_has_volatile_bits(u64 spte)
507 {
508         /*
509          * Always atomicly update spte if it can be updated
510          * out of mmu-lock, it can ensure dirty bit is not lost,
511          * also, it can help us to get a stable is_writable_pte()
512          * to ensure tlb flush is not missed.
513          */
514         if (spte_is_locklessly_modifiable(spte))
515                 return true;
516
517         if (!shadow_accessed_mask)
518                 return false;
519
520         if (!is_shadow_present_pte(spte))
521                 return false;
522
523         if ((spte & shadow_accessed_mask) &&
524               (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
525                 return false;
526
527         return true;
528 }
529
530 static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
531 {
532         return (old_spte & bit_mask) && !(new_spte & bit_mask);
533 }
534
535 static bool spte_is_bit_changed(u64 old_spte, u64 new_spte, u64 bit_mask)
536 {
537         return (old_spte & bit_mask) != (new_spte & bit_mask);
538 }
539
540 /* Rules for using mmu_spte_set:
541  * Set the sptep from nonpresent to present.
542  * Note: the sptep being assigned *must* be either not present
543  * or in a state where the hardware will not attempt to update
544  * the spte.
545  */
546 static void mmu_spte_set(u64 *sptep, u64 new_spte)
547 {
548         WARN_ON(is_shadow_present_pte(*sptep));
549         __set_spte(sptep, new_spte);
550 }
551
552 /* Rules for using mmu_spte_update:
553  * Update the state bits, it means the mapped pfn is not changged.
554  *
555  * Whenever we overwrite a writable spte with a read-only one we
556  * should flush remote TLBs. Otherwise rmap_write_protect
557  * will find a read-only spte, even though the writable spte
558  * might be cached on a CPU's TLB, the return value indicates this
559  * case.
560  */
561 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
562 {
563         u64 old_spte = *sptep;
564         bool ret = false;
565
566         WARN_ON(!is_rmap_spte(new_spte));
567
568         if (!is_shadow_present_pte(old_spte)) {
569                 mmu_spte_set(sptep, new_spte);
570                 return ret;
571         }
572
573         if (!spte_has_volatile_bits(old_spte))
574                 __update_clear_spte_fast(sptep, new_spte);
575         else
576                 old_spte = __update_clear_spte_slow(sptep, new_spte);
577
578         /*
579          * For the spte updated out of mmu-lock is safe, since
580          * we always atomicly update it, see the comments in
581          * spte_has_volatile_bits().
582          */
583         if (spte_is_locklessly_modifiable(old_spte) &&
584               !is_writable_pte(new_spte))
585                 ret = true;
586
587         if (!shadow_accessed_mask)
588                 return ret;
589
590         /*
591          * Flush TLB when accessed/dirty bits are changed in the page tables,
592          * to guarantee consistency between TLB and page tables.
593          */
594         if (spte_is_bit_changed(old_spte, new_spte,
595                                 shadow_accessed_mask | shadow_dirty_mask))
596                 ret = true;
597
598         if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
599                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
600         if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
601                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
602
603         return ret;
604 }
605
606 /*
607  * Rules for using mmu_spte_clear_track_bits:
608  * It sets the sptep from present to nonpresent, and track the
609  * state bits, it is used to clear the last level sptep.
610  */
611 static int mmu_spte_clear_track_bits(u64 *sptep)
612 {
613         pfn_t pfn;
614         u64 old_spte = *sptep;
615
616         if (!spte_has_volatile_bits(old_spte))
617                 __update_clear_spte_fast(sptep, 0ull);
618         else
619                 old_spte = __update_clear_spte_slow(sptep, 0ull);
620
621         if (!is_rmap_spte(old_spte))
622                 return 0;
623
624         pfn = spte_to_pfn(old_spte);
625
626         /*
627          * KVM does not hold the refcount of the page used by
628          * kvm mmu, before reclaiming the page, we should
629          * unmap it from mmu first.
630          */
631         WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
632
633         if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
634                 kvm_set_pfn_accessed(pfn);
635         if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
636                 kvm_set_pfn_dirty(pfn);
637         return 1;
638 }
639
640 /*
641  * Rules for using mmu_spte_clear_no_track:
642  * Directly clear spte without caring the state bits of sptep,
643  * it is used to set the upper level spte.
644  */
645 static void mmu_spte_clear_no_track(u64 *sptep)
646 {
647         __update_clear_spte_fast(sptep, 0ull);
648 }
649
650 static u64 mmu_spte_get_lockless(u64 *sptep)
651 {
652         return __get_spte_lockless(sptep);
653 }
654
655 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
656 {
657         /*
658          * Prevent page table teardown by making any free-er wait during
659          * kvm_flush_remote_tlbs() IPI to all active vcpus.
660          */
661         local_irq_disable();
662         vcpu->mode = READING_SHADOW_PAGE_TABLES;
663         /*
664          * Make sure a following spte read is not reordered ahead of the write
665          * to vcpu->mode.
666          */
667         smp_mb();
668 }
669
670 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
671 {
672         /*
673          * Make sure the write to vcpu->mode is not reordered in front of
674          * reads to sptes.  If it does, kvm_commit_zap_page() can see us
675          * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
676          */
677         smp_mb();
678         vcpu->mode = OUTSIDE_GUEST_MODE;
679         local_irq_enable();
680 }
681
682 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
683                                   struct kmem_cache *base_cache, int min)
684 {
685         void *obj;
686
687         if (cache->nobjs >= min)
688                 return 0;
689         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
690                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
691                 if (!obj)
692                         return -ENOMEM;
693                 cache->objects[cache->nobjs++] = obj;
694         }
695         return 0;
696 }
697
698 static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
699 {
700         return cache->nobjs;
701 }
702
703 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
704                                   struct kmem_cache *cache)
705 {
706         while (mc->nobjs)
707                 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
708 }
709
710 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
711                                        int min)
712 {
713         void *page;
714
715         if (cache->nobjs >= min)
716                 return 0;
717         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
718                 page = (void *)__get_free_page(GFP_KERNEL);
719                 if (!page)
720                         return -ENOMEM;
721                 cache->objects[cache->nobjs++] = page;
722         }
723         return 0;
724 }
725
726 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
727 {
728         while (mc->nobjs)
729                 free_page((unsigned long)mc->objects[--mc->nobjs]);
730 }
731
732 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
733 {
734         int r;
735
736         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
737                                    pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
738         if (r)
739                 goto out;
740         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
741         if (r)
742                 goto out;
743         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
744                                    mmu_page_header_cache, 4);
745 out:
746         return r;
747 }
748
749 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
750 {
751         mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
752                                 pte_list_desc_cache);
753         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
754         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
755                                 mmu_page_header_cache);
756 }
757
758 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
759 {
760         void *p;
761
762         BUG_ON(!mc->nobjs);
763         p = mc->objects[--mc->nobjs];
764         return p;
765 }
766
767 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
768 {
769         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
770 }
771
772 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
773 {
774         kmem_cache_free(pte_list_desc_cache, pte_list_desc);
775 }
776
777 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
778 {
779         if (!sp->role.direct)
780                 return sp->gfns[index];
781
782         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
783 }
784
785 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
786 {
787         if (sp->role.direct)
788                 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
789         else
790                 sp->gfns[index] = gfn;
791 }
792
793 /*
794  * Return the pointer to the large page information for a given gfn,
795  * handling slots that are not large page aligned.
796  */
797 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
798                                               struct kvm_memory_slot *slot,
799                                               int level)
800 {
801         unsigned long idx;
802
803         idx = gfn_to_index(gfn, slot->base_gfn, level);
804         return &slot->arch.lpage_info[level - 2][idx];
805 }
806
807 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
808 {
809         struct kvm_memory_slot *slot;
810         struct kvm_lpage_info *linfo;
811         int i;
812
813         slot = gfn_to_memslot(kvm, gfn);
814         for (i = PT_DIRECTORY_LEVEL;
815              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
816                 linfo = lpage_info_slot(gfn, slot, i);
817                 linfo->write_count += 1;
818         }
819         kvm->arch.indirect_shadow_pages++;
820 }
821
822 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
823 {
824         struct kvm_memory_slot *slot;
825         struct kvm_lpage_info *linfo;
826         int i;
827
828         slot = gfn_to_memslot(kvm, gfn);
829         for (i = PT_DIRECTORY_LEVEL;
830              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
831                 linfo = lpage_info_slot(gfn, slot, i);
832                 linfo->write_count -= 1;
833                 WARN_ON(linfo->write_count < 0);
834         }
835         kvm->arch.indirect_shadow_pages--;
836 }
837
838 static int has_wrprotected_page(struct kvm *kvm,
839                                 gfn_t gfn,
840                                 int level)
841 {
842         struct kvm_memory_slot *slot;
843         struct kvm_lpage_info *linfo;
844
845         slot = gfn_to_memslot(kvm, gfn);
846         if (slot) {
847                 linfo = lpage_info_slot(gfn, slot, level);
848                 return linfo->write_count;
849         }
850
851         return 1;
852 }
853
854 static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
855 {
856         unsigned long page_size;
857         int i, ret = 0;
858
859         page_size = kvm_host_page_size(kvm, gfn);
860
861         for (i = PT_PAGE_TABLE_LEVEL;
862              i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
863                 if (page_size >= KVM_HPAGE_SIZE(i))
864                         ret = i;
865                 else
866                         break;
867         }
868
869         return ret;
870 }
871
872 static struct kvm_memory_slot *
873 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
874                             bool no_dirty_log)
875 {
876         struct kvm_memory_slot *slot;
877
878         slot = gfn_to_memslot(vcpu->kvm, gfn);
879         if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
880               (no_dirty_log && slot->dirty_bitmap))
881                 slot = NULL;
882
883         return slot;
884 }
885
886 static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
887 {
888         return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
889 }
890
891 static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
892 {
893         int host_level, level, max_level;
894
895         host_level = host_mapping_level(vcpu->kvm, large_gfn);
896
897         if (host_level == PT_PAGE_TABLE_LEVEL)
898                 return host_level;
899
900         max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
901
902         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
903                 if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
904                         break;
905
906         return level - 1;
907 }
908
909 /*
910  * Pte mapping structures:
911  *
912  * If pte_list bit zero is zero, then pte_list point to the spte.
913  *
914  * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
915  * pte_list_desc containing more mappings.
916  *
917  * Returns the number of pte entries before the spte was added or zero if
918  * the spte was not added.
919  *
920  */
921 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
922                         unsigned long *pte_list)
923 {
924         struct pte_list_desc *desc;
925         int i, count = 0;
926
927         if (!*pte_list) {
928                 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
929                 *pte_list = (unsigned long)spte;
930         } else if (!(*pte_list & 1)) {
931                 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
932                 desc = mmu_alloc_pte_list_desc(vcpu);
933                 desc->sptes[0] = (u64 *)*pte_list;
934                 desc->sptes[1] = spte;
935                 *pte_list = (unsigned long)desc | 1;
936                 ++count;
937         } else {
938                 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
939                 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
940                 while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
941                         desc = desc->more;
942                         count += PTE_LIST_EXT;
943                 }
944                 if (desc->sptes[PTE_LIST_EXT-1]) {
945                         desc->more = mmu_alloc_pte_list_desc(vcpu);
946                         desc = desc->more;
947                 }
948                 for (i = 0; desc->sptes[i]; ++i)
949                         ++count;
950                 desc->sptes[i] = spte;
951         }
952         return count;
953 }
954
955 static void
956 pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
957                            int i, struct pte_list_desc *prev_desc)
958 {
959         int j;
960
961         for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
962                 ;
963         desc->sptes[i] = desc->sptes[j];
964         desc->sptes[j] = NULL;
965         if (j != 0)
966                 return;
967         if (!prev_desc && !desc->more)
968                 *pte_list = (unsigned long)desc->sptes[0];
969         else
970                 if (prev_desc)
971                         prev_desc->more = desc->more;
972                 else
973                         *pte_list = (unsigned long)desc->more | 1;
974         mmu_free_pte_list_desc(desc);
975 }
976
977 static void pte_list_remove(u64 *spte, unsigned long *pte_list)
978 {
979         struct pte_list_desc *desc;
980         struct pte_list_desc *prev_desc;
981         int i;
982
983         if (!*pte_list) {
984                 printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
985                 BUG();
986         } else if (!(*pte_list & 1)) {
987                 rmap_printk("pte_list_remove:  %p 1->0\n", spte);
988                 if ((u64 *)*pte_list != spte) {
989                         printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
990                         BUG();
991                 }
992                 *pte_list = 0;
993         } else {
994                 rmap_printk("pte_list_remove:  %p many->many\n", spte);
995                 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
996                 prev_desc = NULL;
997                 while (desc) {
998                         for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
999                                 if (desc->sptes[i] == spte) {
1000                                         pte_list_desc_remove_entry(pte_list,
1001                                                                desc, i,
1002                                                                prev_desc);
1003                                         return;
1004                                 }
1005                         prev_desc = desc;
1006                         desc = desc->more;
1007                 }
1008                 pr_err("pte_list_remove: %p many->many\n", spte);
1009                 BUG();
1010         }
1011 }
1012
1013 typedef void (*pte_list_walk_fn) (u64 *spte);
1014 static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
1015 {
1016         struct pte_list_desc *desc;
1017         int i;
1018
1019         if (!*pte_list)
1020                 return;
1021
1022         if (!(*pte_list & 1))
1023                 return fn((u64 *)*pte_list);
1024
1025         desc = (struct pte_list_desc *)(*pte_list & ~1ul);
1026         while (desc) {
1027                 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
1028                         fn(desc->sptes[i]);
1029                 desc = desc->more;
1030         }
1031 }
1032
1033 static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
1034                                     struct kvm_memory_slot *slot)
1035 {
1036         unsigned long idx;
1037
1038         idx = gfn_to_index(gfn, slot->base_gfn, level);
1039         return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1040 }
1041
1042 /*
1043  * Take gfn and return the reverse mapping to it.
1044  */
1045 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
1046 {
1047         struct kvm_memory_slot *slot;
1048
1049         slot = gfn_to_memslot(kvm, gfn);
1050         return __gfn_to_rmap(gfn, level, slot);
1051 }
1052
1053 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1054 {
1055         struct kvm_mmu_memory_cache *cache;
1056
1057         cache = &vcpu->arch.mmu_pte_list_desc_cache;
1058         return mmu_memory_cache_free_objects(cache);
1059 }
1060
1061 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1062 {
1063         struct kvm_mmu_page *sp;
1064         unsigned long *rmapp;
1065
1066         sp = page_header(__pa(spte));
1067         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1068         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1069         return pte_list_add(vcpu, spte, rmapp);
1070 }
1071
1072 static void rmap_remove(struct kvm *kvm, u64 *spte)
1073 {
1074         struct kvm_mmu_page *sp;
1075         gfn_t gfn;
1076         unsigned long *rmapp;
1077
1078         sp = page_header(__pa(spte));
1079         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1080         rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
1081         pte_list_remove(spte, rmapp);
1082 }
1083
1084 /*
1085  * Used by the following functions to iterate through the sptes linked by a
1086  * rmap.  All fields are private and not assumed to be used outside.
1087  */
1088 struct rmap_iterator {
1089         /* private fields */
1090         struct pte_list_desc *desc;     /* holds the sptep if not NULL */
1091         int pos;                        /* index of the sptep */
1092 };
1093
1094 /*
1095  * Iteration must be started by this function.  This should also be used after
1096  * removing/dropping sptes from the rmap link because in such cases the
1097  * information in the itererator may not be valid.
1098  *
1099  * Returns sptep if found, NULL otherwise.
1100  */
1101 static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
1102 {
1103         if (!rmap)
1104                 return NULL;
1105
1106         if (!(rmap & 1)) {
1107                 iter->desc = NULL;
1108                 return (u64 *)rmap;
1109         }
1110
1111         iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
1112         iter->pos = 0;
1113         return iter->desc->sptes[iter->pos];
1114 }
1115
1116 /*
1117  * Must be used with a valid iterator: e.g. after rmap_get_first().
1118  *
1119  * Returns sptep if found, NULL otherwise.
1120  */
1121 static u64 *rmap_get_next(struct rmap_iterator *iter)
1122 {
1123         if (iter->desc) {
1124                 if (iter->pos < PTE_LIST_EXT - 1) {
1125                         u64 *sptep;
1126
1127                         ++iter->pos;
1128                         sptep = iter->desc->sptes[iter->pos];
1129                         if (sptep)
1130                                 return sptep;
1131                 }
1132
1133                 iter->desc = iter->desc->more;
1134
1135                 if (iter->desc) {
1136                         iter->pos = 0;
1137                         /* desc->sptes[0] cannot be NULL */
1138                         return iter->desc->sptes[iter->pos];
1139                 }
1140         }
1141
1142         return NULL;
1143 }
1144
1145 static void drop_spte(struct kvm *kvm, u64 *sptep)
1146 {
1147         if (mmu_spte_clear_track_bits(sptep))
1148                 rmap_remove(kvm, sptep);
1149 }
1150
1151
1152 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1153 {
1154         if (is_large_pte(*sptep)) {
1155                 WARN_ON(page_header(__pa(sptep))->role.level ==
1156                         PT_PAGE_TABLE_LEVEL);
1157                 drop_spte(kvm, sptep);
1158                 --kvm->stat.lpages;
1159                 return true;
1160         }
1161
1162         return false;
1163 }
1164
1165 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1166 {
1167         if (__drop_large_spte(vcpu->kvm, sptep))
1168                 kvm_flush_remote_tlbs(vcpu->kvm);
1169 }
1170
1171 /*
1172  * Write-protect on the specified @sptep, @pt_protect indicates whether
1173  * spte write-protection is caused by protecting shadow page table.
1174  *
1175  * Note: write protection is difference between dirty logging and spte
1176  * protection:
1177  * - for dirty logging, the spte can be set to writable at anytime if
1178  *   its dirty bitmap is properly set.
1179  * - for spte protection, the spte can be writable only after unsync-ing
1180  *   shadow page.
1181  *
1182  * Return true if tlb need be flushed.
1183  */
1184 static bool spte_write_protect(struct kvm *kvm, u64 *sptep, bool pt_protect)
1185 {
1186         u64 spte = *sptep;
1187
1188         if (!is_writable_pte(spte) &&
1189               !(pt_protect && spte_is_locklessly_modifiable(spte)))
1190                 return false;
1191
1192         rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
1193
1194         if (pt_protect)
1195                 spte &= ~SPTE_MMU_WRITEABLE;
1196         spte = spte & ~PT_WRITABLE_MASK;
1197
1198         return mmu_spte_update(sptep, spte);
1199 }
1200
1201 static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
1202                                  bool pt_protect)
1203 {
1204         u64 *sptep;
1205         struct rmap_iterator iter;
1206         bool flush = false;
1207
1208         for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
1209                 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1210
1211                 flush |= spte_write_protect(kvm, sptep, pt_protect);
1212                 sptep = rmap_get_next(&iter);
1213         }
1214
1215         return flush;
1216 }
1217
1218 static bool spte_clear_dirty(struct kvm *kvm, u64 *sptep)
1219 {
1220         u64 spte = *sptep;
1221
1222         rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);
1223
1224         spte &= ~shadow_dirty_mask;
1225
1226         return mmu_spte_update(sptep, spte);
1227 }
1228
1229 static bool __rmap_clear_dirty(struct kvm *kvm, unsigned long *rmapp)
1230 {
1231         u64 *sptep;
1232         struct rmap_iterator iter;
1233         bool flush = false;
1234
1235         for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
1236                 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1237
1238                 flush |= spte_clear_dirty(kvm, sptep);
1239                 sptep = rmap_get_next(&iter);
1240         }
1241
1242         return flush;
1243 }
1244
1245 static bool spte_set_dirty(struct kvm *kvm, u64 *sptep)
1246 {
1247         u64 spte = *sptep;
1248
1249         rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);
1250
1251         spte |= shadow_dirty_mask;
1252
1253         return mmu_spte_update(sptep, spte);
1254 }
1255
1256 static bool __rmap_set_dirty(struct kvm *kvm, unsigned long *rmapp)
1257 {
1258         u64 *sptep;
1259         struct rmap_iterator iter;
1260         bool flush = false;
1261
1262         for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
1263                 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1264
1265                 flush |= spte_set_dirty(kvm, sptep);
1266                 sptep = rmap_get_next(&iter);
1267         }
1268
1269         return flush;
1270 }
1271
1272 /**
1273  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1274  * @kvm: kvm instance
1275  * @slot: slot to protect
1276  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1277  * @mask: indicates which pages we should protect
1278  *
1279  * Used when we do not need to care about huge page mappings: e.g. during dirty
1280  * logging we do not have any such mappings.
1281  */
1282 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1283                                      struct kvm_memory_slot *slot,
1284                                      gfn_t gfn_offset, unsigned long mask)
1285 {
1286         unsigned long *rmapp;
1287
1288         while (mask) {
1289                 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1290                                       PT_PAGE_TABLE_LEVEL, slot);
1291                 __rmap_write_protect(kvm, rmapp, false);
1292
1293                 /* clear the first set bit */
1294                 mask &= mask - 1;
1295         }
1296 }
1297
1298 /**
1299  * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages
1300  * @kvm: kvm instance
1301  * @slot: slot to clear D-bit
1302  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1303  * @mask: indicates which pages we should clear D-bit
1304  *
1305  * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1306  */
1307 void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1308                                      struct kvm_memory_slot *slot,
1309                                      gfn_t gfn_offset, unsigned long mask)
1310 {
1311         unsigned long *rmapp;
1312
1313         while (mask) {
1314                 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1315                                       PT_PAGE_TABLE_LEVEL, slot);
1316                 __rmap_clear_dirty(kvm, rmapp);
1317
1318                 /* clear the first set bit */
1319                 mask &= mask - 1;
1320         }
1321 }
1322 EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);
1323
1324 /**
1325  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1326  * PT level pages.
1327  *
1328  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1329  * enable dirty logging for them.
1330  *
1331  * Used when we do not need to care about huge page mappings: e.g. during dirty
1332  * logging we do not have any such mappings.
1333  */
1334 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1335                                 struct kvm_memory_slot *slot,
1336                                 gfn_t gfn_offset, unsigned long mask)
1337 {
1338         if (kvm_x86_ops->enable_log_dirty_pt_masked)
1339                 kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
1340                                 mask);
1341         else
1342                 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1343 }
1344
1345 static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
1346 {
1347         struct kvm_memory_slot *slot;
1348         unsigned long *rmapp;
1349         int i;
1350         bool write_protected = false;
1351
1352         slot = gfn_to_memslot(kvm, gfn);
1353
1354         for (i = PT_PAGE_TABLE_LEVEL;
1355              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
1356                 rmapp = __gfn_to_rmap(gfn, i, slot);
1357                 write_protected |= __rmap_write_protect(kvm, rmapp, true);
1358         }
1359
1360         return write_protected;
1361 }
1362
1363 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
1364                            struct kvm_memory_slot *slot, gfn_t gfn, int level,
1365                            unsigned long data)
1366 {
1367         u64 *sptep;
1368         struct rmap_iterator iter;
1369         int need_tlb_flush = 0;
1370
1371         while ((sptep = rmap_get_first(*rmapp, &iter))) {
1372                 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1373                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx gfn %llx (%d)\n",
1374                              sptep, *sptep, gfn, level);
1375
1376                 drop_spte(kvm, sptep);
1377                 need_tlb_flush = 1;
1378         }
1379
1380         return need_tlb_flush;
1381 }
1382
1383 static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
1384                              struct kvm_memory_slot *slot, gfn_t gfn, int level,
1385                              unsigned long data)
1386 {
1387         u64 *sptep;
1388         struct rmap_iterator iter;
1389         int need_flush = 0;
1390         u64 new_spte;
1391         pte_t *ptep = (pte_t *)data;
1392         pfn_t new_pfn;
1393
1394         WARN_ON(pte_huge(*ptep));
1395         new_pfn = pte_pfn(*ptep);
1396
1397         for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
1398                 BUG_ON(!is_shadow_present_pte(*sptep));
1399                 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1400                              sptep, *sptep, gfn, level);
1401
1402                 need_flush = 1;
1403
1404                 if (pte_write(*ptep)) {
1405                         drop_spte(kvm, sptep);
1406                         sptep = rmap_get_first(*rmapp, &iter);
1407                 } else {
1408                         new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1409                         new_spte |= (u64)new_pfn << PAGE_SHIFT;
1410
1411                         new_spte &= ~PT_WRITABLE_MASK;
1412                         new_spte &= ~SPTE_HOST_WRITEABLE;
1413                         new_spte &= ~shadow_accessed_mask;
1414
1415                         mmu_spte_clear_track_bits(sptep);
1416                         mmu_spte_set(sptep, new_spte);
1417                         sptep = rmap_get_next(&iter);
1418                 }
1419         }
1420
1421         if (need_flush)
1422                 kvm_flush_remote_tlbs(kvm);
1423
1424         return 0;
1425 }
1426
1427 static int kvm_handle_hva_range(struct kvm *kvm,
1428                                 unsigned long start,
1429                                 unsigned long end,
1430                                 unsigned long data,
1431                                 int (*handler)(struct kvm *kvm,
1432                                                unsigned long *rmapp,
1433                                                struct kvm_memory_slot *slot,
1434                                                gfn_t gfn,
1435                                                int level,
1436                                                unsigned long data))
1437 {
1438         int j;
1439         int ret = 0;
1440         struct kvm_memslots *slots;
1441         struct kvm_memory_slot *memslot;
1442
1443         slots = kvm_memslots(kvm);
1444
1445         kvm_for_each_memslot(memslot, slots) {
1446                 unsigned long hva_start, hva_end;
1447                 gfn_t gfn_start, gfn_end;
1448
1449                 hva_start = max(start, memslot->userspace_addr);
1450                 hva_end = min(end, memslot->userspace_addr +
1451                                         (memslot->npages << PAGE_SHIFT));
1452                 if (hva_start >= hva_end)
1453                         continue;
1454                 /*
1455                  * {gfn(page) | page intersects with [hva_start, hva_end)} =
1456                  * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1457                  */
1458                 gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1459                 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1460
1461                 for (j = PT_PAGE_TABLE_LEVEL;
1462                      j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
1463                         unsigned long idx, idx_end;
1464                         unsigned long *rmapp;
1465                         gfn_t gfn = gfn_start;
1466
1467                         /*
1468                          * {idx(page_j) | page_j intersects with
1469                          *  [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
1470                          */
1471                         idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
1472                         idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
1473
1474                         rmapp = __gfn_to_rmap(gfn_start, j, memslot);
1475
1476                         for (; idx <= idx_end;
1477                                ++idx, gfn += (1UL << KVM_HPAGE_GFN_SHIFT(j)))
1478                                 ret |= handler(kvm, rmapp++, memslot,
1479                                                gfn, j, data);
1480                 }
1481         }
1482
1483         return ret;
1484 }
1485
1486 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1487                           unsigned long data,
1488                           int (*handler)(struct kvm *kvm, unsigned long *rmapp,
1489                                          struct kvm_memory_slot *slot,
1490                                          gfn_t gfn, int level,
1491                                          unsigned long data))
1492 {
1493         return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1494 }
1495
1496 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1497 {
1498         return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
1499 }
1500
1501 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1502 {
1503         return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1504 }
1505
1506 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1507 {
1508         kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1509 }
1510
1511 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1512                          struct kvm_memory_slot *slot, gfn_t gfn, int level,
1513                          unsigned long data)
1514 {
1515         u64 *sptep;
1516         struct rmap_iterator uninitialized_var(iter);
1517         int young = 0;
1518
1519         BUG_ON(!shadow_accessed_mask);
1520
1521         for (sptep = rmap_get_first(*rmapp, &iter); sptep;
1522              sptep = rmap_get_next(&iter)) {
1523                 BUG_ON(!is_shadow_present_pte(*sptep));
1524
1525                 if (*sptep & shadow_accessed_mask) {
1526                         young = 1;
1527                         clear_bit((ffs(shadow_accessed_mask) - 1),
1528                                  (unsigned long *)sptep);
1529                 }
1530         }
1531         trace_kvm_age_page(gfn, level, slot, young);
1532         return young;
1533 }
1534
1535 static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1536                               struct kvm_memory_slot *slot, gfn_t gfn,
1537                               int level, unsigned long data)
1538 {
1539         u64 *sptep;
1540         struct rmap_iterator iter;
1541         int young = 0;
1542
1543         /*
1544          * If there's no access bit in the secondary pte set by the
1545          * hardware it's up to gup-fast/gup to set the access bit in
1546          * the primary pte or in the page structure.
1547          */
1548         if (!shadow_accessed_mask)
1549                 goto out;
1550
1551         for (sptep = rmap_get_first(*rmapp, &iter); sptep;
1552              sptep = rmap_get_next(&iter)) {
1553                 BUG_ON(!is_shadow_present_pte(*sptep));
1554
1555                 if (*sptep & shadow_accessed_mask) {
1556                         young = 1;
1557                         break;
1558                 }
1559         }
1560 out:
1561         return young;
1562 }
1563
1564 #define RMAP_RECYCLE_THRESHOLD 1000
1565
1566 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1567 {
1568         unsigned long *rmapp;
1569         struct kvm_mmu_page *sp;
1570
1571         sp = page_header(__pa(spte));
1572
1573         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1574
1575         kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0);
1576         kvm_flush_remote_tlbs(vcpu->kvm);
1577 }
1578
1579 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1580 {
1581         /*
1582          * In case of absence of EPT Access and Dirty Bits supports,
1583          * emulate the accessed bit for EPT, by checking if this page has
1584          * an EPT mapping, and clearing it if it does. On the next access,
1585          * a new EPT mapping will be established.
1586          * This has some overhead, but not as much as the cost of swapping
1587          * out actively used pages or breaking up actively used hugepages.
1588          */
1589         if (!shadow_accessed_mask) {
1590                 /*
1591                  * We are holding the kvm->mmu_lock, and we are blowing up
1592                  * shadow PTEs. MMU notifier consumers need to be kept at bay.
1593                  * This is correct as long as we don't decouple the mmu_lock
1594                  * protected regions (like invalidate_range_start|end does).
1595                  */
1596                 kvm->mmu_notifier_seq++;
1597                 return kvm_handle_hva_range(kvm, start, end, 0,
1598                                             kvm_unmap_rmapp);
1599         }
1600
1601         return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1602 }
1603
1604 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1605 {
1606         return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1607 }
1608
1609 #ifdef MMU_DEBUG
1610 static int is_empty_shadow_page(u64 *spt)
1611 {
1612         u64 *pos;
1613         u64 *end;
1614
1615         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1616                 if (is_shadow_present_pte(*pos)) {
1617                         printk(KERN_ERR "%s: %p %llx\n", __func__,
1618                                pos, *pos);
1619                         return 0;
1620                 }
1621         return 1;
1622 }
1623 #endif
1624
1625 /*
1626  * This value is the sum of all of the kvm instances's
1627  * kvm->arch.n_used_mmu_pages values.  We need a global,
1628  * aggregate version in order to make the slab shrinker
1629  * faster
1630  */
1631 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
1632 {
1633         kvm->arch.n_used_mmu_pages += nr;
1634         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1635 }
1636
1637 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1638 {
1639         MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1640         hlist_del(&sp->hash_link);
1641         list_del(&sp->link);
1642         free_page((unsigned long)sp->spt);
1643         if (!sp->role.direct)
1644                 free_page((unsigned long)sp->gfns);
1645         kmem_cache_free(mmu_page_header_cache, sp);
1646 }
1647
1648 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1649 {
1650         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1651 }
1652
1653 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1654                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1655 {
1656         if (!parent_pte)
1657                 return;
1658
1659         pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1660 }
1661
1662 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1663                                        u64 *parent_pte)
1664 {
1665         pte_list_remove(parent_pte, &sp->parent_ptes);
1666 }
1667
1668 static void drop_parent_pte(struct kvm_mmu_page *sp,
1669                             u64 *parent_pte)
1670 {
1671         mmu_page_remove_parent_pte(sp, parent_pte);
1672         mmu_spte_clear_no_track(parent_pte);
1673 }
1674
1675 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
1676                                                u64 *parent_pte, int direct)
1677 {
1678         struct kvm_mmu_page *sp;
1679
1680         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1681         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1682         if (!direct)
1683                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1684         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1685
1686         /*
1687          * The active_mmu_pages list is the FIFO list, do not move the
1688          * page until it is zapped. kvm_zap_obsolete_pages depends on
1689          * this feature. See the comments in kvm_zap_obsolete_pages().
1690          */
1691         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1692         sp->parent_ptes = 0;
1693         mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1694         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1695         return sp;
1696 }
1697
1698 static void mark_unsync(u64 *spte);
1699 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1700 {
1701         pte_list_walk(&sp->parent_ptes, mark_unsync);
1702 }
1703
1704 static void mark_unsync(u64 *spte)
1705 {
1706         struct kvm_mmu_page *sp;
1707         unsigned int index;
1708
1709         sp = page_header(__pa(spte));
1710         index = spte - sp->spt;
1711         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1712                 return;
1713         if (sp->unsync_children++)
1714                 return;
1715         kvm_mmu_mark_parents_unsync(sp);
1716 }
1717
1718 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1719                                struct kvm_mmu_page *sp)
1720 {
1721         return 1;
1722 }
1723
1724 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1725 {
1726 }
1727
1728 static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1729                                  struct kvm_mmu_page *sp, u64 *spte,
1730                                  const void *pte)
1731 {
1732         WARN_ON(1);
1733 }
1734
1735 #define KVM_PAGE_ARRAY_NR 16
1736
1737 struct kvm_mmu_pages {
1738         struct mmu_page_and_offset {
1739                 struct kvm_mmu_page *sp;
1740                 unsigned int idx;
1741         } page[KVM_PAGE_ARRAY_NR];
1742         unsigned int nr;
1743 };
1744
1745 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1746                          int idx)
1747 {
1748         int i;
1749
1750         if (sp->unsync)
1751                 for (i=0; i < pvec->nr; i++)
1752                         if (pvec->page[i].sp == sp)
1753                                 return 0;
1754
1755         pvec->page[pvec->nr].sp = sp;
1756         pvec->page[pvec->nr].idx = idx;
1757         pvec->nr++;
1758         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1759 }
1760
1761 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1762                            struct kvm_mmu_pages *pvec)
1763 {
1764         int i, ret, nr_unsync_leaf = 0;
1765
1766         for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1767                 struct kvm_mmu_page *child;
1768                 u64 ent = sp->spt[i];
1769
1770                 if (!is_shadow_present_pte(ent) || is_large_pte(ent))
1771                         goto clear_child_bitmap;
1772
1773                 child = page_header(ent & PT64_BASE_ADDR_MASK);
1774
1775                 if (child->unsync_children) {
1776                         if (mmu_pages_add(pvec, child, i))
1777                                 return -ENOSPC;
1778
1779                         ret = __mmu_unsync_walk(child, pvec);
1780                         if (!ret)
1781                                 goto clear_child_bitmap;
1782                         else if (ret > 0)
1783                                 nr_unsync_leaf += ret;
1784                         else
1785                                 return ret;
1786                 } else if (child->unsync) {
1787                         nr_unsync_leaf++;
1788                         if (mmu_pages_add(pvec, child, i))
1789                                 return -ENOSPC;
1790                 } else
1791                          goto clear_child_bitmap;
1792
1793                 continue;
1794
1795 clear_child_bitmap:
1796                 __clear_bit(i, sp->unsync_child_bitmap);
1797                 sp->unsync_children--;
1798                 WARN_ON((int)sp->unsync_children < 0);
1799         }
1800
1801
1802         return nr_unsync_leaf;
1803 }
1804
1805 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1806                            struct kvm_mmu_pages *pvec)
1807 {
1808         if (!sp->unsync_children)
1809                 return 0;
1810
1811         mmu_pages_add(pvec, sp, 0);
1812         return __mmu_unsync_walk(sp, pvec);
1813 }
1814
1815 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1816 {
1817         WARN_ON(!sp->unsync);
1818         trace_kvm_mmu_sync_page(sp);
1819         sp->unsync = 0;
1820         --kvm->stat.mmu_unsync;
1821 }
1822
1823 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1824                                     struct list_head *invalid_list);
1825 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1826                                     struct list_head *invalid_list);
1827
1828 /*
1829  * NOTE: we should pay more attention on the zapped-obsolete page
1830  * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
1831  * since it has been deleted from active_mmu_pages but still can be found
1832  * at hast list.
1833  *
1834  * for_each_gfn_indirect_valid_sp has skipped that kind of page and
1835  * kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
1836  * all the obsolete pages.
1837  */
1838 #define for_each_gfn_sp(_kvm, _sp, _gfn)                                \
1839         hlist_for_each_entry(_sp,                                       \
1840           &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
1841                 if ((_sp)->gfn != (_gfn)) {} else
1842
1843 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)                 \
1844         for_each_gfn_sp(_kvm, _sp, _gfn)                                \
1845                 if ((_sp)->role.direct || (_sp)->role.invalid) {} else
1846
1847 /* @sp->gfn should be write-protected at the call site */
1848 static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1849                            struct list_head *invalid_list, bool clear_unsync)
1850 {
1851         if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1852                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1853                 return 1;
1854         }
1855
1856         if (clear_unsync)
1857                 kvm_unlink_unsync_page(vcpu->kvm, sp);
1858
1859         if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1860                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1861                 return 1;
1862         }
1863
1864         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1865         return 0;
1866 }
1867
1868 static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
1869                                    struct kvm_mmu_page *sp)
1870 {
1871         LIST_HEAD(invalid_list);
1872         int ret;
1873
1874         ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1875         if (ret)
1876                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1877
1878         return ret;
1879 }
1880
1881 #ifdef CONFIG_KVM_MMU_AUDIT
1882 #include "mmu_audit.c"
1883 #else
1884 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
1885 static void mmu_audit_disable(void) { }
1886 #endif
1887
1888 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1889                          struct list_head *invalid_list)
1890 {
1891         return __kvm_sync_page(vcpu, sp, invalid_list, true);
1892 }
1893
1894 /* @gfn should be write-protected at the call site */
1895 static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1896 {
1897         struct kvm_mmu_page *s;
1898         LIST_HEAD(invalid_list);
1899         bool flush = false;
1900
1901         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
1902                 if (!s->unsync)
1903                         continue;
1904
1905                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1906                 kvm_unlink_unsync_page(vcpu->kvm, s);
1907                 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1908                         (vcpu->arch.mmu.sync_page(vcpu, s))) {
1909                         kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1910                         continue;
1911                 }
1912                 flush = true;
1913         }
1914
1915         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1916         if (flush)
1917                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1918 }
1919
1920 struct mmu_page_path {
1921         struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1922         unsigned int idx[PT64_ROOT_LEVEL-1];
1923 };
1924
1925 #define for_each_sp(pvec, sp, parents, i)                       \
1926                 for (i = mmu_pages_next(&pvec, &parents, -1),   \
1927                         sp = pvec.page[i].sp;                   \
1928                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1929                         i = mmu_pages_next(&pvec, &parents, i))
1930
1931 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1932                           struct mmu_page_path *parents,
1933                           int i)
1934 {
1935         int n;
1936
1937         for (n = i+1; n < pvec->nr; n++) {
1938                 struct kvm_mmu_page *sp = pvec->page[n].sp;
1939
1940                 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1941                         parents->idx[0] = pvec->page[n].idx;
1942                         return n;
1943                 }
1944
1945                 parents->parent[sp->role.level-2] = sp;
1946                 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1947         }
1948
1949         return n;
1950 }
1951
1952 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1953 {
1954         struct kvm_mmu_page *sp;
1955         unsigned int level = 0;
1956
1957         do {
1958                 unsigned int idx = parents->idx[level];
1959
1960                 sp = parents->parent[level];
1961                 if (!sp)
1962                         return;
1963
1964                 --sp->unsync_children;
1965                 WARN_ON((int)sp->unsync_children < 0);
1966                 __clear_bit(idx, sp->unsync_child_bitmap);
1967                 level++;
1968         } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1969 }
1970
1971 static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1972                                struct mmu_page_path *parents,
1973                                struct kvm_mmu_pages *pvec)
1974 {
1975         parents->parent[parent->role.level-1] = NULL;
1976         pvec->nr = 0;
1977 }
1978
1979 static void mmu_sync_children(struct kvm_vcpu *vcpu,
1980                               struct kvm_mmu_page *parent)
1981 {
1982         int i;
1983         struct kvm_mmu_page *sp;
1984         struct mmu_page_path parents;
1985         struct kvm_mmu_pages pages;
1986         LIST_HEAD(invalid_list);
1987
1988         kvm_mmu_pages_init(parent, &parents, &pages);
1989         while (mmu_unsync_walk(parent, &pages)) {
1990                 bool protected = false;
1991
1992                 for_each_sp(pages, sp, parents, i)
1993                         protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1994
1995                 if (protected)
1996                         kvm_flush_remote_tlbs(vcpu->kvm);
1997
1998                 for_each_sp(pages, sp, parents, i) {
1999                         kvm_sync_page(vcpu, sp, &invalid_list);
2000                         mmu_pages_clear_parents(&parents);
2001                 }
2002                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2003                 cond_resched_lock(&vcpu->kvm->mmu_lock);
2004                 kvm_mmu_pages_init(parent, &parents, &pages);
2005         }
2006 }
2007
2008 static void init_shadow_page_table(struct kvm_mmu_page *sp)
2009 {
2010         int i;
2011
2012         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2013                 sp->spt[i] = 0ull;
2014 }
2015
2016 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2017 {
2018         sp->write_flooding_count = 0;
2019 }
2020
2021 static void clear_sp_write_flooding_count(u64 *spte)
2022 {
2023         struct kvm_mmu_page *sp =  page_header(__pa(spte));
2024
2025         __clear_sp_write_flooding_count(sp);
2026 }
2027
2028 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
2029 {
2030         return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
2031 }
2032
2033 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2034                                              gfn_t gfn,
2035                                              gva_t gaddr,
2036                                              unsigned level,
2037                                              int direct,
2038                                              unsigned access,
2039                                              u64 *parent_pte)
2040 {
2041         union kvm_mmu_page_role role;
2042         unsigned quadrant;
2043         struct kvm_mmu_page *sp;
2044         bool need_sync = false;
2045
2046         role = vcpu->arch.mmu.base_role;
2047         role.level = level;
2048         role.direct = direct;
2049         if (role.direct)
2050                 role.cr4_pae = 0;
2051         role.access = access;
2052         if (!vcpu->arch.mmu.direct_map
2053             && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
2054                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2055                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2056                 role.quadrant = quadrant;
2057         }
2058         for_each_gfn_sp(vcpu->kvm, sp, gfn) {
2059                 if (is_obsolete_sp(vcpu->kvm, sp))
2060                         continue;
2061
2062                 if (!need_sync && sp->unsync)
2063                         need_sync = true;
2064
2065                 if (sp->role.word != role.word)
2066                         continue;
2067
2068                 if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
2069                         break;
2070
2071                 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
2072                 if (sp->unsync_children) {
2073                         kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2074                         kvm_mmu_mark_parents_unsync(sp);
2075                 } else if (sp->unsync)
2076                         kvm_mmu_mark_parents_unsync(sp);
2077
2078                 __clear_sp_write_flooding_count(sp);
2079                 trace_kvm_mmu_get_page(sp, false);
2080                 return sp;
2081         }
2082         ++vcpu->kvm->stat.mmu_cache_miss;
2083         sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
2084         if (!sp)
2085                 return sp;
2086         sp->gfn = gfn;
2087         sp->role = role;
2088         hlist_add_head(&sp->hash_link,
2089                 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2090         if (!direct) {
2091                 if (rmap_write_protect(vcpu->kvm, gfn))
2092                         kvm_flush_remote_tlbs(vcpu->kvm);
2093                 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2094                         kvm_sync_pages(vcpu, gfn);
2095
2096                 account_shadowed(vcpu->kvm, gfn);
2097         }
2098         sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2099         init_shadow_page_table(sp);
2100         trace_kvm_mmu_get_page(sp, true);
2101         return sp;
2102 }
2103
2104 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2105                              struct kvm_vcpu *vcpu, u64 addr)
2106 {
2107         iterator->addr = addr;
2108         iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
2109         iterator->level = vcpu->arch.mmu.shadow_root_level;
2110
2111         if (iterator->level == PT64_ROOT_LEVEL &&
2112             vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
2113             !vcpu->arch.mmu.direct_map)
2114                 --iterator->level;
2115
2116         if (iterator->level == PT32E_ROOT_LEVEL) {
2117                 iterator->shadow_addr
2118                         = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
2119                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2120                 --iterator->level;
2121                 if (!iterator->shadow_addr)
2122                         iterator->level = 0;
2123         }
2124 }
2125
2126 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2127 {
2128         if (iterator->level < PT_PAGE_TABLE_LEVEL)
2129                 return false;
2130
2131         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2132         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2133         return true;
2134 }
2135
2136 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2137                                u64 spte)
2138 {
2139         if (is_last_spte(spte, iterator->level)) {
2140                 iterator->level = 0;
2141                 return;
2142         }
2143
2144         iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2145         --iterator->level;
2146 }
2147
2148 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2149 {
2150         return __shadow_walk_next(iterator, *iterator->sptep);
2151 }
2152
2153 static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp, bool accessed)
2154 {
2155         u64 spte;
2156
2157         BUILD_BUG_ON(VMX_EPT_READABLE_MASK != PT_PRESENT_MASK ||
2158                         VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2159
2160         spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
2161                shadow_user_mask | shadow_x_mask;
2162
2163         if (accessed)
2164                 spte |= shadow_accessed_mask;
2165
2166         mmu_spte_set(sptep, spte);
2167 }
2168
2169 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2170                                    unsigned direct_access)
2171 {
2172         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2173                 struct kvm_mmu_page *child;
2174
2175                 /*
2176                  * For the direct sp, if the guest pte's dirty bit
2177                  * changed form clean to dirty, it will corrupt the
2178                  * sp's access: allow writable in the read-only sp,
2179                  * so we should update the spte at this point to get
2180                  * a new sp with the correct access.
2181                  */
2182                 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
2183                 if (child->role.access == direct_access)
2184                         return;
2185
2186                 drop_parent_pte(child, sptep);
2187                 kvm_flush_remote_tlbs(vcpu->kvm);
2188         }
2189 }
2190
2191 static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2192                              u64 *spte)
2193 {
2194         u64 pte;
2195         struct kvm_mmu_page *child;
2196
2197         pte = *spte;
2198         if (is_shadow_present_pte(pte)) {
2199                 if (is_last_spte(pte, sp->role.level)) {
2200                         drop_spte(kvm, spte);
2201                         if (is_large_pte(pte))
2202                                 --kvm->stat.lpages;
2203                 } else {
2204                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2205                         drop_parent_pte(child, spte);
2206                 }
2207                 return true;
2208         }
2209
2210         if (is_mmio_spte(pte))
2211                 mmu_spte_clear_no_track(spte);
2212
2213         return false;
2214 }
2215
2216 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2217                                          struct kvm_mmu_page *sp)
2218 {
2219         unsigned i;
2220
2221         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2222                 mmu_page_zap_pte(kvm, sp, sp->spt + i);
2223 }
2224
2225 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
2226 {
2227         mmu_page_remove_parent_pte(sp, parent_pte);
2228 }
2229
2230 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2231 {
2232         u64 *sptep;
2233         struct rmap_iterator iter;
2234
2235         while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
2236                 drop_parent_pte(sp, sptep);
2237 }
2238
2239 static int mmu_zap_unsync_children(struct kvm *kvm,
2240                                    struct kvm_mmu_page *parent,
2241                                    struct list_head *invalid_list)
2242 {
2243         int i, zapped = 0;
2244         struct mmu_page_path parents;
2245         struct kvm_mmu_pages pages;
2246
2247         if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2248                 return 0;
2249
2250         kvm_mmu_pages_init(parent, &parents, &pages);
2251         while (mmu_unsync_walk(parent, &pages)) {
2252                 struct kvm_mmu_page *sp;
2253
2254                 for_each_sp(pages, sp, parents, i) {
2255                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2256                         mmu_pages_clear_parents(&parents);
2257                         zapped++;
2258                 }
2259                 kvm_mmu_pages_init(parent, &parents, &pages);
2260         }
2261
2262         return zapped;
2263 }
2264
2265 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2266                                     struct list_head *invalid_list)
2267 {
2268         int ret;
2269
2270         trace_kvm_mmu_prepare_zap_page(sp);
2271         ++kvm->stat.mmu_shadow_zapped;
2272         ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2273         kvm_mmu_page_unlink_children(kvm, sp);
2274         kvm_mmu_unlink_parents(kvm, sp);
2275
2276         if (!sp->role.invalid && !sp->role.direct)
2277                 unaccount_shadowed(kvm, sp->gfn);
2278
2279         if (sp->unsync)
2280                 kvm_unlink_unsync_page(kvm, sp);
2281         if (!sp->root_count) {
2282                 /* Count self */
2283                 ret++;
2284                 list_move(&sp->link, invalid_list);
2285                 kvm_mod_used_mmu_pages(kvm, -1);
2286         } else {
2287                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
2288
2289                 /*
2290                  * The obsolete pages can not be used on any vcpus.
2291                  * See the comments in kvm_mmu_invalidate_zap_all_pages().
2292                  */
2293                 if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
2294                         kvm_reload_remote_mmus(kvm);
2295         }
2296
2297         sp->role.invalid = 1;
2298         return ret;
2299 }
2300
2301 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2302                                     struct list_head *invalid_list)
2303 {
2304         struct kvm_mmu_page *sp, *nsp;
2305
2306         if (list_empty(invalid_list))
2307                 return;
2308
2309         /*
2310          * wmb: make sure everyone sees our modifications to the page tables
2311          * rmb: make sure we see changes to vcpu->mode
2312          */
2313         smp_mb();
2314
2315         /*
2316          * Wait for all vcpus to exit guest mode and/or lockless shadow
2317          * page table walks.
2318          */
2319         kvm_flush_remote_tlbs(kvm);
2320
2321         list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2322                 WARN_ON(!sp->role.invalid || sp->root_count);
2323                 kvm_mmu_free_page(sp);
2324         }
2325 }
2326
2327 static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
2328                                         struct list_head *invalid_list)
2329 {
2330         struct kvm_mmu_page *sp;
2331
2332         if (list_empty(&kvm->arch.active_mmu_pages))
2333                 return false;
2334
2335         sp = list_entry(kvm->arch.active_mmu_pages.prev,
2336                         struct kvm_mmu_page, link);
2337         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2338
2339         return true;
2340 }
2341
2342 /*
2343  * Changing the number of mmu pages allocated to the vm
2344  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2345  */
2346 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2347 {
2348         LIST_HEAD(invalid_list);
2349
2350         spin_lock(&kvm->mmu_lock);
2351
2352         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2353                 /* Need to free some mmu pages to achieve the goal. */
2354                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
2355                         if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
2356                                 break;
2357
2358                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
2359                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2360         }
2361
2362         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2363
2364         spin_unlock(&kvm->mmu_lock);
2365 }
2366
2367 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2368 {
2369         struct kvm_mmu_page *sp;
2370         LIST_HEAD(invalid_list);
2371         int r;
2372
2373         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2374         r = 0;
2375         spin_lock(&kvm->mmu_lock);
2376         for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2377                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2378                          sp->role.word);
2379                 r = 1;
2380                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2381         }
2382         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2383         spin_unlock(&kvm->mmu_lock);
2384
2385         return r;
2386 }
2387 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2388
2389 /*
2390  * The function is based on mtrr_type_lookup() in
2391  * arch/x86/kernel/cpu/mtrr/generic.c
2392  */
2393 static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
2394                          u64 start, u64 end)
2395 {
2396         int i;
2397         u64 base, mask;
2398         u8 prev_match, curr_match;
2399         int num_var_ranges = KVM_NR_VAR_MTRR;
2400
2401         if (!mtrr_state->enabled)
2402                 return 0xFF;
2403
2404         /* Make end inclusive end, instead of exclusive */
2405         end--;
2406
2407         /* Look in fixed ranges. Just return the type as per start */
2408         if (mtrr_state->have_fixed && (start < 0x100000)) {
2409                 int idx;
2410
2411                 if (start < 0x80000) {
2412                         idx = 0;
2413                         idx += (start >> 16);
2414                         return mtrr_state->fixed_ranges[idx];
2415                 } else if (start < 0xC0000) {
2416                         idx = 1 * 8;
2417                         idx += ((start - 0x80000) >> 14);
2418                         return mtrr_state->fixed_ranges[idx];
2419                 } else if (start < 0x1000000) {
2420                         idx = 3 * 8;
2421                         idx += ((start - 0xC0000) >> 12);
2422                         return mtrr_state->fixed_ranges[idx];
2423                 }
2424         }
2425
2426         /*
2427          * Look in variable ranges
2428          * Look of multiple ranges matching this address and pick type
2429          * as per MTRR precedence
2430          */
2431         if (!(mtrr_state->enabled & 2))
2432                 return mtrr_state->def_type;
2433
2434         prev_match = 0xFF;
2435         for (i = 0; i < num_var_ranges; ++i) {
2436                 unsigned short start_state, end_state;
2437
2438                 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
2439                         continue;
2440
2441                 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
2442                        (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
2443                 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
2444                        (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
2445
2446                 start_state = ((start & mask) == (base & mask));
2447                 end_state = ((end & mask) == (base & mask));
2448                 if (start_state != end_state)
2449                         return 0xFE;
2450
2451                 if ((start & mask) != (base & mask))
2452                         continue;
2453
2454                 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
2455                 if (prev_match == 0xFF) {
2456                         prev_match = curr_match;
2457                         continue;
2458                 }
2459
2460                 if (prev_match == MTRR_TYPE_UNCACHABLE ||
2461                     curr_match == MTRR_TYPE_UNCACHABLE)
2462                         return MTRR_TYPE_UNCACHABLE;
2463
2464                 if ((prev_match == MTRR_TYPE_WRBACK &&
2465                      curr_match == MTRR_TYPE_WRTHROUGH) ||
2466                     (prev_match == MTRR_TYPE_WRTHROUGH &&
2467                      curr_match == MTRR_TYPE_WRBACK)) {
2468                         prev_match = MTRR_TYPE_WRTHROUGH;
2469                         curr_match = MTRR_TYPE_WRTHROUGH;
2470                 }
2471
2472                 if (prev_match != curr_match)
2473                         return MTRR_TYPE_UNCACHABLE;
2474         }
2475
2476         if (prev_match != 0xFF)
2477                 return prev_match;
2478
2479         return mtrr_state->def_type;
2480 }
2481
2482 u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
2483 {
2484         u8 mtrr;
2485
2486         mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
2487                              (gfn << PAGE_SHIFT) + PAGE_SIZE);
2488         if (mtrr == 0xfe || mtrr == 0xff)
2489                 mtrr = MTRR_TYPE_WRBACK;
2490         return mtrr;
2491 }
2492 EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
2493
2494 static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2495 {
2496         trace_kvm_mmu_unsync_page(sp);
2497         ++vcpu->kvm->stat.mmu_unsync;
2498         sp->unsync = 1;
2499
2500         kvm_mmu_mark_parents_unsync(sp);
2501 }
2502
2503 static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
2504 {
2505         struct kvm_mmu_page *s;
2506
2507         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2508                 if (s->unsync)
2509                         continue;
2510                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
2511                 __kvm_unsync_page(vcpu, s);
2512         }
2513 }
2514
2515 static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
2516                                   bool can_unsync)
2517 {
2518         struct kvm_mmu_page *s;
2519         bool need_unsync = false;
2520
2521         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2522                 if (!can_unsync)
2523                         return 1;
2524
2525                 if (s->role.level != PT_PAGE_TABLE_LEVEL)
2526                         return 1;
2527
2528                 if (!s->unsync)
2529                         need_unsync = true;
2530         }
2531         if (need_unsync)
2532                 kvm_unsync_pages(vcpu, gfn);
2533         return 0;
2534 }
2535
2536 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2537                     unsigned pte_access, int level,
2538                     gfn_t gfn, pfn_t pfn, bool speculative,
2539                     bool can_unsync, bool host_writable)
2540 {
2541         u64 spte;
2542         int ret = 0;
2543
2544         if (set_mmio_spte(vcpu->kvm, sptep, gfn, pfn, pte_access))
2545                 return 0;
2546
2547         spte = PT_PRESENT_MASK;
2548         if (!speculative)
2549                 spte |= shadow_accessed_mask;
2550
2551         if (pte_access & ACC_EXEC_MASK)
2552                 spte |= shadow_x_mask;
2553         else
2554                 spte |= shadow_nx_mask;
2555
2556         if (pte_access & ACC_USER_MASK)
2557                 spte |= shadow_user_mask;
2558
2559         if (level > PT_PAGE_TABLE_LEVEL)
2560                 spte |= PT_PAGE_SIZE_MASK;
2561         if (tdp_enabled)
2562                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2563                         kvm_is_reserved_pfn(pfn));
2564
2565         if (host_writable)
2566                 spte |= SPTE_HOST_WRITEABLE;
2567         else
2568                 pte_access &= ~ACC_WRITE_MASK;
2569
2570         spte |= (u64)pfn << PAGE_SHIFT;
2571
2572         if (pte_access & ACC_WRITE_MASK) {
2573
2574                 /*
2575                  * Other vcpu creates new sp in the window between
2576                  * mapping_level() and acquiring mmu-lock. We can
2577                  * allow guest to retry the access, the mapping can
2578                  * be fixed if guest refault.
2579                  */
2580                 if (level > PT_PAGE_TABLE_LEVEL &&
2581                     has_wrprotected_page(vcpu->kvm, gfn, level))
2582                         goto done;
2583
2584                 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2585
2586                 /*
2587                  * Optimization: for pte sync, if spte was writable the hash
2588                  * lookup is unnecessary (and expensive). Write protection
2589                  * is responsibility of mmu_get_page / kvm_sync_page.
2590                  * Same reasoning can be applied to dirty page accounting.
2591                  */
2592                 if (!can_unsync && is_writable_pte(*sptep))
2593                         goto set_pte;
2594
2595                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2596                         pgprintk("%s: found shadow page for %llx, marking ro\n",
2597                                  __func__, gfn);
2598                         ret = 1;
2599                         pte_access &= ~ACC_WRITE_MASK;
2600                         spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2601                 }
2602         }
2603
2604         if (pte_access & ACC_WRITE_MASK) {
2605                 mark_page_dirty(vcpu->kvm, gfn);
2606                 spte |= shadow_dirty_mask;
2607         }
2608
2609 set_pte:
2610         if (mmu_spte_update(sptep, spte))
2611                 kvm_flush_remote_tlbs(vcpu->kvm);
2612 done:
2613         return ret;
2614 }
2615
2616 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2617                          unsigned pte_access, int write_fault, int *emulate,
2618                          int level, gfn_t gfn, pfn_t pfn, bool speculative,
2619                          bool host_writable)
2620 {
2621         int was_rmapped = 0;
2622         int rmap_count;
2623
2624         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2625                  *sptep, write_fault, gfn);
2626
2627         if (is_rmap_spte(*sptep)) {
2628                 /*
2629                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2630                  * the parent of the now unreachable PTE.
2631                  */
2632                 if (level > PT_PAGE_TABLE_LEVEL &&
2633                     !is_large_pte(*sptep)) {
2634                         struct kvm_mmu_page *child;
2635                         u64 pte = *sptep;
2636
2637                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2638                         drop_parent_pte(child, sptep);
2639                         kvm_flush_remote_tlbs(vcpu->kvm);
2640                 } else if (pfn != spte_to_pfn(*sptep)) {
2641                         pgprintk("hfn old %llx new %llx\n",
2642                                  spte_to_pfn(*sptep), pfn);
2643                         drop_spte(vcpu->kvm, sptep);
2644                         kvm_flush_remote_tlbs(vcpu->kvm);
2645                 } else
2646                         was_rmapped = 1;
2647         }
2648
2649         if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
2650               true, host_writable)) {
2651                 if (write_fault)
2652                         *emulate = 1;
2653                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2654         }
2655
2656         if (unlikely(is_mmio_spte(*sptep) && emulate))
2657                 *emulate = 1;
2658
2659         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2660         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
2661                  is_large_pte(*sptep)? "2MB" : "4kB",
2662                  *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2663                  *sptep, sptep);
2664         if (!was_rmapped && is_large_pte(*sptep))
2665                 ++vcpu->kvm->stat.lpages;
2666
2667         if (is_shadow_present_pte(*sptep)) {
2668                 if (!was_rmapped) {
2669                         rmap_count = rmap_add(vcpu, sptep, gfn);
2670                         if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2671                                 rmap_recycle(vcpu, sptep, gfn);
2672                 }
2673         }
2674
2675         kvm_release_pfn_clean(pfn);
2676 }
2677
2678 static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2679                                      bool no_dirty_log)
2680 {
2681         struct kvm_memory_slot *slot;
2682
2683         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2684         if (!slot)
2685                 return KVM_PFN_ERR_FAULT;
2686
2687         return gfn_to_pfn_memslot_atomic(slot, gfn);
2688 }
2689
2690 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2691                                     struct kvm_mmu_page *sp,
2692                                     u64 *start, u64 *end)
2693 {
2694         struct page *pages[PTE_PREFETCH_NUM];
2695         unsigned access = sp->role.access;
2696         int i, ret;
2697         gfn_t gfn;
2698
2699         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2700         if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
2701                 return -1;
2702
2703         ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
2704         if (ret <= 0)
2705                 return -1;
2706
2707         for (i = 0; i < ret; i++, gfn++, start++)
2708                 mmu_set_spte(vcpu, start, access, 0, NULL,
2709                              sp->role.level, gfn, page_to_pfn(pages[i]),
2710                              true, true);
2711
2712         return 0;
2713 }
2714
2715 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2716                                   struct kvm_mmu_page *sp, u64 *sptep)
2717 {
2718         u64 *spte, *start = NULL;
2719         int i;
2720
2721         WARN_ON(!sp->role.direct);
2722
2723         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2724         spte = sp->spt + i;
2725
2726         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2727                 if (is_shadow_present_pte(*spte) || spte == sptep) {
2728                         if (!start)
2729                                 continue;
2730                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2731                                 break;
2732                         start = NULL;
2733                 } else if (!start)
2734                         start = spte;
2735         }
2736 }
2737
2738 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2739 {
2740         struct kvm_mmu_page *sp;
2741
2742         /*
2743          * Since it's no accessed bit on EPT, it's no way to
2744          * distinguish between actually accessed translations
2745          * and prefetched, so disable pte prefetch if EPT is
2746          * enabled.
2747          */
2748         if (!shadow_accessed_mask)
2749                 return;
2750
2751         sp = page_header(__pa(sptep));
2752         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2753                 return;
2754
2755         __direct_pte_prefetch(vcpu, sp, sptep);
2756 }
2757
2758 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2759                         int map_writable, int level, gfn_t gfn, pfn_t pfn,
2760                         bool prefault)
2761 {
2762         struct kvm_shadow_walk_iterator iterator;
2763         struct kvm_mmu_page *sp;
2764         int emulate = 0;
2765         gfn_t pseudo_gfn;
2766
2767         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2768                 return 0;
2769
2770         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2771                 if (iterator.level == level) {
2772                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
2773                                      write, &emulate, level, gfn, pfn,
2774                                      prefault, map_writable);
2775                         direct_pte_prefetch(vcpu, iterator.sptep);
2776                         ++vcpu->stat.pf_fixed;
2777                         break;
2778                 }
2779
2780                 drop_large_spte(vcpu, iterator.sptep);
2781                 if (!is_shadow_present_pte(*iterator.sptep)) {
2782                         u64 base_addr = iterator.addr;
2783
2784                         base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2785                         pseudo_gfn = base_addr >> PAGE_SHIFT;
2786                         sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2787                                               iterator.level - 1,
2788                                               1, ACC_ALL, iterator.sptep);
2789
2790                         link_shadow_page(iterator.sptep, sp, true);
2791                 }
2792         }
2793         return emulate;
2794 }
2795
2796 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2797 {
2798         siginfo_t info;
2799
2800         info.si_signo   = SIGBUS;
2801         info.si_errno   = 0;
2802         info.si_code    = BUS_MCEERR_AR;
2803         info.si_addr    = (void __user *)address;
2804         info.si_addr_lsb = PAGE_SHIFT;
2805
2806         send_sig_info(SIGBUS, &info, tsk);
2807 }
2808
2809 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
2810 {
2811         /*
2812          * Do not cache the mmio info caused by writing the readonly gfn
2813          * into the spte otherwise read access on readonly gfn also can
2814          * caused mmio page fault and treat it as mmio access.
2815          * Return 1 to tell kvm to emulate it.
2816          */
2817         if (pfn == KVM_PFN_ERR_RO_FAULT)
2818                 return 1;
2819
2820         if (pfn == KVM_PFN_ERR_HWPOISON) {
2821                 kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
2822                 return 0;
2823         }
2824
2825         return -EFAULT;
2826 }
2827
2828 static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
2829                                         gfn_t *gfnp, pfn_t *pfnp, int *levelp)
2830 {
2831         pfn_t pfn = *pfnp;
2832         gfn_t gfn = *gfnp;
2833         int level = *levelp;
2834
2835         /*
2836          * Check if it's a transparent hugepage. If this would be an
2837          * hugetlbfs page, level wouldn't be set to
2838          * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
2839          * here.
2840          */
2841         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
2842             level == PT_PAGE_TABLE_LEVEL &&
2843             PageTransCompound(pfn_to_page(pfn)) &&
2844             !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
2845                 unsigned long mask;
2846                 /*
2847                  * mmu_notifier_retry was successful and we hold the
2848                  * mmu_lock here, so the pmd can't become splitting
2849                  * from under us, and in turn
2850                  * __split_huge_page_refcount() can't run from under
2851                  * us and we can safely transfer the refcount from
2852                  * PG_tail to PG_head as we switch the pfn to tail to
2853                  * head.
2854                  */
2855                 *levelp = level = PT_DIRECTORY_LEVEL;
2856                 mask = KVM_PAGES_PER_HPAGE(level) - 1;
2857                 VM_BUG_ON((gfn & mask) != (pfn & mask));
2858                 if (pfn & mask) {
2859                         gfn &= ~mask;
2860                         *gfnp = gfn;
2861                         kvm_release_pfn_clean(pfn);
2862                         pfn &= ~mask;
2863                         kvm_get_pfn(pfn);
2864                         *pfnp = pfn;
2865                 }
2866         }
2867 }
2868
2869 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
2870                                 pfn_t pfn, unsigned access, int *ret_val)
2871 {
2872         bool ret = true;
2873
2874         /* The pfn is invalid, report the error! */
2875         if (unlikely(is_error_pfn(pfn))) {
2876                 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
2877                 goto exit;
2878         }
2879
2880         if (unlikely(is_noslot_pfn(pfn)))
2881                 vcpu_cache_mmio_info(vcpu, gva, gfn, access);
2882
2883         ret = false;
2884 exit:
2885         return ret;
2886 }
2887
2888 static bool page_fault_can_be_fast(u32 error_code)
2889 {
2890         /*
2891          * Do not fix the mmio spte with invalid generation number which
2892          * need to be updated by slow page fault path.
2893          */
2894         if (unlikely(error_code & PFERR_RSVD_MASK))
2895                 return false;
2896
2897         /*
2898          * #PF can be fast only if the shadow page table is present and it
2899          * is caused by write-protect, that means we just need change the
2900          * W bit of the spte which can be done out of mmu-lock.
2901          */
2902         if (!(error_code & PFERR_PRESENT_MASK) ||
2903               !(error_code & PFERR_WRITE_MASK))
2904                 return false;
2905
2906         return true;
2907 }
2908
2909 static bool
2910 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2911                         u64 *sptep, u64 spte)
2912 {
2913         gfn_t gfn;
2914
2915         WARN_ON(!sp->role.direct);
2916
2917         /*
2918          * The gfn of direct spte is stable since it is calculated
2919          * by sp->gfn.
2920          */
2921         gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
2922
2923         /*
2924          * Theoretically we could also set dirty bit (and flush TLB) here in
2925          * order to eliminate unnecessary PML logging. See comments in
2926          * set_spte. But fast_page_fault is very unlikely to happen with PML
2927          * enabled, so we do not do this. This might result in the same GPA
2928          * to be logged in PML buffer again when the write really happens, and
2929          * eventually to be called by mark_page_dirty twice. But it's also no
2930          * harm. This also avoids the TLB flush needed after setting dirty bit
2931          * so non-PML cases won't be impacted.
2932          *
2933          * Compare with set_spte where instead shadow_dirty_mask is set.
2934          */
2935         if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
2936                 mark_page_dirty(vcpu->kvm, gfn);
2937
2938         return true;
2939 }
2940
2941 /*
2942  * Return value:
2943  * - true: let the vcpu to access on the same address again.
2944  * - false: let the real page fault path to fix it.
2945  */
2946 static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
2947                             u32 error_code)
2948 {
2949         struct kvm_shadow_walk_iterator iterator;
2950         struct kvm_mmu_page *sp;
2951         bool ret = false;
2952         u64 spte = 0ull;
2953
2954         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2955                 return false;
2956
2957         if (!page_fault_can_be_fast(error_code))
2958                 return false;
2959
2960         walk_shadow_page_lockless_begin(vcpu);
2961         for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
2962                 if (!is_shadow_present_pte(spte) || iterator.level < level)
2963                         break;
2964
2965         /*
2966          * If the mapping has been changed, let the vcpu fault on the
2967          * same address again.
2968          */
2969         if (!is_rmap_spte(spte)) {
2970                 ret = true;
2971                 goto exit;
2972         }
2973
2974         sp = page_header(__pa(iterator.sptep));
2975         if (!is_last_spte(spte, sp->role.level))
2976                 goto exit;
2977
2978         /*
2979          * Check if it is a spurious fault caused by TLB lazily flushed.
2980          *
2981          * Need not check the access of upper level table entries since
2982          * they are always ACC_ALL.
2983          */
2984          if (is_writable_pte(spte)) {
2985                 ret = true;
2986                 goto exit;
2987         }
2988
2989         /*
2990          * Currently, to simplify the code, only the spte write-protected
2991          * by dirty-log can be fast fixed.
2992          */
2993         if (!spte_is_locklessly_modifiable(spte))
2994                 goto exit;
2995
2996         /*
2997          * Do not fix write-permission on the large spte since we only dirty
2998          * the first page into the dirty-bitmap in fast_pf_fix_direct_spte()
2999          * that means other pages are missed if its slot is dirty-logged.
3000          *
3001          * Instead, we let the slow page fault path create a normal spte to
3002          * fix the access.
3003          *
3004          * See the comments in kvm_arch_commit_memory_region().
3005          */
3006         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
3007                 goto exit;
3008
3009         /*
3010          * Currently, fast page fault only works for direct mapping since
3011          * the gfn is not stable for indirect shadow page.
3012          * See Documentation/virtual/kvm/locking.txt to get more detail.
3013          */
3014         ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);
3015 exit:
3016         trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
3017                               spte, ret);
3018         walk_shadow_page_lockless_end(vcpu);
3019
3020         return ret;
3021 }
3022
3023 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3024                          gva_t gva, pfn_t *pfn, bool write, bool *writable);
3025 static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
3026
3027 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
3028                          gfn_t gfn, bool prefault)
3029 {
3030         int r;
3031         int level;
3032         int force_pt_level;
3033         pfn_t pfn;
3034         unsigned long mmu_seq;
3035         bool map_writable, write = error_code & PFERR_WRITE_MASK;
3036
3037         force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
3038         if (likely(!force_pt_level)) {
3039                 level = mapping_level(vcpu, gfn);
3040                 /*
3041                  * This path builds a PAE pagetable - so we can map
3042                  * 2mb pages at maximum. Therefore check if the level
3043                  * is larger than that.
3044                  */
3045                 if (level > PT_DIRECTORY_LEVEL)
3046                         level = PT_DIRECTORY_LEVEL;
3047
3048                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3049         } else
3050                 level = PT_PAGE_TABLE_LEVEL;
3051
3052         if (fast_page_fault(vcpu, v, level, error_code))
3053                 return 0;
3054
3055         mmu_seq = vcpu->kvm->mmu_notifier_seq;
3056         smp_rmb();
3057
3058         if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3059                 return 0;
3060
3061         if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
3062                 return r;
3063
3064         spin_lock(&vcpu->kvm->mmu_lock);
3065         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3066                 goto out_unlock;
3067         make_mmu_pages_available(vcpu);
3068         if (likely(!force_pt_level))
3069                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3070         r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
3071                          prefault);
3072         spin_unlock(&vcpu->kvm->mmu_lock);
3073
3074
3075         return r;
3076
3077 out_unlock:
3078         spin_unlock(&vcpu->kvm->mmu_lock);
3079         kvm_release_pfn_clean(pfn);
3080         return 0;
3081 }
3082
3083
3084 static void mmu_free_roots(struct kvm_vcpu *vcpu)
3085 {
3086         int i;
3087         struct kvm_mmu_page *sp;
3088         LIST_HEAD(invalid_list);
3089
3090         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3091                 return;
3092
3093         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
3094             (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
3095              vcpu->arch.mmu.direct_map)) {
3096                 hpa_t root = vcpu->arch.mmu.root_hpa;
3097
3098                 spin_lock(&vcpu->kvm->mmu_lock);
3099                 sp = page_header(root);
3100                 --sp->root_count;
3101                 if (!sp->root_count && sp->role.invalid) {
3102                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3103                         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3104                 }
3105                 spin_unlock(&vcpu->kvm->mmu_lock);
3106                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3107                 return;
3108         }
3109
3110         spin_lock(&vcpu->kvm->mmu_lock);
3111         for (i = 0; i < 4; ++i) {
3112                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3113
3114                 if (root) {
3115                         root &= PT64_BASE_ADDR_MASK;
3116                         sp = page_header(root);
3117                         --sp->root_count;
3118                         if (!sp->root_count && sp->role.invalid)
3119                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3120                                                          &invalid_list);
3121                 }
3122                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3123         }
3124         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3125         spin_unlock(&vcpu->kvm->mmu_lock);
3126         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3127 }
3128
3129 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3130 {
3131         int ret = 0;
3132
3133         if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3134                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3135                 ret = 1;
3136         }
3137
3138         return ret;
3139 }
3140
3141 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3142 {
3143         struct kvm_mmu_page *sp;
3144         unsigned i;
3145
3146         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3147                 spin_lock(&vcpu->kvm->mmu_lock);
3148                 make_mmu_pages_available(vcpu);
3149                 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
3150                                       1, ACC_ALL, NULL);
3151                 ++sp->root_count;
3152                 spin_unlock(&vcpu->kvm->mmu_lock);
3153                 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
3154         } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
3155                 for (i = 0; i < 4; ++i) {
3156                         hpa_t root = vcpu->arch.mmu.pae_root[i];
3157
3158                         MMU_WARN_ON(VALID_PAGE(root));
3159                         spin_lock(&vcpu->kvm->mmu_lock);
3160                         make_mmu_pages_available(vcpu);
3161                         sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3162                                               i << 30,
3163                                               PT32_ROOT_LEVEL, 1, ACC_ALL,
3164                                               NULL);
3165                         root = __pa(sp->spt);
3166                         ++sp->root_count;
3167                         spin_unlock(&vcpu->kvm->mmu_lock);
3168                         vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
3169                 }
3170                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3171         } else
3172                 BUG();
3173
3174         return 0;
3175 }
3176
3177 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3178 {
3179         struct kvm_mmu_page *sp;
3180         u64 pdptr, pm_mask;
3181         gfn_t root_gfn;
3182         int i;
3183
3184         root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3185
3186         if (mmu_check_root(vcpu, root_gfn))
3187                 return 1;
3188
3189         /*
3190          * Do we shadow a long mode page table? If so we need to
3191          * write-protect the guests page table root.
3192          */
3193         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3194                 hpa_t root = vcpu->arch.mmu.root_hpa;
3195
3196                 MMU_WARN_ON(VALID_PAGE(root));
3197
3198                 spin_lock(&vcpu->kvm->mmu_lock);
3199                 make_mmu_pages_available(vcpu);
3200                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3201                                       0, ACC_ALL, NULL);
3202                 root = __pa(sp->spt);
3203                 ++sp->root_count;
3204                 spin_unlock(&vcpu->kvm->mmu_lock);
3205                 vcpu->arch.mmu.root_hpa = root;
3206                 return 0;
3207         }
3208
3209         /*
3210          * We shadow a 32 bit page table. This may be a legacy 2-level
3211          * or a PAE 3-level page table. In either case we need to be aware that
3212          * the shadow page table may be a PAE or a long mode page table.
3213          */
3214         pm_mask = PT_PRESENT_MASK;
3215         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
3216                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3217
3218         for (i = 0; i < 4; ++i) {
3219                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3220
3221                 MMU_WARN_ON(VALID_PAGE(root));
3222                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3223                         pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
3224                         if (!is_present_gpte(pdptr)) {
3225                                 vcpu->arch.mmu.pae_root[i] = 0;
3226                                 continue;
3227                         }
3228                         root_gfn = pdptr >> PAGE_SHIFT;
3229                         if (mmu_check_root(vcpu, root_gfn))
3230                                 return 1;
3231                 }
3232                 spin_lock(&vcpu->kvm->mmu_lock);
3233                 make_mmu_pages_available(vcpu);
3234                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
3235                                       PT32_ROOT_LEVEL, 0,
3236                                       ACC_ALL, NULL);
3237                 root = __pa(sp->spt);
3238                 ++sp->root_count;
3239                 spin_unlock(&vcpu->kvm->mmu_lock);
3240
3241                 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3242         }
3243         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3244
3245         /*
3246          * If we shadow a 32 bit page table with a long mode page
3247          * table we enter this path.
3248          */
3249         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3250                 if (vcpu->arch.mmu.lm_root == NULL) {
3251                         /*
3252                          * The additional page necessary for this is only
3253                          * allocated on demand.
3254                          */
3255
3256                         u64 *lm_root;
3257
3258                         lm_root = (void*)get_zeroed_page(GFP_KERNEL);
3259                         if (lm_root == NULL)
3260                                 return 1;
3261
3262                         lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
3263
3264                         vcpu->arch.mmu.lm_root = lm_root;
3265                 }
3266
3267                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
3268         }
3269
3270         return 0;
3271 }
3272
3273 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
3274 {
3275         if (vcpu->arch.mmu.direct_map)
3276                 return mmu_alloc_direct_roots(vcpu);
3277         else
3278                 return mmu_alloc_shadow_roots(vcpu);
3279 }
3280
3281 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
3282 {
3283         int i;
3284         struct kvm_mmu_page *sp;
3285
3286         if (vcpu->arch.mmu.direct_map)
3287                 return;
3288
3289         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3290                 return;
3291
3292         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3293         kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3294         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3295                 hpa_t root = vcpu->arch.mmu.root_hpa;
3296                 sp = page_header(root);
3297                 mmu_sync_children(vcpu, sp);
3298                 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3299                 return;
3300         }
3301         for (i = 0; i < 4; ++i) {
3302                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3303
3304                 if (root && VALID_PAGE(root)) {
3305                         root &= PT64_BASE_ADDR_MASK;
3306                         sp = page_header(root);
3307                         mmu_sync_children(vcpu, sp);
3308                 }
3309         }
3310         kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3311 }
3312
3313 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3314 {
3315         spin_lock(&vcpu->kvm->mmu_lock);
3316         mmu_sync_roots(vcpu);
3317         spin_unlock(&vcpu->kvm->mmu_lock);
3318 }
3319 EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3320
3321 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3322                                   u32 access, struct x86_exception *exception)
3323 {
3324         if (exception)
3325                 exception->error_code = 0;
3326         return vaddr;
3327 }
3328
3329 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3330                                          u32 access,
3331                                          struct x86_exception *exception)
3332 {
3333         if (exception)
3334                 exception->error_code = 0;
3335         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3336 }
3337
3338 static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3339 {
3340         if (direct)
3341                 return vcpu_match_mmio_gpa(vcpu, addr);
3342
3343         return vcpu_match_mmio_gva(vcpu, addr);
3344 }
3345
3346
3347 /*
3348  * On direct hosts, the last spte is only allows two states
3349  * for mmio page fault:
3350  *   - It is the mmio spte
3351  *   - It is zapped or it is being zapped.
3352  *
3353  * This function completely checks the spte when the last spte
3354  * is not the mmio spte.
3355  */
3356 static bool check_direct_spte_mmio_pf(u64 spte)
3357 {
3358         return __check_direct_spte_mmio_pf(spte);
3359 }
3360
3361 static u64 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr)
3362 {
3363         struct kvm_shadow_walk_iterator iterator;
3364         u64 spte = 0ull;
3365
3366         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3367                 return spte;
3368
3369         walk_shadow_page_lockless_begin(vcpu);
3370         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
3371                 if (!is_shadow_present_pte(spte))
3372                         break;
3373         walk_shadow_page_lockless_end(vcpu);
3374
3375         return spte;
3376 }
3377
3378 int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3379 {
3380         u64 spte;
3381