2 * Copyright (C) 2016 - ARM Ltd
4 * stage2 page table helpers
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #ifndef __ARM64_S2_PGTABLE_H_
20 #define __ARM64_S2_PGTABLE_H_
22 #include <linux/hugetlb.h>
23 #include <asm/pgtable.h>
26 * PGDIR_SHIFT determines the size a top-level page table entry can map
27 * and depends on the number of levels in the page table. Compute the
28 * PGDIR_SHIFT for a given number of levels.
30 #define pt_levels_pgdir_shift(lvls) ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - (lvls))
33 * The hardware supports concatenation of up to 16 tables at stage2 entry level
34 * and we use the feature whenever possible.
36 * Now, the minimum number of bits resolved at any level is (PAGE_SHIFT - 3).
37 * On arm64, the smallest PAGE_SIZE supported is 4k, which means
38 * (PAGE_SHIFT - 3) > 4 holds for all page sizes.
39 * This implies, the total number of page table levels at stage2 expected
40 * by the hardware is actually the number of levels required for (IPA_SHIFT - 4)
41 * in normal translations(e.g, stage1), since we cannot have another level in
42 * the range (IPA_SHIFT, IPA_SHIFT - 4).
44 #define stage2_pgtable_levels(ipa) ARM64_HW_PGTABLE_LEVELS((ipa) - 4)
45 #define kvm_stage2_levels(kvm) VTCR_EL2_LVLS(kvm->arch.vtcr)
47 /* stage2_pgdir_shift() is the size mapped by top-level stage2 entry for the VM */
48 #define stage2_pgdir_shift(kvm) pt_levels_pgdir_shift(kvm_stage2_levels(kvm))
49 #define stage2_pgdir_size(kvm) (1ULL << stage2_pgdir_shift(kvm))
50 #define stage2_pgdir_mask(kvm) ~(stage2_pgdir_size(kvm) - 1)
53 * The number of PTRS across all concatenated stage2 tables given by the
54 * number of bits resolved at the initial level.
55 * If we force more levels than necessary, we may have (stage2_pgdir_shift > IPA),
56 * in which case, stage2_pgd_ptrs will have one entry.
58 #define pgd_ptrs_shift(ipa, pgdir_shift) \
59 ((ipa) > (pgdir_shift) ? ((ipa) - (pgdir_shift)) : 0)
60 #define __s2_pgd_ptrs(ipa, lvls) \
61 (1 << (pgd_ptrs_shift((ipa), pt_levels_pgdir_shift(lvls))))
62 #define __s2_pgd_size(ipa, lvls) (__s2_pgd_ptrs((ipa), (lvls)) * sizeof(pgd_t))
64 #define stage2_pgd_ptrs(kvm) __s2_pgd_ptrs(kvm_phys_shift(kvm), kvm_stage2_levels(kvm))
65 #define stage2_pgd_size(kvm) __s2_pgd_size(kvm_phys_shift(kvm), kvm_stage2_levels(kvm))
68 * kvm_mmmu_cache_min_pages() is the number of pages required to install
69 * a stage-2 translation. We pre-allocate the entry level page table at
72 #define kvm_mmu_cache_min_pages(kvm) (kvm_stage2_levels(kvm) - 1)
74 /* Stage2 PUD definitions when the level is present */
75 static inline bool kvm_stage2_has_pud(struct kvm *kvm)
77 return (CONFIG_PGTABLE_LEVELS > 3) && (kvm_stage2_levels(kvm) > 3);
80 #define S2_PUD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
81 #define S2_PUD_SIZE (1UL << S2_PUD_SHIFT)
82 #define S2_PUD_MASK (~(S2_PUD_SIZE - 1))
84 static inline bool stage2_pgd_none(struct kvm *kvm, pgd_t pgd)
86 if (kvm_stage2_has_pud(kvm))
92 static inline void stage2_pgd_clear(struct kvm *kvm, pgd_t *pgdp)
94 if (kvm_stage2_has_pud(kvm))
98 static inline bool stage2_pgd_present(struct kvm *kvm, pgd_t pgd)
100 if (kvm_stage2_has_pud(kvm))
101 return pgd_present(pgd);
106 static inline void stage2_pgd_populate(struct kvm *kvm, pgd_t *pgd, pud_t *pud)
108 if (kvm_stage2_has_pud(kvm))
109 pgd_populate(NULL, pgd, pud);
112 static inline pud_t *stage2_pud_offset(struct kvm *kvm,
113 pgd_t *pgd, unsigned long address)
115 if (kvm_stage2_has_pud(kvm))
116 return pud_offset(pgd, address);
121 static inline void stage2_pud_free(struct kvm *kvm, pud_t *pud)
123 if (kvm_stage2_has_pud(kvm))
127 static inline bool stage2_pud_table_empty(struct kvm *kvm, pud_t *pudp)
129 if (kvm_stage2_has_pud(kvm))
130 return kvm_page_empty(pudp);
135 static inline phys_addr_t
136 stage2_pud_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
138 if (kvm_stage2_has_pud(kvm)) {
139 phys_addr_t boundary = (addr + S2_PUD_SIZE) & S2_PUD_MASK;
141 return (boundary - 1 < end - 1) ? boundary : end;
147 /* Stage2 PMD definitions when the level is present */
148 static inline bool kvm_stage2_has_pmd(struct kvm *kvm)
150 return (CONFIG_PGTABLE_LEVELS > 2) && (kvm_stage2_levels(kvm) > 2);
153 #define S2_PMD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
154 #define S2_PMD_SIZE (1UL << S2_PMD_SHIFT)
155 #define S2_PMD_MASK (~(S2_PMD_SIZE - 1))
157 static inline bool stage2_pud_none(struct kvm *kvm, pud_t pud)
159 if (kvm_stage2_has_pmd(kvm))
160 return pud_none(pud);
165 static inline void stage2_pud_clear(struct kvm *kvm, pud_t *pud)
167 if (kvm_stage2_has_pmd(kvm))
171 static inline bool stage2_pud_present(struct kvm *kvm, pud_t pud)
173 if (kvm_stage2_has_pmd(kvm))
174 return pud_present(pud);
179 static inline void stage2_pud_populate(struct kvm *kvm, pud_t *pud, pmd_t *pmd)
181 if (kvm_stage2_has_pmd(kvm))
182 pud_populate(NULL, pud, pmd);
185 static inline pmd_t *stage2_pmd_offset(struct kvm *kvm,
186 pud_t *pud, unsigned long address)
188 if (kvm_stage2_has_pmd(kvm))
189 return pmd_offset(pud, address);
194 static inline void stage2_pmd_free(struct kvm *kvm, pmd_t *pmd)
196 if (kvm_stage2_has_pmd(kvm))
200 static inline bool stage2_pud_huge(struct kvm *kvm, pud_t pud)
202 if (kvm_stage2_has_pmd(kvm))
203 return pud_huge(pud);
208 static inline bool stage2_pmd_table_empty(struct kvm *kvm, pmd_t *pmdp)
210 if (kvm_stage2_has_pmd(kvm))
211 return kvm_page_empty(pmdp);
216 static inline phys_addr_t
217 stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
219 if (kvm_stage2_has_pmd(kvm)) {
220 phys_addr_t boundary = (addr + S2_PMD_SIZE) & S2_PMD_MASK;
222 return (boundary - 1 < end - 1) ? boundary : end;
228 static inline bool stage2_pte_table_empty(struct kvm *kvm, pte_t *ptep)
230 return kvm_page_empty(ptep);
233 static inline unsigned long stage2_pgd_index(struct kvm *kvm, phys_addr_t addr)
235 return (((addr) >> stage2_pgdir_shift(kvm)) & (stage2_pgd_ptrs(kvm) - 1));
238 static inline phys_addr_t
239 stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
241 phys_addr_t boundary = (addr + stage2_pgdir_size(kvm)) & stage2_pgdir_mask(kvm);
243 return (boundary - 1 < end - 1) ? boundary : end;
246 #endif /* __ARM64_S2_PGTABLE_H_ */