2 * Helper routines for building identity mapping page tables. This is
3 * included by both the compressed kernel and the regular kernel.
6 static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
7 unsigned long addr, unsigned long end)
10 for (; addr < end; addr += PMD_SIZE) {
11 pmd_t *pmd = pmd_page + pmd_index(addr);
13 if (pmd_present(*pmd))
16 set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
20 static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
21 unsigned long addr, unsigned long end)
25 for (; addr < end; addr = next) {
26 pud_t *pud = pud_page + pud_index(addr);
29 next = (addr & PUD_MASK) + PUD_SIZE;
33 if (info->direct_gbpages) {
36 if (pud_present(*pud))
40 pudval = __pud((addr - info->offset) | info->page_flag);
45 if (pud_present(*pud)) {
46 pmd = pmd_offset(pud, 0);
47 ident_pmd_init(info, pmd, addr, next);
50 pmd = (pmd_t *)info->alloc_pgt_page(info->context);
53 ident_pmd_init(info, pmd, addr, next);
54 set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
60 static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
61 unsigned long addr, unsigned long end)
65 for (; addr < end; addr = next) {
66 p4d_t *p4d = p4d_page + p4d_index(addr);
69 next = (addr & P4D_MASK) + P4D_SIZE;
73 if (p4d_present(*p4d)) {
74 pud = pud_offset(p4d, 0);
75 ident_pud_init(info, pud, addr, next);
78 pud = (pud_t *)info->alloc_pgt_page(info->context);
81 ident_pud_init(info, pud, addr, next);
82 set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag));
88 int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
89 unsigned long pstart, unsigned long pend)
91 unsigned long addr = pstart + info->offset;
92 unsigned long end = pend + info->offset;
96 /* Set the default pagetable flags if not supplied */
97 if (!info->kernpg_flag)
98 info->kernpg_flag = _KERNPG_TABLE;
100 for (; addr < end; addr = next) {
101 pgd_t *pgd = pgd_page + pgd_index(addr);
104 next = (addr & PGDIR_MASK) + PGDIR_SIZE;
108 if (pgd_present(*pgd)) {
109 p4d = p4d_offset(pgd, 0);
110 result = ident_p4d_init(info, p4d, addr, next);
116 p4d = (p4d_t *)info->alloc_pgt_page(info->context);
119 result = ident_p4d_init(info, p4d, addr, next);
122 if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
123 set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
126 * With p4d folded, pgd is equal to p4d.
127 * The pgd entry has to point to the pud page table in this case.
129 pud_t *pud = pud_offset(p4d, 0);
130 set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag));