ARM: pgtable: add pud-level code

Add pud_offset() et.al. between the pgd and pmd code in preparation of
using pgtable-nopud.h rather than 4level-fixup.h.

This incorporates a fix from Jamie Iles <jamie@jamieiles.com> for
uaccess_with_memcpy.c.

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index ca567914d303b89c77a988bd874784ccb0a8825d..5750704e02718b9247704021a0832f161bcbaf3b 100644 (file)
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -301,6 +301,7 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 #define pgd_present(pgd)       (1)
 #define pgd_clear(pgdp)                do { } while (0)
 #define set_pgd(pgd,pgdp)      do { } while (0)
+#define set_pud(pud,pudp)      do { } while (0)
 
 
 /* Find an entry in the second-level page table.. */

diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
index e2d2f2cd0c4f3b5c9b9c958aa65d073bcbbe722f..8b9b13649f8170304245466a9409ce5e5ae2018b 100644 (file)
--- a/arch/arm/lib/uaccess_with_memcpy.c
+++ b/arch/arm/lib/uaccess_with_memcpy.c
@@ -27,13 +27,18 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
        pgd_t *pgd;
        pmd_t *pmd;
        pte_t *pte;
+       pud_t *pud;
        spinlock_t *ptl;
 
        pgd = pgd_offset(current->mm, addr);
        if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
                return 0;
 
-       pmd = pmd_offset(pgd, addr);
+       pud = pud_offset(pgd, addr);
+       if (unlikely(pud_none(*pud) || pud_bad(*pud)))
+               return 0;
+
+       pmd = pmd_offset(pud, addr);
        if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
                return 0;
 

diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index ac6a36142fcd5a28084b3c669fac9e800fd65497..a9bdfcda23f498db2a15207310ce7450833038ab 100644 (file)
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -148,6 +148,7 @@ static int __init consistent_init(void)
 {
        int ret = 0;
        pgd_t *pgd;
+       pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
        int i = 0;
@@ -155,7 +156,15 @@ static int __init consistent_init(void)
 
        do {
                pgd = pgd_offset(&init_mm, base);
-               pmd = pmd_alloc(&init_mm, pgd, base);
+
+               pud = pud_alloc(&init_mm, pgd, base);
+               if (!pud) {
+                       printk(KERN_ERR "%s: no pud tables\n", __func__);
+                       ret = -ENOMEM;
+                       break;
+               }
+
+               pmd = pmd_alloc(&init_mm, pud, base);
                if (!pmd) {
                        printk(KERN_ERR "%s: no pmd tables\n", __func__);
                        ret = -ENOMEM;

diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 01210dba02217302a3757ecc93810d46d089d43a..7cab791794218b6c49917bf0471152ea810f9e55 100644 (file)
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -95,6 +95,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
 {
        spinlock_t *ptl;
        pgd_t *pgd;
+       pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
        int ret;
@@ -103,7 +104,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
        if (pgd_none_or_clear_bad(pgd))
                return 0;
 
-       pmd = pmd_offset(pgd, address);
+       pud = pud_offset(pgd, address);
+       if (pud_none_or_clear_bad(pud))
+               return 0;
+
+       pmd = pmd_offset(pud, address);
        if (pmd_none_or_clear_bad(pmd))
                return 0;
 

diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index ef0e24f578ef203c89df15493840cba009f7d94e..bc0e1d88fd3ba8b7863edfc9eca9cb11d90413dd 100644 (file)
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -80,6 +80,7 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
                        addr, (long long)pgd_val(*pgd));
 
        do {
+               pud_t *pud;
                pmd_t *pmd;
                pte_t *pte;
 
@@ -91,7 +92,19 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
                        break;
                }
 
-               pmd = pmd_offset(pgd, addr);
+               pud = pud_offset(pgd, addr);
+               if (PTRS_PER_PUD != 1)
+                       printk(", *pud=%08lx", pud_val(*pud));
+
+               if (pud_none(*pud))
+                       break;
+
+               if (pud_bad(*pud)) {
+                       printk("(bad)");
+                       break;
+               }
+
+               pmd = pmd_offset(pud, addr);
                if (PTRS_PER_PMD != 1)
                        printk(", *pmd=%08llx", (long long)pmd_val(*pmd));
 
@@ -390,6 +403,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 {
        unsigned int index;
        pgd_t *pgd, *pgd_k;
+       pud_t *pud, *pud_k;
        pmd_t *pmd, *pmd_k;
 
        if (addr < TASK_SIZE)
@@ -408,12 +422,19 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 
        if (pgd_none(*pgd_k))
                goto bad_area;
-
        if (!pgd_present(*pgd))
                set_pgd(pgd, *pgd_k);
 
-       pmd_k = pmd_offset(pgd_k, addr);
-       pmd   = pmd_offset(pgd, addr);
+       pud = pud_offset(pgd, addr);
+       pud_k = pud_offset(pgd_k, addr);
+
+       if (pud_none(*pud_k))
+               goto bad_area;
+       if (!pud_present(*pud))
+               set_pud(pud, *pud_k);
+
+       pmd = pmd_offset(pud, addr);
+       pmd_k = pmd_offset(pud_k, addr);
 
        /*
         * On ARM one Linux PGD entry contains two hardware entries (see page

diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 57299446f7871d8b77adacdc27f7644b5e4c2168..2be9139a4ef3cc5af97f03a30eefefe7d019740e 100644 (file)
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -4,10 +4,10 @@
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 
-static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end,
+static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
        unsigned long prot)
 {
-       pmd_t *pmd = pmd_offset(pgd, addr);
+       pmd_t *pmd = pmd_offset(pud, addr);
 
        addr = (addr & PMD_MASK) | prot;
        pmd[0] = __pmd(addr);
@@ -16,6 +16,18 @@ static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end,
        flush_pmd_entry(pmd);
 }
 
+static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+       unsigned long prot)
+{
+       pud_t *pud = pud_offset(pgd, addr);
+       unsigned long next;
+
+       do {
+               next = pud_addr_end(addr, end);
+               idmap_add_pmd(pud, addr, next, prot);
+       } while (pud++, addr = next, addr != end);
+}
+
 void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
 {
        unsigned long prot, next;
@@ -27,17 +39,28 @@ void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
        pgd += pgd_index(addr);
        do {
                next = pgd_addr_end(addr, end);
-               idmap_add_pmd(pgd, addr, next, prot);
+               idmap_add_pud(pgd, addr, next, prot);
        } while (pgd++, addr = next, addr != end);
 }
 
 #ifdef CONFIG_SMP
-static void idmap_del_pmd(pgd_t *pgd, unsigned long addr, unsigned long end)
+static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end)
 {
-       pmd_t *pmd = pmd_offset(pgd, addr);
+       pmd_t *pmd = pmd_offset(pud, addr);
        pmd_clear(pmd);
 }
 
+static void idmap_del_pud(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+       pud_t *pud = pud_offset(pgd, addr);
+       unsigned long next;
+
+       do {
+               next = pud_addr_end(addr, end);
+               idmap_del_pmd(pud, addr, next);
+       } while (pud++, addr = next, addr != end);
+}
+
 void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
 {
        unsigned long next;
@@ -45,7 +68,7 @@ void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
        pgd += pgd_index(addr);
        do {
                next = pgd_addr_end(addr, end);
-               idmap_del_pmd(pgd, addr, next);
+               idmap_del_pud(pgd, addr, next);
        } while (pgd++, addr = next, addr != end);
 }
 #endif

diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index 36960df5fb762a990be65bba9d92efba3657d8ed..d2384106af9cbb32f96a61d54570ef39d391c5b7 100644 (file)
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -7,7 +7,7 @@ extern pmd_t *top_pmd;
 
 static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt)
 {
-       return pmd_offset(pgd, virt);
+       return pmd_offset(pud_offset(pgd, virt), virt);
 }
 
 static inline pmd_t *pmd_off_k(unsigned long virt)

diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 176749010935ed24aa5e84c44db50652fca3bae6..82ef6966ae098c2e034073394896c6e02d5aae6b 100644 (file)
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -550,11 +550,11 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
        } while (pte++, addr += PAGE_SIZE, addr != end);
 }
 
-static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
+static void __init alloc_init_section(pud_t *pud, unsigned long addr,
                                      unsigned long end, phys_addr_t phys,
                                      const struct mem_type *type)
 {
-       pmd_t *pmd = pmd_offset(pgd, addr);
+       pmd_t *pmd = pmd_offset(pud, addr);
 
        /*
         * Try a section mapping - end, addr and phys must all be aligned
@@ -583,6 +583,19 @@ static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
        }
 }
 
+static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+       unsigned long phys, const struct mem_type *type)
+{
+       pud_t *pud = pud_offset(pgd, addr);
+       unsigned long next;
+
+       do {
+               next = pud_addr_end(addr, end);
+               alloc_init_section(pud, addr, next, phys, type);
+               phys += next - addr;
+       } while (pud++, addr = next, addr != end);
+}
+
 static void __init create_36bit_mapping(struct map_desc *md,
                                        const struct mem_type *type)
 {
@@ -630,7 +643,8 @@ static void __init create_36bit_mapping(struct map_desc *md,
        pgd = pgd_offset_k(addr);
        end = addr + length;
        do {
-               pmd_t *pmd = pmd_offset(pgd, addr);
+               pud_t *pud = pud_offset(pgd, addr);
+               pmd_t *pmd = pmd_offset(pud, addr);
                int i;
 
                for (i = 0; i < 16; i++)
@@ -696,7 +710,7 @@ static void __init create_mapping(struct map_desc *md)
        do {
                unsigned long next = pgd_addr_end(addr, end);
 
-               alloc_init_section(pgd, addr, next, phys, type);
+               alloc_init_pud(pgd, addr, next, phys, type);
 
                phys += next - addr;
                addr = next;

diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 93292a18cf77f35bd9369ad84f3f6f56e948ddc8..f7fafb1741f4c6a78e89b239cd864936f6e5189c 100644 (file)
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -23,6 +23,7 @@
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
        pgd_t *new_pgd, *init_pgd;
+       pud_t *new_pud, *init_pud;
        pmd_t *new_pmd, *init_pmd;
        pte_t *new_pte, *init_pte;
 
@@ -46,7 +47,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
                 * On ARM, first page must always be allocated since it
                 * contains the machine vectors.
                 */
-               new_pmd = pmd_alloc(mm, new_pgd, 0);
+               new_pud = pud_alloc(mm, new_pgd, 0);
+               if (!new_pud)
+                       goto no_pud;
+
+               new_pmd = pmd_alloc(mm, new_pud, 0);
                if (!new_pmd)
                        goto no_pmd;
 
@@ -54,7 +59,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
                if (!new_pte)
                        goto no_pte;
 
-               init_pmd = pmd_offset(init_pgd, 0);
+               init_pud = pud_offset(init_pgd, 0);
+               init_pmd = pmd_offset(init_pud, 0);
                init_pte = pte_offset_map(init_pmd, 0);
                set_pte_ext(new_pte, *init_pte, 0);
                pte_unmap(init_pte);
@@ -66,6 +72,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 no_pte:
        pmd_free(mm, new_pmd);
 no_pmd:
+       pud_free(mm, new_pud);
+no_pud:
        free_pages((unsigned long)new_pgd, 2);
 no_pgd:
        return NULL;
@@ -74,6 +82,7 @@ no_pgd:
 void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
 {
        pgd_t *pgd;
+       pud_t *pud;
        pmd_t *pmd;
        pgtable_t pte;
 
@@ -84,7 +93,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
        if (pgd_none_or_clear_bad(pgd))
                goto no_pgd;
 
-       pmd = pmd_offset(pgd, 0);
+       pud = pud_offset(pgd, 0);
+       if (pud_none_or_clear_bad(pud))
+               goto no_pud;
+
+       pmd = pmd_offset(pud, 0);
        if (pmd_none_or_clear_bad(pmd))
                goto no_pmd;
 
@@ -92,8 +105,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
        pmd_clear(pmd);
        pte_free(mm, pte);
 no_pmd:
-       pgd_clear(pgd);
+       pud_clear(pud);
        pmd_free(mm, pmd);
+no_pud:
+       pgd_clear(pgd);
+       pud_free(mm, pud);
 no_pgd:
        free_pages((unsigned long) pgd_base, 2);
 }