mm: introduce wrappers to access mm->nr_ptes

Let's add wrappers for ->nr_ptes with the same interface as for nr_pmd
and nr_pud.

The patch also makes nr_ptes accounting dependent onto CONFIG_MMU.  Page
table accounting doesn't make sense if you don't have page tables.

It's preparation for consolidation of page-table counters in mm_struct.

Link: http://lkml.kernel.org/r/20171006100651.44742-1-kirill.shutemov@linux.intel.com
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index c1c1a5c67da1418c8822fda4b282578175bf6ec3..61e281cb29fb8280e00c88a3ebd0a575645a8c4e 100644 (file)
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -141,7 +141,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
        pte = pmd_pgtable(*pmd);
        pmd_clear(pmd);
        pte_free(mm, pte);
-       atomic_long_dec(&mm->nr_ptes);
+       mm_dec_nr_ptes(mm);
 no_pmd:
        pud_clear(pud);
        pmd_free(mm, pmd);

diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 01f63b4ee2b42bf0c2ed705b44195146db94f1e0..0112d694228861af66efa86c53aae42393b57abb 100644 (file)
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -397,7 +397,7 @@ static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
 
        pmd_clear(pmd);
        pte_free_tlb(tlb, token, addr);
-       atomic_long_dec(&tlb->mm->nr_ptes);
+       mm_dec_nr_ptes(tlb->mm);
 }
 
 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,

diff --git a/arch/unicore32/mm/pgd.c b/arch/unicore32/mm/pgd.c
index c572a28c76c98997f5a4a80af3dc63d3119abc0f..a830a300aaa1b8fa9ee2993640e032bf23c32b75 100644 (file)
--- a/arch/unicore32/mm/pgd.c
+++ b/arch/unicore32/mm/pgd.c
@@ -97,7 +97,7 @@ void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
        pte = pmd_pgtable(*pmd);
        pmd_clear(pmd);
        pte_free(mm, pte);
-       atomic_long_dec(&mm->nr_ptes);
+       mm_dec_nr_ptes(mm);
        pmd_free(mm, pmd);
        mm_dec_nr_pmds(mm);
 free:

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index a05ce6186f996571723ddbaad396cb2a1715938c..9bd2a0294ac17ca63719e198b54b6e1adaeaf2be 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -50,7 +50,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm)
        text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
        lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
        swap = get_mm_counter(mm, MM_SWAPENTS);
-       ptes = PTRS_PER_PTE * sizeof(pte_t) * atomic_long_read(&mm->nr_ptes);
+       ptes = PTRS_PER_PTE * sizeof(pte_t) * mm_nr_ptes(mm);
        pmds = PTRS_PER_PMD * sizeof(pmd_t) * mm_nr_pmds(mm);
        puds = PTRS_PER_PUD * sizeof(pud_t) * mm_nr_puds(mm);
        seq_printf(m,

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 9af86f39d9284d1e78aa2d56be477d9a9809847f..2ca799f0d76294eb110ab7ac5513276ed63dd02d 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1680,6 +1680,38 @@ static inline void mm_dec_nr_pmds(struct mm_struct *mm)
 }
 #endif
 
+#ifdef CONFIG_MMU
+static inline void mm_nr_ptes_init(struct mm_struct *mm)
+{
+       atomic_long_set(&mm->nr_ptes, 0);
+}
+
+static inline unsigned long mm_nr_ptes(const struct mm_struct *mm)
+{
+       return atomic_long_read(&mm->nr_ptes);
+}
+
+static inline void mm_inc_nr_ptes(struct mm_struct *mm)
+{
+       atomic_long_inc(&mm->nr_ptes);
+}
+
+static inline void mm_dec_nr_ptes(struct mm_struct *mm)
+{
+       atomic_long_dec(&mm->nr_ptes);
+}
+#else
+static inline void mm_nr_ptes_init(struct mm_struct *mm) {}
+
+static inline unsigned long mm_nr_ptes(const struct mm_struct *mm)
+{
+       return 0;
+}
+
+static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}
+static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
+#endif
+
 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
 

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index e9e561e02d229fa69a7428bc739b770a2552efa4..e42048020664dab052298170d161a9108a8ef95a 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -401,7 +401,9 @@ struct mm_struct {
         */
        atomic_t mm_count;
 
+#ifdef CONFIG_MMU
        atomic_long_t nr_ptes;                  /* PTE page table pages */
+#endif
 #if CONFIG_PGTABLE_LEVELS > 2
        atomic_long_t nr_pmds;                  /* PMD page table pages */
 #endif

diff --git a/kernel/fork.c b/kernel/fork.c
index a4eb6f2893657a0e41dd92742cb8fdf0c1839acb..946922a30ede53b9a017ae25b1c089f505b62ee9 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -817,7 +817,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
        init_rwsem(&mm->mmap_sem);
        INIT_LIST_HEAD(&mm->mmlist);
        mm->core_state = NULL;
-       atomic_long_set(&mm->nr_ptes, 0);
+       mm_nr_ptes_init(mm);
        mm_nr_pmds_init(mm);
        mm_nr_puds_init(mm);
        mm->map_count = 0;
@@ -873,9 +873,9 @@ static void check_mm(struct mm_struct *mm)
                                          "mm:%p idx:%d val:%ld\n", mm, i, x);
        }
 
-       if (atomic_long_read(&mm->nr_ptes))
+       if (mm_nr_ptes(mm))
                pr_alert("BUG: non-zero nr_ptes on freeing mm: %ld\n",
-                               atomic_long_read(&mm->nr_ptes));
+                               mm_nr_ptes(mm));
        if (mm_nr_pmds(mm))
                pr_alert("BUG: non-zero nr_pmds on freeing mm: %ld\n",
                                mm_nr_pmds(mm));

diff --git a/mm/debug.c b/mm/debug.c
index a12d826bb774c0f333ca7fabc6119d63ef8cc7ee..c9888a6d7875325c95673c8621a9e54a515d9279 100644 (file)
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -136,7 +136,7 @@ void dump_mm(const struct mm_struct *mm)
                mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end,
                mm->pgd, atomic_read(&mm->mm_users),
                atomic_read(&mm->mm_count),
-               atomic_long_read((atomic_long_t *)&mm->nr_ptes),
+               mm_nr_ptes(mm),
                mm_nr_pmds(mm),
                mm_nr_puds(mm),
                mm->map_count,

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index cc65fb87c9db0afd9b6a5b314ae5c3968b9bbf25..3610d81c062af43c77b9a5959c6f356ecc64dafa 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -606,7 +606,7 @@ static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
                pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
                set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
                add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
-               atomic_long_inc(&vma->vm_mm->nr_ptes);
+               mm_inc_nr_ptes(vma->vm_mm);
                spin_unlock(vmf->ptl);
                count_vm_event(THP_FAULT_ALLOC);
        }
@@ -662,7 +662,7 @@ static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
        if (pgtable)
                pgtable_trans_huge_deposit(mm, pmd, pgtable);
        set_pmd_at(mm, haddr, pmd, entry);
-       atomic_long_inc(&mm->nr_ptes);
+       mm_inc_nr_ptes(mm);
        return true;
 }
 
@@ -747,7 +747,7 @@ static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
 
        if (pgtable) {
                pgtable_trans_huge_deposit(mm, pmd, pgtable);
-               atomic_long_inc(&mm->nr_ptes);
+               mm_inc_nr_ptes(mm);
        }
 
        set_pmd_at(mm, addr, pmd, entry);
@@ -978,7 +978,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
        get_page(src_page);
        page_dup_rmap(src_page, true);
        add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
-       atomic_long_inc(&dst_mm->nr_ptes);
+       mm_inc_nr_ptes(dst_mm);
        pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
 
        pmdp_set_wrprotect(src_mm, addr, src_pmd);
@@ -1695,7 +1695,7 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
 
        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
        pte_free(mm, pgtable);
-       atomic_long_dec(&mm->nr_ptes);
+       mm_dec_nr_ptes(mm);
 }
 
 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 43cb3043311bdab1c302955533d7aee137e5b869..ea4ff259b67195b32d3335795d4de80c4e6e5bcf 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1270,7 +1270,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
                        _pmd = pmdp_collapse_flush(vma, addr, pmd);
                        spin_unlock(ptl);
                        up_write(&vma->vm_mm->mmap_sem);
-                       atomic_long_dec(&vma->vm_mm->nr_ptes);
+                       mm_dec_nr_ptes(vma->vm_mm);
                        pte_free(vma->vm_mm, pmd_pgtable(_pmd));
                }
        }

diff --git a/mm/memory.c b/mm/memory.c
index 6bbd4078ec98962cc46c3f1f3c344a74c2ca4a0f..6dec21b182b0df46bfe1c3ab215b58dbb9c7058d 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -438,7 +438,7 @@ static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
        pgtable_t token = pmd_pgtable(*pmd);
        pmd_clear(pmd);
        pte_free_tlb(tlb, token, addr);
-       atomic_long_dec(&tlb->mm->nr_ptes);
+       mm_dec_nr_ptes(tlb->mm);
 }
 
 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
@@ -666,7 +666,7 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
 
        ptl = pmd_lock(mm, pmd);
        if (likely(pmd_none(*pmd))) {   /* Has another populated it ? */
-               atomic_long_inc(&mm->nr_ptes);
+               mm_inc_nr_ptes(mm);
                pmd_populate(mm, pmd, new);
                new = NULL;
        }
@@ -3238,7 +3238,7 @@ static int pte_alloc_one_map(struct vm_fault *vmf)
                        goto map_pte;
                }
 
-               atomic_long_inc(&vma->vm_mm->nr_ptes);
+               mm_inc_nr_ptes(vma->vm_mm);
                pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
                spin_unlock(vmf->ptl);
                vmf->prealloc_pte = NULL;
@@ -3297,7 +3297,7 @@ static void deposit_prealloc_pte(struct vm_fault *vmf)
         * We are going to consume the prealloc table,
         * count that as nr_ptes.
         */
-       atomic_long_inc(&vma->vm_mm->nr_ptes);
+       mm_inc_nr_ptes(vma->vm_mm);
        vmf->prealloc_pte = NULL;
 }
 

diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index f642a45b7f14b1f1d975d84ecaab332ceaedabcd..f9300141480ea182f591a7c4f0b7cf8d941f3f10 100644 (file)
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -221,8 +221,7 @@ unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
         * task's rss, pagetable and swap space use.
         */
        points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
-               atomic_long_read(&p->mm->nr_ptes) + mm_nr_pmds(p->mm) +
-               mm_nr_puds(p->mm);
+               mm_nr_ptes(p->mm) + mm_nr_pmds(p->mm) + mm_nr_puds(p->mm);
        task_unlock(p);
 
        /*
@@ -417,7 +416,7 @@ static void dump_tasks(struct mem_cgroup *memcg, const nodemask_t *nodemask)
                pr_info("[%5d] %5d %5d %8lu %8lu %7ld %7ld %7ld %8lu         %5hd %s\n",
                        task->pid, from_kuid(&init_user_ns, task_uid(task)),
                        task->tgid, task->mm->total_vm, get_mm_rss(task->mm),
-                       atomic_long_read(&task->mm->nr_ptes),
+                       mm_nr_ptes(task->mm),
                        mm_nr_pmds(task->mm),
                        mm_nr_puds(task->mm),
                        get_mm_counter(task->mm, MM_SWAPENTS),