Merge branch 'xarray' of git://git.infradead.org/users/willy/linux-dax

[sfrench/cifs-2.6.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 9eb79c384616ea72d42c39c8f04013caf350d2f2..4e4ef8fa479d53b7ee7c4c8fcb86985acb790c8a 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -852,11 +852,10 @@ static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
 }
 
 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
-               pmd_t *pmd, int flags)
+               pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
 {
        unsigned long pfn = pmd_pfn(*pmd);
        struct mm_struct *mm = vma->vm_mm;
-       struct dev_pagemap *pgmap;
        struct page *page;
 
        assert_spin_locked(pmd_lockptr(mm, pmd));
@@ -886,12 +885,11 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
                return ERR_PTR(-EEXIST);
 
        pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
-       pgmap = get_dev_pagemap(pfn, NULL);
-       if (!pgmap)
+       *pgmap = get_dev_pagemap(pfn, *pgmap);
+       if (!*pgmap)
                return ERR_PTR(-EFAULT);
        page = pfn_to_page(pfn);
        get_page(page);
-       put_dev_pagemap(pgmap);
 
        return page;
 }
@@ -1000,11 +998,10 @@ static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
 }
 
 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
-               pud_t *pud, int flags)
+               pud_t *pud, int flags, struct dev_pagemap **pgmap)
 {
        unsigned long pfn = pud_pfn(*pud);
        struct mm_struct *mm = vma->vm_mm;
-       struct dev_pagemap *pgmap;
        struct page *page;
 
        assert_spin_locked(pud_lockptr(mm, pud));
@@ -1028,12 +1025,11 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
                return ERR_PTR(-EEXIST);
 
        pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
-       pgmap = get_dev_pagemap(pfn, NULL);
-       if (!pgmap)
+       *pgmap = get_dev_pagemap(pfn, *pgmap);
+       if (!*pgmap)
                return ERR_PTR(-EFAULT);
        page = pfn_to_page(pfn);
        get_page(page);
-       put_dev_pagemap(pgmap);
 
        return page;
 }
@@ -1562,8 +1558,20 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
         * We are not sure a pending tlb flush here is for a huge page
         * mapping or not. Hence use the tlb range variant
         */
-       if (mm_tlb_flush_pending(vma->vm_mm))
+       if (mm_tlb_flush_pending(vma->vm_mm)) {
                flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
+               /*
+                * change_huge_pmd() released the pmd lock before
+                * invalidating the secondary MMUs sharing the primary
+                * MMU pagetables (with ->invalidate_range()). The
+                * mmu_notifier_invalidate_range_end() (which
+                * internally calls ->invalidate_range()) in
+                * change_pmd_range() will run after us, so we can't
+                * rely on it here and we need an explicit invalidate.
+                */
+               mmu_notifier_invalidate_range(vma->vm_mm, haddr,
+                                             haddr + HPAGE_PMD_SIZE);
+       }
 
        /*
         * Migrate the THP to the requested node, returns with page unlocked
@@ -1780,7 +1788,7 @@ static pmd_t move_soft_dirty_pmd(pmd_t pmd)
 
 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
                  unsigned long new_addr, unsigned long old_end,
-                 pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush)
+                 pmd_t *old_pmd, pmd_t *new_pmd)
 {
        spinlock_t *old_ptl, *new_ptl;
        pmd_t pmd;
@@ -1811,7 +1819,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
                if (new_ptl != old_ptl)
                        spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
                pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
-               if (pmd_present(pmd) && pmd_dirty(pmd))
+               if (pmd_present(pmd))
                        force_flush = true;
                VM_BUG_ON(!pmd_none(*new_pmd));
 
@@ -1822,12 +1830,10 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
                }
                pmd = move_soft_dirty_pmd(pmd);
                set_pmd_at(mm, new_addr, new_pmd, pmd);
-               if (new_ptl != old_ptl)
-                       spin_unlock(new_ptl);
                if (force_flush)
                        flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
-               else
-                       *need_flush = true;
+               if (new_ptl != old_ptl)
+                       spin_unlock(new_ptl);
                spin_unlock(old_ptl);
                return true;
        }
@@ -2371,6 +2377,7 @@ static void __split_huge_page_tail(struct page *head, int tail,
                         (1L << PG_mlocked) |
                         (1L << PG_uptodate) |
                         (1L << PG_active) |
+                        (1L << PG_workingset) |
                         (1L << PG_locked) |
                         (1L << PG_unevictable) |
                         (1L << PG_dirty)));
@@ -2882,9 +2889,6 @@ void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
        if (!(pvmw->pmd && !pvmw->pte))
                return;
 
-       mmu_notifier_invalidate_range_start(mm, address,
-                       address + HPAGE_PMD_SIZE);
-
        flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
        pmdval = *pvmw->pmd;
        pmdp_invalidate(vma, address, pvmw->pmd);
@@ -2897,9 +2901,6 @@ void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
        set_pmd_at(mm, address, pvmw->pmd, pmdswp);
        page_remove_rmap(page, true);
        put_page(page);
-
-       mmu_notifier_invalidate_range_end(mm, address,
-                       address + HPAGE_PMD_SIZE);
 }
 
 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
@@ -2928,7 +2929,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
        else
                page_add_file_rmap(new, true);
        set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
-       if (vma->vm_flags & VM_LOCKED)
+       if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
                mlock_vma_page(new);
        update_mmu_cache_pmd(vma, address, pvmw->pmd);
 }