Merge tag 'x86_shstk_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git...

[sfrench/cifs-2.6.git] / mm / internal.h

diff --git a/mm/internal.h b/mm/internal.h
index 018f5c342f90799819c672c4cb5dd8bac64be483..30cf724ddbce3399999d6a9e9816fa133e9c5c4b 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -61,6 +61,12 @@ void page_writeback_init(void);
 #define COMPOUND_MAPPED                0x800000
 #define FOLIO_PAGES_MAPPED     (COMPOUND_MAPPED - 1)
 
+/*
+ * Flags passed to __show_mem() and show_free_areas() to suppress output in
+ * various contexts.
+ */
+#define SHOW_MEM_FILTER_NODES          (0x0001u)       /* disallowed nodes */
+
 /*
  * How many individual pages have an elevated _mapcount.  Excludes
  * the folio's entire_mapcount.
@@ -103,7 +109,7 @@ bool __folio_end_writeback(struct folio *folio);
 void deactivate_file_folio(struct folio *folio);
 void folio_activate(struct folio *folio);
 
-void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt,
+void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
                   struct vm_area_struct *start_vma, unsigned long floor,
                   unsigned long ceiling, bool mm_wr_locked);
 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
@@ -170,6 +176,17 @@ static inline void set_page_refcounted(struct page *page)
        set_page_count(page, 1);
 }
 
+/*
+ * Return true if a folio needs ->release_folio() calling upon it.
+ */
+static inline bool folio_needs_release(struct folio *folio)
+{
+       struct address_space *mapping = folio_mapping(folio);
+
+       return folio_has_private(folio) ||
+               (mapping && mapping_release_always(mapping));
+}
+
 extern unsigned long highest_memmap_pfn;
 
 /*
@@ -390,17 +407,18 @@ static inline void folio_set_order(struct folio *folio, unsigned int order)
        if (WARN_ON_ONCE(!order || !folio_test_large(folio)))
                return;
 
-       folio->_folio_order = order;
+       folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order;
 #ifdef CONFIG_64BIT
        folio->_folio_nr_pages = 1U << order;
 #endif
 }
 
+void folio_undo_large_rmappable(struct folio *folio);
+
 static inline void prep_compound_head(struct page *page, unsigned int order)
 {
        struct folio *folio = (struct folio *)page;
 
-       folio_set_compound_dtor(folio, COMPOUND_PAGE_DTOR);
        folio_set_order(folio, order);
        atomic_set(&folio->_entire_mapcount, -1);
        atomic_set(&folio->_nr_pages_mapped, 0);
@@ -689,7 +707,7 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
        if (fault_flag_allow_retry_first(flags) &&
            !(flags & FAULT_FLAG_RETRY_NOWAIT)) {
                fpin = get_file(vmf->vma->vm_file);
-               mmap_read_unlock(vmf->vma->vm_mm);
+               release_fault_lock(vmf);
        }
        return fpin;
 }
@@ -924,6 +942,13 @@ int migrate_device_coherent_page(struct page *page);
 struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
 int __must_check try_grab_page(struct page *page, unsigned int flags);
 
+/*
+ * mm/huge_memory.c
+ */
+struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
+                                  unsigned long addr, pmd_t *pmd,
+                                  unsigned int flags);
+
 enum {
        /* mark page accessed */
        FOLL_TOUCH = 1 << 16,
@@ -997,6 +1022,16 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma,
        if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
                smp_rmb();
 
+       /*
+        * During GUP-fast we might not get called on the head page for a
+        * hugetlb page that is mapped using cont-PTE, because GUP-fast does
+        * not work with the abstracted hugetlb PTEs that always point at the
+        * head page. For hugetlb, PageAnonExclusive only applies on the head
+        * page (as it cannot be partially COW-shared), so lookup the head page.
+        */
+       if (unlikely(!PageHead(page) && PageHuge(page)))
+               page = compound_head(page);
+
        /*
         * Note that PageKsm() pages cannot be exclusive, and consequently,
         * cannot get pinned.
@@ -1005,6 +1040,7 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma,
 }
 
 extern bool mirrored_kernelcore;
+extern bool memblock_has_mirror(void);
 
 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
 {
@@ -1024,21 +1060,39 @@ static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
        return !(vma->vm_flags & VM_SOFTDIRTY);
 }
 
+static inline void vma_iter_config(struct vma_iterator *vmi,
+               unsigned long index, unsigned long last)
+{
+       MAS_BUG_ON(&vmi->mas, vmi->mas.node != MAS_START &&
+                  (vmi->mas.index > index || vmi->mas.last < index));
+       __mas_set_range(&vmi->mas, index, last - 1);
+}
+
 /*
  * VMA Iterator functions shared between nommu and mmap
  */
-static inline int vma_iter_prealloc(struct vma_iterator *vmi)
+static inline int vma_iter_prealloc(struct vma_iterator *vmi,
+               struct vm_area_struct *vma)
 {
-       return mas_preallocate(&vmi->mas, GFP_KERNEL);
+       return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
 }
 
-static inline void vma_iter_clear(struct vma_iterator *vmi,
-                                 unsigned long start, unsigned long end)
+static inline void vma_iter_clear(struct vma_iterator *vmi)
 {
-       mas_set_range(&vmi->mas, start, end - 1);
        mas_store_prealloc(&vmi->mas, NULL);
 }
 
+static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
+                       unsigned long start, unsigned long end, gfp_t gfp)
+{
+       __mas_set_range(&vmi->mas, start, end - 1);
+       mas_store_gfp(&vmi->mas, NULL, gfp);
+       if (unlikely(mas_is_err(&vmi->mas)))
+               return -ENOMEM;
+
+       return 0;
+}
+
 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
 {
        return mas_walk(&vmi->mas);
@@ -1068,8 +1122,7 @@ static inline void vma_iter_store(struct vma_iterator *vmi,
            ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
                vma_iter_invalidate(vmi);
 
-       vmi->mas.index = vma->vm_start;
-       vmi->mas.last = vma->vm_end - 1;
+       __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
        mas_store_prealloc(&vmi->mas, vma);
 }
 
@@ -1080,8 +1133,7 @@ static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
            ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
                vma_iter_invalidate(vmi);
 
-       vmi->mas.index = vma->vm_start;
-       vmi->mas.last = vma->vm_end - 1;
+       __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
        mas_store_gfp(&vmi->mas, vma, gfp);
        if (unlikely(mas_is_err(&vmi->mas)))
                return -ENOMEM;