mm: optimize compound_head() by avoiding a shared page flag

[sfrench/cifs-2.6.git] / include / linux / mm.h

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 26adfcc0d61a1bd953ba0f3b47d78e37e5840848..695b90437bbc763bba0bb935793450d102f1dbf7 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -267,17 +267,21 @@ static inline int get_page_unless_zero(struct page *page)
        return atomic_inc_not_zero(&page->_count);
 }
 
+static inline struct page *compound_head(struct page *page)
+{
+       if (unlikely(PageTail(page)))
+               return page->first_page;
+       return page;
+}
+
 static inline int page_count(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
-       return atomic_read(&page->_count);
+       return atomic_read(&compound_head(page)->_count);
 }
 
 static inline void get_page(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
+       page = compound_head(page);
        VM_BUG_ON(atomic_read(&page->_count) == 0);
        atomic_inc(&page->_count);
 }
@@ -314,6 +318,18 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
        return (compound_page_dtor *)page[1].lru.next;
 }
 
+static inline int compound_order(struct page *page)
+{
+       if (!PageHead(page))
+               return 0;
+       return (unsigned long)page[1].lru.prev;
+}
+
+static inline void set_compound_order(struct page *page, unsigned long order)
+{
+       page[1].lru.prev = (void *)order;
+}
+
 /*
  * Multiple processes may "see" the same page. E.g. for untouched
  * mappings of /dev/null, all processes see the same page full of
@@ -637,6 +653,7 @@ static inline int page_mapped(struct page *page)
  */
 #define NOPFN_SIGBUS   ((unsigned long) -1)
 #define NOPFN_OOM      ((unsigned long) -2)
+#define NOPFN_REFAULT  ((unsigned long) -3)
 
 /*
  * Different kinds of faults, as returned by handle_mm_fault().
@@ -659,15 +676,11 @@ static inline int page_mapped(struct page *page)
 extern void show_free_areas(void);
 
 #ifdef CONFIG_SHMEM
-struct page *shmem_nopage(struct vm_area_struct *vma,
-                       unsigned long address, int *type);
 int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new);
 struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
                                        unsigned long addr);
 int shmem_lock(struct file *file, int lock, struct user_struct *user);
 #else
-#define shmem_nopage filemap_nopage
-
 static inline int shmem_lock(struct file *file, int lock,
                             struct user_struct *user)
 {
@@ -687,7 +700,6 @@ static inline struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
 }
 #endif
 struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
-extern int shmem_mmap(struct file *file, struct vm_area_struct *vma);
 
 int shmem_zero_setup(struct vm_area_struct *);
 
@@ -785,6 +797,7 @@ extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
 extern void do_invalidatepage(struct page *page, unsigned long offset);
 
 int __set_page_dirty_nobuffers(struct page *page);
+int __set_page_dirty_no_writeback(struct page *page);
 int redirty_page_for_writepage(struct writeback_control *wbc,
                                struct page *page);
 int FASTCALL(set_page_dirty(struct page *page));
@@ -853,8 +866,26 @@ static inline int vma_wants_writenotify(struct vm_area_struct *vma)
 
 extern pte_t *FASTCALL(get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl));
 
+#ifdef __PAGETABLE_PUD_FOLDED
+static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
+                                               unsigned long address)
+{
+       return 0;
+}
+#else
 int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+#endif
+
+#ifdef __PAGETABLE_PMD_FOLDED
+static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
+                                               unsigned long address)
+{
+       return 0;
+}
+#else
 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
+#endif
+
 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
 
@@ -1123,6 +1154,8 @@ unsigned long vmalloc_to_pfn(void *addr);
 int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
                        unsigned long pfn, unsigned long size, pgprot_t);
 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
+int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
+                       unsigned long pfn);
 
 struct page *follow_page(struct vm_area_struct *, unsigned long address,
                        unsigned int foll_flags);
@@ -1131,6 +1164,11 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address,
 #define FOLL_GET       0x04    /* do get_page on page */
 #define FOLL_ANON      0x08    /* give ZERO_PAGE if no pgtable */
 
+typedef int (*pte_fn_t)(pte_t *pte, struct page *pmd_page, unsigned long addr,
+                       void *data);
+extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
+                              unsigned long size, pte_fn_t fn, void *data);
+
 #ifdef CONFIG_PROC_FS
 void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
 #else