mm: optimize compound_head() by avoiding a shared page flag

[sfrench/cifs-2.6.git] / include / linux / mm.h

diff --git a/include/linux/mm.h b/include/linux/mm.h
index a0eec16eb0bdbb6a2092b443f299f28c5c11cc25..695b90437bbc763bba0bb935793450d102f1dbf7 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -267,17 +267,21 @@ static inline int get_page_unless_zero(struct page *page)
        return atomic_inc_not_zero(&page->_count);
 }
 
+static inline struct page *compound_head(struct page *page)
+{
+       if (unlikely(PageTail(page)))
+               return page->first_page;
+       return page;
+}
+
 static inline int page_count(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
-       return atomic_read(&page->_count);
+       return atomic_read(&compound_head(page)->_count);
 }
 
 static inline void get_page(struct page *page)
 {
-       if (unlikely(PageCompound(page)))
-               page = (struct page *)page_private(page);
+       page = compound_head(page);
        VM_BUG_ON(atomic_read(&page->_count) == 0);
        atomic_inc(&page->_count);
 }
@@ -314,6 +318,18 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
        return (compound_page_dtor *)page[1].lru.next;
 }
 
+static inline int compound_order(struct page *page)
+{
+       if (!PageHead(page))
+               return 0;
+       return (unsigned long)page[1].lru.prev;
+}
+
+static inline void set_compound_order(struct page *page, unsigned long order)
+{
+       page[1].lru.prev = (void *)order;
+}
+
 /*
  * Multiple processes may "see" the same page. E.g. for untouched
  * mappings of /dev/null, all processes see the same page full of
@@ -660,15 +676,11 @@ static inline int page_mapped(struct page *page)
 extern void show_free_areas(void);
 
 #ifdef CONFIG_SHMEM
-struct page *shmem_nopage(struct vm_area_struct *vma,
-                       unsigned long address, int *type);
 int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new);
 struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
                                        unsigned long addr);
 int shmem_lock(struct file *file, int lock, struct user_struct *user);
 #else
-#define shmem_nopage filemap_nopage
-
 static inline int shmem_lock(struct file *file, int lock,
                             struct user_struct *user)
 {
@@ -688,7 +700,6 @@ static inline struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
 }
 #endif
 struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
-extern int shmem_mmap(struct file *file, struct vm_area_struct *vma);
 
 int shmem_zero_setup(struct vm_area_struct *);
 
@@ -855,8 +866,26 @@ static inline int vma_wants_writenotify(struct vm_area_struct *vma)
 
 extern pte_t *FASTCALL(get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl));
 
+#ifdef __PAGETABLE_PUD_FOLDED
+static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
+                                               unsigned long address)
+{
+       return 0;
+}
+#else
 int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+#endif
+
+#ifdef __PAGETABLE_PMD_FOLDED
+static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
+                                               unsigned long address)
+{
+       return 0;
+}
+#else
 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
+#endif
+
 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
 
@@ -1135,6 +1164,11 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address,
 #define FOLL_GET       0x04    /* do get_page on page */
 #define FOLL_ANON      0x08    /* give ZERO_PAGE if no pgtable */
 
+typedef int (*pte_fn_t)(pte_t *pte, struct page *pmd_page, unsigned long addr,
+                       void *data);
+extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
+                              unsigned long size, pte_fn_t fn, void *data);
+
 #ifdef CONFIG_PROC_FS
 void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
 #else