mm: optimize compound_head() by avoiding a shared page flag

[sfrench/cifs-2.6.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 019ceda6a8b620d51099d8a4b54ef75bcffe22ea..36d713e216e8d4dd22d42b3c1f72b570ff0965ea 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -156,10 +156,8 @@ static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 
 static int page_is_consistent(struct zone *zone, struct page *page)
 {
-#ifdef CONFIG_HOLES_IN_ZONE
-       if (!pfn_valid(page_to_pfn(page)))
+       if (!pfn_valid_within(page_to_pfn(page)))
                return 0;
-#endif
        if (zone != page_zone(page))
                return 0;
 
@@ -227,7 +225,7 @@ static void bad_page(struct page *page)
 
 static void free_compound_page(struct page *page)
 {
-       __free_pages_ok(page, (unsigned long)page[1].lru.prev);
+       __free_pages_ok(page, compound_order(page));
 }
 
 static void prep_compound_page(struct page *page, unsigned long order)
@@ -236,12 +234,13 @@ static void prep_compound_page(struct page *page, unsigned long order)
        int nr_pages = 1 << order;
 
        set_compound_page_dtor(page, free_compound_page);
-       page[1].lru.prev = (void *)order;
-       for (i = 0; i < nr_pages; i++) {
+       set_compound_order(page, order);
+       __SetPageHead(page);
+       for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               __SetPageCompound(p);
-               set_page_private(p, (unsigned long)page);
+               __SetPageTail(p);
+               p->first_page = page;
        }
 }
 
@@ -250,16 +249,19 @@ static void destroy_compound_page(struct page *page, unsigned long order)
        int i;
        int nr_pages = 1 << order;
 
-       if (unlikely((unsigned long)page[1].lru.prev != order))
+       if (unlikely(compound_order(page) != order))
                bad_page(page);
 
-       for (i = 0; i < nr_pages; i++) {
+       if (unlikely(!PageHead(page)))
+                       bad_page(page);
+       __ClearPageHead(page);
+       for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               if (unlikely(!PageCompound(p) |
-                               (page_private(p) != (unsigned long)page)))
+               if (unlikely(!PageTail(p) |
+                               (p->first_page != page)))
                        bad_page(page);
-               __ClearPageCompound(p);
+               __ClearPageTail(p);
        }
 }
 
@@ -346,10 +348,8 @@ __find_combined_index(unsigned long page_idx, unsigned int order)
 static inline int page_is_buddy(struct page *page, struct page *buddy,
                                                                int order)
 {
-#ifdef CONFIG_HOLES_IN_ZONE
-       if (!pfn_valid(page_to_pfn(buddy)))
+       if (!pfn_valid_within(page_to_pfn(buddy)))
                return 0;
-#endif
 
        if (page_zone_id(page) != page_zone_id(buddy))
                return 0;
@@ -433,13 +433,18 @@ static inline int free_pages_check(struct page *page)
                        1 << PG_private |
                        1 << PG_locked  |
                        1 << PG_active  |
-                       1 << PG_reclaim |
                        1 << PG_slab    |
                        1 << PG_swapcache |
                        1 << PG_writeback |
                        1 << PG_reserved |
                        1 << PG_buddy ))))
                bad_page(page);
+       /*
+        * PageReclaim == PageTail. It is only an error
+        * for PageReclaim to be set if PageCompound is clear.
+        */
+       if (unlikely(!PageCompound(page) && PageReclaim(page)))
+               bad_page(page);
        if (PageDirty(page))
                __ClearPageDirty(page);
        /*
@@ -3203,7 +3208,8 @@ int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
 {
        proc_dointvec(table, write, file, buffer, length, ppos);
-       setup_per_zone_pages_min();
+       if (write)
+               setup_per_zone_pages_min();
        return 0;
 }