mm: optimize compound_head() by avoiding a shared page flag

[sfrench/cifs-2.6.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 542fc088ff508b15bf1416dcd68c80e8c3fb2fac..36d713e216e8d4dd22d42b3c1f72b570ff0965ea 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -225,7 +225,7 @@ static void bad_page(struct page *page)
 
 static void free_compound_page(struct page *page)
 {
-       __free_pages_ok(page, (unsigned long)page[1].lru.prev);
+       __free_pages_ok(page, compound_order(page));
 }
 
 static void prep_compound_page(struct page *page, unsigned long order)
@@ -234,12 +234,13 @@ static void prep_compound_page(struct page *page, unsigned long order)
        int nr_pages = 1 << order;
 
        set_compound_page_dtor(page, free_compound_page);
-       page[1].lru.prev = (void *)order;
-       for (i = 0; i < nr_pages; i++) {
+       set_compound_order(page, order);
+       __SetPageHead(page);
+       for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               __SetPageCompound(p);
-               set_page_private(p, (unsigned long)page);
+               __SetPageTail(p);
+               p->first_page = page;
        }
 }
 
@@ -248,16 +249,19 @@ static void destroy_compound_page(struct page *page, unsigned long order)
        int i;
        int nr_pages = 1 << order;
 
-       if (unlikely((unsigned long)page[1].lru.prev != order))
+       if (unlikely(compound_order(page) != order))
                bad_page(page);
 
-       for (i = 0; i < nr_pages; i++) {
+       if (unlikely(!PageHead(page)))
+                       bad_page(page);
+       __ClearPageHead(page);
+       for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               if (unlikely(!PageCompound(p) |
-                               (page_private(p) != (unsigned long)page)))
+               if (unlikely(!PageTail(p) |
+                               (p->first_page != page)))
                        bad_page(page);
-               __ClearPageCompound(p);
+               __ClearPageTail(p);
        }
 }
 
@@ -429,13 +433,18 @@ static inline int free_pages_check(struct page *page)
                        1 << PG_private |
                        1 << PG_locked  |
                        1 << PG_active  |
-                       1 << PG_reclaim |
                        1 << PG_slab    |
                        1 << PG_swapcache |
                        1 << PG_writeback |
                        1 << PG_reserved |
                        1 << PG_buddy ))))
                bad_page(page);
+       /*
+        * PageReclaim == PageTail. It is only an error
+        * for PageReclaim to be set if PageCompound is clear.
+        */
+       if (unlikely(!PageCompound(page) && PageReclaim(page)))
+               bad_page(page);
        if (PageDirty(page))
                __ClearPageDirty(page);
        /*