tile: implement gettimeofday() via vDSO

[sfrench/cifs-2.6.git] / mm / swap.c

diff --git a/mm/swap.c b/mm/swap.c
index 868b493431c22ff9630fd5448d56121cc45c167d..4a1d0d2c52fa4ab4ca66ff4a2025e093521556c9 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -432,6 +432,33 @@ void activate_page(struct page *page)
 }
 #endif
 
+static void __lru_cache_activate_page(struct page *page)
+{
+       struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
+       int i;
+
+       /*
+        * Search backwards on the optimistic assumption that the page being
+        * activated has just been added to this pagevec. Note that only
+        * the local pagevec is examined as a !PageLRU page could be in the
+        * process of being released, reclaimed, migrated or on a remote
+        * pagevec that is currently being drained. Furthermore, marking
+        * a remote pagevec's page PageActive potentially hits a race where
+        * a page is marked PageActive just after it is added to the inactive
+        * list causing accounting errors and BUG_ON checks to trigger.
+        */
+       for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
+               struct page *pagevec_page = pvec->pages[i];
+
+               if (pagevec_page == page) {
+                       SetPageActive(page);
+                       break;
+               }
+       }
+
+       put_cpu_var(lru_add_pvec);
+}
+
 /*
  * Mark a page as having seen activity.
  *
@@ -442,8 +469,18 @@ void activate_page(struct page *page)
 void mark_page_accessed(struct page *page)
 {
        if (!PageActive(page) && !PageUnevictable(page) &&
-                       PageReferenced(page) && PageLRU(page)) {
-               activate_page(page);
+                       PageReferenced(page)) {
+
+               /*
+                * If the page is on the LRU, queue it for activation via
+                * activate_page_pvecs. Otherwise, assume the page is on a
+                * pagevec, mark it active and it'll be moved to the active
+                * LRU on the next drain.
+                */
+               if (PageLRU(page))
+                       activate_page(page);
+               else
+                       __lru_cache_activate_page(page);
                ClearPageReferenced(page);
        } else if (!PageReferenced(page)) {
                SetPageReferenced(page);
@@ -457,29 +494,23 @@ EXPORT_SYMBOL(mark_page_accessed);
  * pagevec is drained. This gives a chance for the caller of __lru_cache_add()
  * have the page added to the active list using mark_page_accessed().
  */
-void __lru_cache_add(struct page *page, enum lru_list lru)
+void __lru_cache_add(struct page *page)
 {
        struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
 
-       if (is_active_lru(lru))
-               SetPageActive(page);
-       else
-               ClearPageActive(page);
-
        page_cache_get(page);
        if (!pagevec_space(pvec))
-               __pagevec_lru_add(pvec, lru);
+               __pagevec_lru_add(pvec);
        pagevec_add(pvec, page);
        put_cpu_var(lru_add_pvec);
 }
 EXPORT_SYMBOL(__lru_cache_add);
 
 /**
- * lru_cache_add_lru - add a page to a page list
+ * lru_cache_add - add a page to a page list
  * @page: the page to be added to the LRU.
- * @lru: the LRU list to which the page is added.
  */
-void lru_cache_add_lru(struct page *page, enum lru_list lru)
+void lru_cache_add(struct page *page)
 {
        if (PageActive(page)) {
                VM_BUG_ON(PageUnevictable(page));
@@ -488,7 +519,7 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)
        }
 
        VM_BUG_ON(PageLRU(page));
-       __lru_cache_add(page, lru);
+       __lru_cache_add(page);
 }
 
 /**
@@ -591,7 +622,7 @@ void lru_add_drain_cpu(int cpu)
        struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
 
        if (pagevec_count(pvec))
-               __pagevec_lru_add(pvec, NR_LRU_LISTS);
+               __pagevec_lru_add(pvec);
 
        pvec = &per_cpu(lru_rotate_pvecs, cpu);
        if (pagevec_count(pvec)) {
@@ -708,6 +739,9 @@ void release_pages(struct page **pages, int nr, int cold)
                        del_page_from_lru_list(page, lruvec, page_off_lru(page));
                }
 
+               /* Clear Active bit in case of parallel mark_page_accessed */
+               ClearPageActive(page);
+
                list_add(&page->lru, &pages_to_free);
        }
        if (zone)
@@ -795,12 +829,10 @@ void lru_add_page_tail(struct page *page, struct page *page_tail,
 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
                                 void *arg)
 {
-       enum lru_list requested_lru = (enum lru_list)arg;
        int file = page_is_file_cache(page);
        int active = PageActive(page);
        enum lru_list lru = page_lru(page);
 
-       WARN_ON_ONCE(requested_lru < NR_LRU_LISTS && requested_lru != lru);
        VM_BUG_ON(PageUnevictable(page));
        VM_BUG_ON(PageLRU(page));
 
@@ -814,11 +846,9 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
  * Add the passed pages to the LRU, then drop the caller's refcount
  * on them.  Reinitialises the caller's pagevec.
  */
-void __pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
+void __pagevec_lru_add(struct pagevec *pvec)
 {
-       VM_BUG_ON(is_unevictable_lru(lru));
-
-       pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, (void *)lru);
+       pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
 }
 EXPORT_SYMBOL(__pagevec_lru_add);