mm: memcontrol: flush percpu vmstats before releasing memcg

[sfrench/cifs-2.6.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index cdbb7a84cb6e186242afeebb32f4c5c7fb2d0639..1a32e32e7ac3ae3324808d2274951c42569b960b 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -768,6 +768,26 @@ void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
        __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
 }
 
+void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
+{
+       struct page *page = virt_to_head_page(p);
+       pg_data_t *pgdat = page_pgdat(page);
+       struct mem_cgroup *memcg;
+       struct lruvec *lruvec;
+
+       rcu_read_lock();
+       memcg = memcg_from_slab_page(page);
+
+       /* Untracked pages have no memcg, no lruvec. Update only the node */
+       if (!memcg || memcg == root_mem_cgroup) {
+               __mod_node_page_state(pgdat, idx, val);
+       } else {
+               lruvec = mem_cgroup_lruvec(pgdat, memcg);
+               __mod_lruvec_state(lruvec, idx, val);
+       }
+       rcu_read_unlock();
+}
+
 /**
  * __count_memcg_events - account VM events in a cgroup
  * @memcg: the memory cgroup
@@ -1130,26 +1150,45 @@ void mem_cgroup_iter_break(struct mem_cgroup *root,
                css_put(&prev->css);
 }
 
-static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
+static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
+                                       struct mem_cgroup *dead_memcg)
 {
-       struct mem_cgroup *memcg = dead_memcg;
        struct mem_cgroup_reclaim_iter *iter;
        struct mem_cgroup_per_node *mz;
        int nid;
        int i;
 
-       for (; memcg; memcg = parent_mem_cgroup(memcg)) {
-               for_each_node(nid) {
-                       mz = mem_cgroup_nodeinfo(memcg, nid);
-                       for (i = 0; i <= DEF_PRIORITY; i++) {
-                               iter = &mz->iter[i];
-                               cmpxchg(&iter->position,
-                                       dead_memcg, NULL);
-                       }
+       for_each_node(nid) {
+               mz = mem_cgroup_nodeinfo(from, nid);
+               for (i = 0; i <= DEF_PRIORITY; i++) {
+                       iter = &mz->iter[i];
+                       cmpxchg(&iter->position,
+                               dead_memcg, NULL);
                }
        }
 }
 
+static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
+{
+       struct mem_cgroup *memcg = dead_memcg;
+       struct mem_cgroup *last;
+
+       do {
+               __invalidate_reclaim_iterators(memcg, dead_memcg);
+               last = memcg;
+       } while ((memcg = parent_mem_cgroup(memcg)));
+
+       /*
+        * When cgruop1 non-hierarchy mode is used,
+        * parent_mem_cgroup() does not walk all the way up to the
+        * cgroup root (root_mem_cgroup). So we have to handle
+        * dead_memcg from cgroup root separately.
+        */
+       if (last != root_mem_cgroup)
+               __invalidate_reclaim_iterators(root_mem_cgroup,
+                                               dead_memcg);
+}
+
 /**
  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
  * @memcg: hierarchy root
@@ -3221,6 +3260,41 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
        }
 }
 
+static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
+{
+       unsigned long stat[MEMCG_NR_STAT];
+       struct mem_cgroup *mi;
+       int node, cpu, i;
+
+       for (i = 0; i < MEMCG_NR_STAT; i++)
+               stat[i] = 0;
+
+       for_each_online_cpu(cpu)
+               for (i = 0; i < MEMCG_NR_STAT; i++)
+                       stat[i] += raw_cpu_read(memcg->vmstats_percpu->stat[i]);
+
+       for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+               for (i = 0; i < MEMCG_NR_STAT; i++)
+                       atomic_long_add(stat[i], &mi->vmstats[i]);
+
+       for_each_node(node) {
+               struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
+               struct mem_cgroup_per_node *pi;
+
+               for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                       stat[i] = 0;
+
+               for_each_online_cpu(cpu)
+                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                               stat[i] += raw_cpu_read(
+                                       pn->lruvec_stat_cpu->count[i]);
+
+               for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
+                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                               atomic_long_add(stat[i], &pi->lruvec_stat[i]);
+       }
+}
+
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_online_kmem(struct mem_cgroup *memcg)
 {
@@ -4643,6 +4717,11 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
        int node;
 
+       /*
+        * Flush percpu vmstats to guarantee the value correctness
+        * on parent's and all ancestor levels.
+        */
+       memcg_flush_percpu_vmstats(memcg);
        for_each_node(node)
                free_mem_cgroup_per_node_info(memcg, node);
        free_percpu(memcg->vmstats_percpu);