mm: memcontrol: flush percpu vmstats before releasing memcg
[sfrench/cifs-2.6.git] / mm / memcontrol.c
index 6f5c0c517c497dbddad9016ef2d1993e4e1bcf05..1a32e32e7ac3ae3324808d2274951c42569b960b 100644 (file)
@@ -3260,6 +3260,41 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
        }
 }
 
+static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
+{
+       unsigned long stat[MEMCG_NR_STAT];
+       struct mem_cgroup *mi;
+       int node, cpu, i;
+
+       for (i = 0; i < MEMCG_NR_STAT; i++)
+               stat[i] = 0;
+
+       for_each_online_cpu(cpu)
+               for (i = 0; i < MEMCG_NR_STAT; i++)
+                       stat[i] += raw_cpu_read(memcg->vmstats_percpu->stat[i]);
+
+       for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+               for (i = 0; i < MEMCG_NR_STAT; i++)
+                       atomic_long_add(stat[i], &mi->vmstats[i]);
+
+       for_each_node(node) {
+               struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
+               struct mem_cgroup_per_node *pi;
+
+               for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                       stat[i] = 0;
+
+               for_each_online_cpu(cpu)
+                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                               stat[i] += raw_cpu_read(
+                                       pn->lruvec_stat_cpu->count[i]);
+
+               for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
+                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                               atomic_long_add(stat[i], &pi->lruvec_stat[i]);
+       }
+}
+
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_online_kmem(struct mem_cgroup *memcg)
 {
@@ -4682,6 +4717,11 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
        int node;
 
+       /*
+        * Flush percpu vmstats to guarantee the value correctness
+        * on parent's and all ancestor levels.
+        */
+       memcg_flush_percpu_vmstats(memcg);
        for_each_node(node)
                free_mem_cgroup_per_node_info(memcg, node);
        free_percpu(memcg->vmstats_percpu);