mm: memcontrol: move stat/event counting functions out-of-line

[sfrench/cifs-2.6.git] / include / linux / memcontrol.h

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index dbb6118370c1e3c6df88b7da5695aea57dcd08ad..e35e6a651187b87ef35fe8faaa2ae2493b656f27 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -94,8 +94,8 @@ enum mem_cgroup_events_target {
        MEM_CGROUP_NTARGETS,
 };
 
-struct mem_cgroup_stat_cpu {
-       long count[MEMCG_NR_STAT];
+struct memcg_vmstats_percpu {
+       long stat[MEMCG_NR_STAT];
        unsigned long events[NR_VM_EVENT_ITEMS];
        unsigned long nr_page_events;
        unsigned long targets[MEM_CGROUP_NTARGETS];
@@ -274,12 +274,12 @@ struct mem_cgroup {
        struct task_struct      *move_lock_task;
 
        /* memory.stat */
-       struct mem_cgroup_stat_cpu __percpu *stat_cpu;
+       struct memcg_vmstats_percpu __percpu *vmstats_percpu;
 
        MEMCG_PADDING(_pad2_);
 
-       atomic_long_t           stat[MEMCG_NR_STAT];
-       atomic_long_t           events[NR_VM_EVENT_ITEMS];
+       atomic_long_t           vmstats[MEMCG_NR_STAT];
+       atomic_long_t           vmevents[NR_VM_EVENT_ITEMS];
        atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS];
 
        unsigned long           socket_pressure;
@@ -501,22 +501,6 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
                int zid, int nr_pages);
 
-unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
-                                          int nid, unsigned int lru_mask);
-
-static inline
-unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
-{
-       struct mem_cgroup_per_node *mz;
-       unsigned long nr_pages = 0;
-       int zid;
-
-       mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
-       for (zid = 0; zid < MAX_NR_ZONES; zid++)
-               nr_pages += mz->lru_zone_size[zid][lru];
-       return nr_pages;
-}
-
 static inline
 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
                enum lru_list lru, int zone_idx)
@@ -570,10 +554,10 @@ void unlock_page_memcg(struct page *page);
  * idx can be of type enum memcg_stat_item or node_stat_item.
  * Keep in sync with memcg_exact_page_state().
  */
-static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
-                                            int idx)
+static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
+                                                  int idx)
 {
-       long x = atomic_long_read(&memcg->stat[idx]);
+       long x = atomic_long_read(&memcg->vmstats[idx]);
 #ifdef CONFIG_SMP
        if (x < 0)
                x = 0;
@@ -581,22 +565,7 @@ static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
        return x;
 }
 
-/* idx can be of type enum memcg_stat_item or node_stat_item */
-static inline void __mod_memcg_state(struct mem_cgroup *memcg,
-                                    int idx, int val)
-{
-       long x;
-
-       if (mem_cgroup_disabled())
-               return;
-
-       x = val + __this_cpu_read(memcg->stat_cpu->count[idx]);
-       if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
-               atomic_long_add(x, &memcg->stat[idx]);
-               x = 0;
-       }
-       __this_cpu_write(memcg->stat_cpu->count[idx], x);
-}
+void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
 
 /* idx can be of type enum memcg_stat_item or node_stat_item */
 static inline void mod_memcg_state(struct mem_cgroup *memcg,
@@ -640,8 +609,8 @@ static inline void mod_memcg_page_state(struct page *page,
                mod_memcg_state(page->mem_cgroup, idx, val);
 }
 
-static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
-                                             enum node_stat_item idx)
+static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
+                                                   enum node_stat_item idx)
 {
        struct mem_cgroup_per_node *pn;
        long x;
@@ -658,31 +627,8 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
        return x;
 }
 
-static inline void __mod_lruvec_state(struct lruvec *lruvec,
-                                     enum node_stat_item idx, int val)
-{
-       struct mem_cgroup_per_node *pn;
-       long x;
-
-       /* Update node */
-       __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
-
-       if (mem_cgroup_disabled())
-               return;
-
-       pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
-
-       /* Update memcg */
-       __mod_memcg_state(pn->memcg, idx, val);
-
-       /* Update lruvec */
-       x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
-       if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
-               atomic_long_add(x, &pn->lruvec_stat[idx]);
-               x = 0;
-       }
-       __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
-}
+void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
+                       int val);
 
 static inline void mod_lruvec_state(struct lruvec *lruvec,
                                    enum node_stat_item idx, int val)
@@ -724,22 +670,8 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
                                                gfp_t gfp_mask,
                                                unsigned long *total_scanned);
 
-static inline void __count_memcg_events(struct mem_cgroup *memcg,
-                                       enum vm_event_item idx,
-                                       unsigned long count)
-{
-       unsigned long x;
-
-       if (mem_cgroup_disabled())
-               return;
-
-       x = count + __this_cpu_read(memcg->stat_cpu->events[idx]);
-       if (unlikely(x > MEMCG_CHARGE_BATCH)) {
-               atomic_long_add(x, &memcg->events[idx]);
-               x = 0;
-       }
-       __this_cpu_write(memcg->stat_cpu->events[idx], x);
-}
+void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
+                         unsigned long count);
 
 static inline void count_memcg_events(struct mem_cgroup *memcg,
                                      enum vm_event_item idx,
@@ -960,11 +892,6 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
        return true;
 }
 
-static inline unsigned long
-mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
-{
-       return 0;
-}
 static inline
 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
                enum lru_list lru, int zone_idx)
@@ -972,13 +899,6 @@ unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
        return 0;
 }
 
-static inline unsigned long
-mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
-                            int nid, unsigned int lru_mask)
-{
-       return 0;
-}
-
 static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
 {
        return 0;
@@ -1039,8 +959,8 @@ static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
 {
 }
 
-static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
-                                            int idx)
+static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
+                                                  int idx)
 {
        return 0;
 }
@@ -1069,8 +989,8 @@ static inline void mod_memcg_page_state(struct page *page,
 {
 }
 
-static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
-                                             enum node_stat_item idx)
+static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
+                                                   enum node_stat_item idx)
 {
        return node_page_state(lruvec_pgdat(lruvec), idx);
 }
@@ -1117,6 +1037,12 @@ static inline void count_memcg_events(struct mem_cgroup *memcg,
 {
 }
 
+static inline void __count_memcg_events(struct mem_cgroup *memcg,
+                                       enum vm_event_item idx,
+                                       unsigned long count)
+{
+}
+
 static inline void count_memcg_page_event(struct page *page,
                                          int idx)
 {