Group short-lived and reclaimable kernel allocations

[sfrench/cifs-2.6.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index bde271c001ba33ef1f61dd0f563f74d319cd1f0e..e34bcb87a6ee19c7fadaa6a0f9ccbb5e956e8b79 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -883,6 +883,7 @@ static void __slab_error(const char *function, struct kmem_cache *cachep,
   */
 
 static int use_alien_caches __read_mostly = 1;
+static int numa_platform __read_mostly = 1;
 static int __init noaliencache_setup(char *s)
 {
        use_alien_caches = 0;
@@ -1399,8 +1400,10 @@ void __init kmem_cache_init(void)
        int order;
        int node;
 
-       if (num_possible_nodes() == 1)
+       if (num_possible_nodes() == 1) {
                use_alien_caches = 0;
+               numa_platform = 0;
+       }
 
        for (i = 0; i < NUM_INIT_LISTS; i++) {
                kmem_list3_init(&initkmem_list3[i]);
@@ -1565,7 +1568,7 @@ void __init kmem_cache_init(void)
                /* Replace the static kmem_list3 structures for the boot cpu */
                init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], node);
 
-               for_each_online_node(nid) {
+               for_each_node_state(nid, N_NORMAL_MEMORY) {
                        init_list(malloc_sizes[INDEX_AC].cs_cachep,
                                  &initkmem_list3[SIZE_AC + nid], nid);
 
@@ -1640,6 +1643,8 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 #endif
 
        flags |= cachep->gfpflags;
+       if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
+               flags |= __GFP_RECLAIMABLE;
 
        page = alloc_pages_node(nodeid, flags, cachep->gfporder);
        if (!page)
@@ -1941,7 +1946,7 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index)
 {
        int node;
 
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                cachep->nodelists[node] = &initkmem_list3[index + node];
                cachep->nodelists[node]->next_reap = jiffies +
                    REAPTIMEOUT_LIST3 +
@@ -2072,7 +2077,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
                        g_cpucache_up = PARTIAL_L3;
                } else {
                        int node;
-                       for_each_online_node(node) {
+                       for_each_node_state(node, N_NORMAL_MEMORY) {
                                cachep->nodelists[node] =
                                    kmalloc_node(sizeof(struct kmem_list3),
                                                GFP_KERNEL, node);
@@ -2743,9 +2748,9 @@ static int cache_grow(struct kmem_cache *cachep,
         * Be lazy and only check for valid flags here,  keeping it out of the
         * critical path in kmem_cache_alloc().
         */
-       BUG_ON(flags & ~(GFP_DMA | __GFP_ZERO | GFP_LEVEL_MASK));
+       BUG_ON(flags & GFP_SLAB_BUG_MASK);
+       local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
 
-       local_flags = (flags & GFP_LEVEL_MASK);
        /* Take the l3 list lock to change the colour_next on this node */
        check_irq_off();
        l3 = cachep->nodelists[nodeid];
@@ -2776,13 +2781,13 @@ static int cache_grow(struct kmem_cache *cachep,
         * 'nodeid'.
         */
        if (!objp)
-               objp = kmem_getpages(cachep, flags, nodeid);
+               objp = kmem_getpages(cachep, local_flags, nodeid);
        if (!objp)
                goto failed;
 
        /* Get slab management. */
        slabp = alloc_slabmgmt(cachep, objp, offset,
-                       local_flags & ~GFP_THISNODE, nodeid);
+                       local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
        if (!slabp)
                goto opps1;
 
@@ -3222,7 +3227,7 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
 
        zonelist = &NODE_DATA(slab_node(current->mempolicy))
                        ->node_zonelists[gfp_zone(flags)];
-       local_flags = (flags & GFP_LEVEL_MASK);
+       local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
 
 retry:
        /*
@@ -3558,7 +3563,14 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
        check_irq_off();
        objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
 
-       if (cache_free_alien(cachep, objp))
+       /*
+        * Skip calling cache_free_alien() when the platform is not numa.
+        * This will avoid cache misses that happen while accessing slabp (which
+        * is per page memory  reference) to get nodeid. Instead use a global
+        * variable to skip the call, which is mostly likely to be present in
+        * the cache.
+        */
+       if (numa_platform && cache_free_alien(cachep, objp))
                return;
 
        if (likely(ac->avail < ac->limit)) {
@@ -3782,7 +3794,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep)
        struct array_cache *new_shared;
        struct array_cache **new_alien = NULL;
 
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
 
                 if (use_alien_caches) {
                         new_alien = alloc_alien_cache(node, cachep->limit);
@@ -4436,7 +4448,8 @@ const struct seq_operations slabstats_op = {
  */
 size_t ksize(const void *objp)
 {
-       if (unlikely(ZERO_OR_NULL_PTR(objp)))
+       BUG_ON(!objp);
+       if (unlikely(objp == ZERO_SIZE_PTR))
                return 0;
 
        return obj_size(virt_to_cache(objp));