Group short-lived and reclaimable kernel allocations

[sfrench/cifs-2.6.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index addb20a6d67d53e73e2dba2da6492ac0ee93aa38..a90c4ffc95765e3c16965473d3b51b262ea91f0b 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1055,6 +1055,9 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
        if (s->flags & SLAB_CACHE_DMA)
                flags |= SLUB_DMA;
 
+       if (s->flags & SLAB_RECLAIM_ACCOUNT)
+               flags |= __GFP_RECLAIMABLE;
+
        if (node == -1)
                page = alloc_pages(flags, s->order);
        else
@@ -1088,12 +1091,13 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        void *last;
        void *p;
 
-       BUG_ON(flags & ~(GFP_DMA | __GFP_ZERO | GFP_LEVEL_MASK));
+       BUG_ON(flags & GFP_SLAB_BUG_MASK);
 
        if (flags & __GFP_WAIT)
                local_irq_enable();
 
-       page = allocate_slab(s, flags & GFP_LEVEL_MASK, node);
+       page = allocate_slab(s,
+               flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
        if (!page)
                goto out;
 
@@ -1878,8 +1882,8 @@ static void init_kmem_cache_node(struct kmem_cache_node *n)
  * Note that this function only works on the kmalloc_node_cache
  * when allocating for the kmalloc_node_cache.
  */
-static struct kmem_cache_node * __init early_kmem_cache_node_alloc(gfp_t gfpflags,
-                                                               int node)
+static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
+                                                          int node)
 {
        struct page *page;
        struct kmem_cache_node *n;
@@ -1921,7 +1925,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 {
        int node;
 
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = s->node[node];
                if (n && n != &s->local_node)
                        kmem_cache_free(kmalloc_caches, n);
@@ -1939,7 +1943,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
        else
                local_node = 0;
 
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;
 
                if (local_node == node)
@@ -2192,7 +2196,7 @@ static inline int kmem_cache_close(struct kmem_cache *s)
        flush_all(s);
 
        /* Attempt to free all objects */
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = get_node(s, node);
 
                n->nr_partial -= free_list(s, n, &n->partial);
@@ -2227,11 +2231,11 @@ EXPORT_SYMBOL(kmem_cache_destroy);
  *             Kmalloc subsystem
  *******************************************************************/
 
-struct kmem_cache kmalloc_caches[KMALLOC_SHIFT_HIGH + 1] __cacheline_aligned;
+struct kmem_cache kmalloc_caches[PAGE_SHIFT] __cacheline_aligned;
 EXPORT_SYMBOL(kmalloc_caches);
 
 #ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_caches_dma[KMALLOC_SHIFT_HIGH + 1];
+static struct kmem_cache *kmalloc_caches_dma[PAGE_SHIFT];
 #endif
 
 static int __init setup_slub_min_order(char *str)
@@ -2397,12 +2401,8 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
                        return ZERO_SIZE_PTR;
 
                index = size_index[(size - 1) / 8];
-       } else {
-               if (size > KMALLOC_MAX_SIZE)
-                       return NULL;
-
+       } else
                index = fls(size - 1);
-       }
 
 #ifdef CONFIG_ZONE_DMA
        if (unlikely((flags & SLUB_DMA)))
@@ -2414,9 +2414,15 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
 
 void *__kmalloc(size_t size, gfp_t flags)
 {
-       struct kmem_cache *s = get_slab(size, flags);
+       struct kmem_cache *s;
+
+       if (unlikely(size > PAGE_SIZE / 2))
+               return (void *)__get_free_pages(flags | __GFP_COMP,
+                                                       get_order(size));
+
+       s = get_slab(size, flags);
 
-       if (ZERO_OR_NULL_PTR(s))
+       if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
        return slab_alloc(s, flags, -1, __builtin_return_address(0));
@@ -2426,9 +2432,15 @@ EXPORT_SYMBOL(__kmalloc);
 #ifdef CONFIG_NUMA
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
-       struct kmem_cache *s = get_slab(size, flags);
+       struct kmem_cache *s;
+
+       if (unlikely(size > PAGE_SIZE / 2))
+               return (void *)__get_free_pages(flags | __GFP_COMP,
+                                                       get_order(size));
+
+       s = get_slab(size, flags);
 
-       if (ZERO_OR_NULL_PTR(s))
+       if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
        return slab_alloc(s, flags, node, __builtin_return_address(0));
@@ -2441,7 +2453,8 @@ size_t ksize(const void *object)
        struct page *page;
        struct kmem_cache *s;
 
-       if (ZERO_OR_NULL_PTR(object))
+       BUG_ON(!object);
+       if (unlikely(object == ZERO_SIZE_PTR))
                return 0;
 
        page = get_object_page(object);
@@ -2473,22 +2486,17 @@ EXPORT_SYMBOL(ksize);
 
 void kfree(const void *x)
 {
-       struct kmem_cache *s;
        struct page *page;
 
-       /*
-        * This has to be an unsigned comparison. According to Linus
-        * some gcc version treat a pointer as a signed entity. Then
-        * this comparison would be true for all "negative" pointers
-        * (which would cover the whole upper half of the address space).
-        */
-       if (ZERO_OR_NULL_PTR(x))
+       if (unlikely(ZERO_OR_NULL_PTR(x)))
                return;
 
        page = virt_to_head_page(x);
-       s = page->slab;
-
-       slab_free(s, page, (void *)x, __builtin_return_address(0));
+       if (unlikely(!PageSlab(page))) {
+               put_page(page);
+               return;
+       }
+       slab_free(page->slab, page, (void *)x, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(kfree);
 
@@ -2517,7 +2525,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
                return -ENOMEM;
 
        flush_all(s);
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                n = get_node(s, node);
 
                if (!n->nr_partial)
@@ -2602,7 +2610,7 @@ void __init kmem_cache_init(void)
                caches++;
        }
 
-       for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
+       for (i = KMALLOC_SHIFT_LOW; i < PAGE_SHIFT; i++) {
                create_kmalloc_cache(&kmalloc_caches[i],
                        "kmalloc", 1 << i, GFP_KERNEL);
                caches++;
@@ -2629,7 +2637,7 @@ void __init kmem_cache_init(void)
        slab_state = UP;
 
        /* Provide the correct kmalloc names now that the caches are up */
-       for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
+       for (i = KMALLOC_SHIFT_LOW; i < PAGE_SHIFT; i++)
                kmalloc_caches[i]. name =
                        kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
 
@@ -2790,9 +2798,14 @@ static struct notifier_block __cpuinitdata slab_notifier =
 
 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller)
 {
-       struct kmem_cache *s = get_slab(size, gfpflags);
+       struct kmem_cache *s;
+
+       if (unlikely(size > PAGE_SIZE / 2))
+               return (void *)__get_free_pages(gfpflags | __GFP_COMP,
+                                                       get_order(size));
+       s = get_slab(size, gfpflags);
 
-       if (ZERO_OR_NULL_PTR(s))
+       if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
        return slab_alloc(s, gfpflags, -1, caller);
@@ -2801,9 +2814,14 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller)
 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
                                        int node, void *caller)
 {
-       struct kmem_cache *s = get_slab(size, gfpflags);
+       struct kmem_cache *s;
+
+       if (unlikely(size > PAGE_SIZE / 2))
+               return (void *)__get_free_pages(gfpflags | __GFP_COMP,
+                                                       get_order(size));
+       s = get_slab(size, gfpflags);
 
-       if (ZERO_OR_NULL_PTR(s))
+       if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
        return slab_alloc(s, gfpflags, node, caller);
@@ -2902,7 +2920,7 @@ static long validate_slab_cache(struct kmem_cache *s)
                return -ENOMEM;
 
        flush_all(s);
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = get_node(s, node);
 
                count += validate_slab_node(s, n, map);
@@ -3122,7 +3140,7 @@ static int list_locations(struct kmem_cache *s, char *buf,
        /* Push back cpu slabs */
        flush_all(s);
 
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = get_node(s, node);
                unsigned long flags;
                struct page *page;
@@ -3249,7 +3267,7 @@ static unsigned long slab_objects(struct kmem_cache *s,
                }
        }
 
-       for_each_online_node(node) {
+       for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = get_node(s, node);
 
                if (flags & SO_PARTIAL) {
@@ -3277,7 +3295,7 @@ static unsigned long slab_objects(struct kmem_cache *s,
 
        x = sprintf(buf, "%lu", total);
 #ifdef CONFIG_NUMA
-       for_each_online_node(node)
+       for_each_node_state(node, N_NORMAL_MEMORY)
                if (nodes[node])
                        x += sprintf(buf + x, " N%d=%lu",
                                        node, nodes[node]);