mm, slab/slub: introduce kmalloc-reclaimable caches

[sfrench/cifs-2.6.git] / include / linux / slab.h

diff --git a/include/linux/slab.h b/include/linux/slab.h
index ed9cbddeb4a6fa68ca54e20eed0492ee2439a34f..918f374e7156f4739b30c178e5c8beaf5137c4d0 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -295,12 +295,43 @@ static inline void __check_heap_object(const void *ptr, unsigned long n,
 #define SLAB_OBJ_MIN_SIZE      (KMALLOC_MIN_SIZE < 16 ? \
                                (KMALLOC_MIN_SIZE) : 16)
 
+/*
+ * Whenever changing this, take care of that kmalloc_type() and
+ * create_kmalloc_caches() still work as intended.
+ */
+enum kmalloc_cache_type {
+       KMALLOC_NORMAL = 0,
+       KMALLOC_RECLAIM,
+#ifdef CONFIG_ZONE_DMA
+       KMALLOC_DMA,
+#endif
+       NR_KMALLOC_TYPES
+};
+
 #ifndef CONFIG_SLOB
-extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
+extern struct kmem_cache *
+kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1];
+
+static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags)
+{
+       int is_dma = 0;
+       int type_dma = 0;
+       int is_reclaimable;
+
 #ifdef CONFIG_ZONE_DMA
-extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
+       is_dma = !!(flags & __GFP_DMA);
+       type_dma = is_dma * KMALLOC_DMA;
 #endif
 
+       is_reclaimable = !!(flags & __GFP_RECLAIMABLE);
+
+       /*
+        * If an allocation is both __GFP_DMA and __GFP_RECLAIMABLE, return
+        * KMALLOC_DMA and effectively ignore __GFP_RECLAIMABLE
+        */
+       return type_dma + (is_reclaimable & !is_dma) * KMALLOC_RECLAIM;
+}
+
 /*
  * Figure out which kmalloc slab an allocation of a certain size
  * belongs to.
@@ -501,18 +532,20 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
 static __always_inline void *kmalloc(size_t size, gfp_t flags)
 {
        if (__builtin_constant_p(size)) {
+#ifndef CONFIG_SLOB
+               unsigned int index;
+#endif
                if (size > KMALLOC_MAX_CACHE_SIZE)
                        return kmalloc_large(size, flags);
 #ifndef CONFIG_SLOB
-               if (!(flags & GFP_DMA)) {
-                       unsigned int index = kmalloc_index(size);
+               index = kmalloc_index(size);
 
-                       if (!index)
-                               return ZERO_SIZE_PTR;
+               if (!index)
+                       return ZERO_SIZE_PTR;
 
-                       return kmem_cache_alloc_trace(kmalloc_caches[index],
-                                       flags, size);
-               }
+               return kmem_cache_alloc_trace(
+                               kmalloc_caches[kmalloc_type(flags)][index],
+                               flags, size);
 #endif
        }
        return __kmalloc(size, flags);
@@ -542,13 +575,14 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
 #ifndef CONFIG_SLOB
        if (__builtin_constant_p(size) &&
-               size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {
+               size <= KMALLOC_MAX_CACHE_SIZE) {
                unsigned int i = kmalloc_index(size);
 
                if (!i)
                        return ZERO_SIZE_PTR;
 
-               return kmem_cache_alloc_node_trace(kmalloc_caches[i],
+               return kmem_cache_alloc_node_trace(
+                               kmalloc_caches[kmalloc_type(flags)][i],
                                                flags, node, size);
        }
 #endif