Merge branch 'irq-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[linux.git] / mm / slab_common.c

diff --git a/mm/slab_common.c b/mm/slab_common.c
index fea3376f98167b76794b83c20b341fca17788353..7eb8dc136c1cb879b74b4102e201015a85867ee6 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -973,14 +973,10 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name,
        return s;
 }
 
-struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
+struct kmem_cache *
+kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
 EXPORT_SYMBOL(kmalloc_caches);
 
-#ifdef CONFIG_ZONE_DMA
-struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
-EXPORT_SYMBOL(kmalloc_dma_caches);
-#endif
-
 /*
  * Conversion table for small slabs sizes / 8 to the index in the
  * kmalloc array. This is necessary for slabs < 192 since we have non power
@@ -1027,25 +1023,20 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
 {
        unsigned int index;
 
-       if (unlikely(size > KMALLOC_MAX_SIZE)) {
-               WARN_ON_ONCE(!(flags & __GFP_NOWARN));
-               return NULL;
-       }
-
        if (size <= 192) {
                if (!size)
                        return ZERO_SIZE_PTR;
 
                index = size_index[size_index_elem(size)];
-       } else
+       } else {
+               if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
+                       WARN_ON(1);
+                       return NULL;
+               }
                index = fls(size - 1);
+       }
 
-#ifdef CONFIG_ZONE_DMA
-       if (unlikely((flags & GFP_DMA)))
-               return kmalloc_dma_caches[index];
-
-#endif
-       return kmalloc_caches[index];
+       return kmalloc_caches[kmalloc_type(flags)][index];
 }
 
 /*
@@ -1059,15 +1050,15 @@ const struct kmalloc_info_struct kmalloc_info[] __initconst = {
        {"kmalloc-16",             16},         {"kmalloc-32",             32},
        {"kmalloc-64",             64},         {"kmalloc-128",           128},
        {"kmalloc-256",           256},         {"kmalloc-512",           512},
-       {"kmalloc-1024",         1024},         {"kmalloc-2048",         2048},
-       {"kmalloc-4096",         4096},         {"kmalloc-8192",         8192},
-       {"kmalloc-16384",       16384},         {"kmalloc-32768",       32768},
-       {"kmalloc-65536",       65536},         {"kmalloc-131072",     131072},
-       {"kmalloc-262144",     262144},         {"kmalloc-524288",     524288},
-       {"kmalloc-1048576",   1048576},         {"kmalloc-2097152",   2097152},
-       {"kmalloc-4194304",   4194304},         {"kmalloc-8388608",   8388608},
-       {"kmalloc-16777216", 16777216},         {"kmalloc-33554432", 33554432},
-       {"kmalloc-67108864", 67108864}
+       {"kmalloc-1k",           1024},         {"kmalloc-2k",           2048},
+       {"kmalloc-4k",           4096},         {"kmalloc-8k",           8192},
+       {"kmalloc-16k",         16384},         {"kmalloc-32k",         32768},
+       {"kmalloc-64k",         65536},         {"kmalloc-128k",       131072},
+       {"kmalloc-256k",       262144},         {"kmalloc-512k",       524288},
+       {"kmalloc-1M",        1048576},         {"kmalloc-2M",        2097152},
+       {"kmalloc-4M",        4194304},         {"kmalloc-8M",        8388608},
+       {"kmalloc-16M",      16777216},         {"kmalloc-32M",      33554432},
+       {"kmalloc-64M",      67108864}
 };
 
 /*
@@ -1117,9 +1108,36 @@ void __init setup_kmalloc_cache_index_table(void)
        }
 }
 
-static void __init new_kmalloc_cache(int idx, slab_flags_t flags)
+static const char *
+kmalloc_cache_name(const char *prefix, unsigned int size)
+{
+
+       static const char units[3] = "\0kM";
+       int idx = 0;
+
+       while (size >= 1024 && (size % 1024 == 0)) {
+               size /= 1024;
+               idx++;
+       }
+
+       return kasprintf(GFP_NOWAIT, "%s-%u%c", prefix, size, units[idx]);
+}
+
+static void __init
+new_kmalloc_cache(int idx, int type, slab_flags_t flags)
 {
-       kmalloc_caches[idx] = create_kmalloc_cache(kmalloc_info[idx].name,
+       const char *name;
+
+       if (type == KMALLOC_RECLAIM) {
+               flags |= SLAB_RECLAIM_ACCOUNT;
+               name = kmalloc_cache_name("kmalloc-rcl",
+                                               kmalloc_info[idx].size);
+               BUG_ON(!name);
+       } else {
+               name = kmalloc_info[idx].name;
+       }
+
+       kmalloc_caches[type][idx] = create_kmalloc_cache(name,
                                        kmalloc_info[idx].size, flags, 0,
                                        kmalloc_info[idx].size);
 }
@@ -1131,21 +1149,25 @@ static void __init new_kmalloc_cache(int idx, slab_flags_t flags)
  */
 void __init create_kmalloc_caches(slab_flags_t flags)
 {
-       int i;
+       int i, type;
 
-       for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
-               if (!kmalloc_caches[i])
-                       new_kmalloc_cache(i, flags);
+       for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
+               for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
+                       if (!kmalloc_caches[type][i])
+                               new_kmalloc_cache(i, type, flags);
 
-               /*
-                * Caches that are not of the two-to-the-power-of size.
-                * These have to be created immediately after the
-                * earlier power of two caches
-                */
-               if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
-                       new_kmalloc_cache(1, flags);
-               if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
-                       new_kmalloc_cache(2, flags);
+                       /*
+                        * Caches that are not of the two-to-the-power-of size.
+                        * These have to be created immediately after the
+                        * earlier power of two caches
+                        */
+                       if (KMALLOC_MIN_SIZE <= 32 && i == 6 &&
+                                       !kmalloc_caches[type][1])
+                               new_kmalloc_cache(1, type, flags);
+                       if (KMALLOC_MIN_SIZE <= 64 && i == 7 &&
+                                       !kmalloc_caches[type][2])
+                               new_kmalloc_cache(2, type, flags);
+               }
        }
 
        /* Kmalloc array is now usable */
@@ -1153,16 +1175,15 @@ void __init create_kmalloc_caches(slab_flags_t flags)
 
 #ifdef CONFIG_ZONE_DMA
        for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
-               struct kmem_cache *s = kmalloc_caches[i];
+               struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
 
                if (s) {
                        unsigned int size = kmalloc_size(i);
-                       char *n = kasprintf(GFP_NOWAIT,
-                                "dma-kmalloc-%u", size);
+                       const char *n = kmalloc_cache_name("dma-kmalloc", size);
 
                        BUG_ON(!n);
-                       kmalloc_dma_caches[i] = create_kmalloc_cache(n,
-                               size, SLAB_CACHE_DMA | flags, 0, 0);
+                       kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
+                               n, size, SLAB_CACHE_DMA | flags, 0, 0);
                }
        }
 #endif