slab: implement slab_root_caches list

[linux.git] / mm / slab_common.c

diff --git a/mm/slab_common.c b/mm/slab_common.c
index c3bbeddaeaaf8ca72a8106ed1be5d3ade39666bb..274697e1a42a54b228a9f3de7c7bce7084277b3a 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -138,6 +138,9 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
 }
 
 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
+
+LIST_HEAD(slab_root_caches);
+
 void slab_init_memcg_params(struct kmem_cache *s)
 {
        s->memcg_params.root_cache = NULL;
@@ -183,9 +186,6 @@ static int update_memcg_params(struct kmem_cache *s, int new_array_size)
 {
        struct memcg_cache_array *old, *new;
 
-       if (!is_root_cache(s))
-               return 0;
-
        new = kzalloc(sizeof(struct memcg_cache_array) +
                      new_array_size * sizeof(void *), GFP_KERNEL);
        if (!new)
@@ -209,7 +209,7 @@ int memcg_update_all_caches(int num_memcgs)
        int ret = 0;
 
        mutex_lock(&slab_mutex);
-       list_for_each_entry(s, &slab_caches, list) {
+       list_for_each_entry(s, &slab_root_caches, root_caches_node) {
                ret = update_memcg_params(s, num_memcgs);
                /*
                 * Instead of freeing the memory, we'll just leave the caches
@@ -222,10 +222,26 @@ int memcg_update_all_caches(int num_memcgs)
        return ret;
 }
 
-static void unlink_memcg_cache(struct kmem_cache *s)
+void memcg_link_cache(struct kmem_cache *s)
+{
+       if (is_root_cache(s)) {
+               list_add(&s->root_caches_node, &slab_root_caches);
+       } else {
+               list_add(&s->memcg_params.children_node,
+                        &s->memcg_params.root_cache->memcg_params.children);
+               list_add(&s->memcg_params.kmem_caches_node,
+                        &s->memcg_params.memcg->kmem_caches);
+       }
+}
+
+static void memcg_unlink_cache(struct kmem_cache *s)
 {
-       list_del(&s->memcg_params.children_node);
-       list_del(&s->memcg_params.kmem_caches_node);
+       if (is_root_cache(s)) {
+               list_del(&s->root_caches_node);
+       } else {
+               list_del(&s->memcg_params.children_node);
+               list_del(&s->memcg_params.kmem_caches_node);
+       }
 }
 #else
 static inline int init_memcg_params(struct kmem_cache *s,
@@ -238,7 +254,7 @@ static inline void destroy_memcg_params(struct kmem_cache *s)
 {
 }
 
-static inline void unlink_memcg_cache(struct kmem_cache *s)
+static inline void memcg_unlink_cache(struct kmem_cache *s)
 {
 }
 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
@@ -285,7 +301,7 @@ struct kmem_cache *find_mergeable(size_t size, size_t align,
        if (flags & SLAB_NEVER_MERGE)
                return NULL;
 
-       list_for_each_entry_reverse(s, &slab_caches, list) {
+       list_for_each_entry_reverse(s, &slab_root_caches, root_caches_node) {
                if (slab_unmergeable(s))
                        continue;
 
@@ -369,6 +385,7 @@ static struct kmem_cache *create_cache(const char *name,
 
        s->refcount = 1;
        list_add(&s->list, &slab_caches);
+       memcg_link_cache(s);
 out:
        if (err)
                return ERR_PTR(err);
@@ -514,9 +531,8 @@ static int shutdown_cache(struct kmem_cache *s)
        if (__kmem_cache_shutdown(s) != 0)
                return -EBUSY;
 
+       memcg_unlink_cache(s);
        list_del(&s->list);
-       if (!is_root_cache(s))
-               unlink_memcg_cache(s);
 
        if (s->flags & SLAB_DESTROY_BY_RCU) {
                list_add_tail(&s->list, &slab_caches_to_rcu_destroy);
@@ -596,10 +612,6 @@ void memcg_create_kmem_cache(struct mem_cgroup *memcg,
                goto out_unlock;
        }
 
-       list_add(&s->memcg_params.children_node,
-                &root_cache->memcg_params.children);
-       list_add(&s->memcg_params.kmem_caches_node, &memcg->kmem_caches);
-
        /*
         * Since readers won't lock (see cache_from_memcg_idx()), we need a
         * barrier here to ensure nobody will see the kmem_cache partially
@@ -627,10 +639,7 @@ void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
        get_online_mems();
 
        mutex_lock(&slab_mutex);
-       list_for_each_entry(s, &slab_caches, list) {
-               if (!is_root_cache(s))
-                       continue;
-
+       list_for_each_entry(s, &slab_root_caches, root_caches_node) {
                arr = rcu_dereference_protected(s->memcg_params.memcg_caches,
                                                lockdep_is_held(&slab_mutex));
                c = arr->entries[idx];
@@ -829,6 +838,7 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
 
        create_boot_cache(s, name, size, flags);
        list_add(&s->list, &slab_caches);
+       memcg_link_cache(s);
        s->refcount = 1;
        return s;
 }
@@ -1136,12 +1146,12 @@ static void print_slabinfo_header(struct seq_file *m)
 void *slab_start(struct seq_file *m, loff_t *pos)
 {
        mutex_lock(&slab_mutex);
-       return seq_list_start(&slab_caches, *pos);
+       return seq_list_start(&slab_root_caches, *pos);
 }
 
 void *slab_next(struct seq_file *m, void *p, loff_t *pos)
 {
-       return seq_list_next(p, &slab_caches, pos);
+       return seq_list_next(p, &slab_root_caches, pos);
 }
 
 void slab_stop(struct seq_file *m, void *p)
@@ -1193,12 +1203,11 @@ static void cache_show(struct kmem_cache *s, struct seq_file *m)
 
 static int slab_show(struct seq_file *m, void *p)
 {
-       struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
+       struct kmem_cache *s = list_entry(p, struct kmem_cache, root_caches_node);
 
-       if (p == slab_caches.next)
+       if (p == slab_root_caches.next)
                print_slabinfo_header(m);
-       if (is_root_cache(s))
-               cache_show(s, m);
+       cache_show(s, m);
        return 0;
 }