mm: slub: be more careful about the double cmpxchg of freelist

[linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 17dc00e33115b73736bfb687a271e7539464b20a..97580b41a24b1f184df311050ebc8ad6c92ec964 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2997,11 +2997,13 @@ static __always_inline void do_slab_free(struct kmem_cache *s,
        barrier();
 
        if (likely(page == c->page)) {
-               set_freepointer(s, tail_obj, c->freelist);
+               void **freelist = READ_ONCE(c->freelist);
+
+               set_freepointer(s, tail_obj, freelist);
 
                if (unlikely(!this_cpu_cmpxchg_double(
                                s->cpu_slab->freelist, s->cpu_slab->tid,
-                               c->freelist, tid,
+                               freelist, tid,
                                head, next_tid(tid)))) {
 
                        note_cmpxchg_failure("slab_free", s, tid);
@@ -3174,6 +3176,15 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                void *object = c->freelist;
 
                if (unlikely(!object)) {
+                       /*
+                        * We may have removed an object from c->freelist using
+                        * the fastpath in the previous iteration; in that case,
+                        * c->tid has not been bumped yet.
+                        * Since ___slab_alloc() may reenable interrupts while
+                        * allocating memory, we should bump c->tid now.
+                        */
+                       c->tid = next_tid(c->tid);
+
                        /*
                         * Invoking slow path likely have side-effect
                         * of re-populating per CPU c->freelist