srcu: Make early-boot call_srcu() reuse workqueue lists

Allocating a list_head structure that is almost never used, and, when
used, is used only during early boot (rcu_init() and earlier), is a bit
wasteful.  This commit therefore eliminates that list_head in favor of
the one in the work_struct structure.  This is safe because the work_struct
structure cannot be used until after rcu_init() returns.

Reported-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Tested-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 2b5c0822e6839ac0d77105c49f4b4a8925bdb95d..f41d2fb09f87bcd42147a388c3b2aa5fa1d91e9b 100644 (file)
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -36,7 +36,6 @@ struct srcu_struct {
        struct rcu_head *srcu_cb_head;  /* Pending callbacks: Head. */
        struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */
        struct work_struct srcu_work;   /* For driving grace periods. */
-       struct list_head srcu_boot_entry; /* Early-boot callbacks. */
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map dep_map;
 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -49,7 +48,6 @@ void srcu_drive_gp(struct work_struct *wp);
        .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),        \
        .srcu_cb_tail = &name.srcu_cb_head,                             \
        .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \
-       .srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry),        \
        __SRCU_DEP_MAP_INIT(name)                                       \
 }
 

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 9cfa4610113a96146340b6a35a38e56329bb8dc8..0ae91b3a7406ec82973d2b6425ee3b7a447d9ed5 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -94,7 +94,6 @@ struct srcu_struct {
                                                /*  callback for the barrier */
                                                /*  operation. */
        struct delayed_work work;
-       struct list_head srcu_boot_entry;       /* Early-boot callbacks. */
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map dep_map;
 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -110,7 +109,7 @@ struct srcu_struct {
        .sda = &pcpu_name,                                              \
        .lock = __SPIN_LOCK_UNLOCKED(name.lock),                        \
        .srcu_gp_seq_needed = -1UL,                                     \
-       .srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry),        \
+       .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),         \
        __SRCU_DEP_MAP_INIT(name)                                       \
 }
 

diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index d233f0c63f6ff023710d25b0c2d349b2db0a4c9f..b46e6683f8c9ec871b26f716d7dd6a3d59c61566 100644 (file)
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -48,7 +48,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp)
        sp->srcu_gp_waiting = false;
        sp->srcu_idx = 0;
        INIT_WORK(&sp->srcu_work, srcu_drive_gp);
-       INIT_LIST_HEAD(&sp->srcu_boot_entry);
+       INIT_LIST_HEAD(&sp->srcu_work.entry);
        return 0;
 }
 
@@ -185,8 +185,8 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
        if (!READ_ONCE(sp->srcu_gp_running)) {
                if (likely(srcu_init_done))
                        schedule_work(&sp->srcu_work);
-               else if (list_empty(&sp->srcu_boot_entry))
-                       list_add(&sp->srcu_boot_entry, &srcu_boot_list);
+               else if (list_empty(&sp->srcu_work.entry))
+                       list_add(&sp->srcu_work.entry, &srcu_boot_list);
        }
 }
 EXPORT_SYMBOL_GPL(call_srcu);
@@ -224,8 +224,8 @@ void __init srcu_init(void)
        srcu_init_done = true;
        while (!list_empty(&srcu_boot_list)) {
                sp = list_first_entry(&srcu_boot_list,
-                                     struct srcu_struct, srcu_boot_entry);
-               list_del_init(&sp->srcu_boot_entry);
+                                     struct srcu_struct, srcu_work.entry);
+               list_del_init(&sp->srcu_work.entry);
                schedule_work(&sp->srcu_work);
        }
 }

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 2e7f6b460150f938fa25280f472cc4cd85840245..86c7fd0a1bfef0a944ce84fbdc729b1050a33139 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -186,7 +186,6 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static)
        mutex_init(&sp->srcu_barrier_mutex);
        atomic_set(&sp->srcu_barrier_cpu_cnt, 0);
        INIT_DELAYED_WORK(&sp->work, process_srcu);
-       INIT_LIST_HEAD(&sp->srcu_boot_entry);
        if (!is_static)
                sp->sda = alloc_percpu(struct srcu_data);
        init_srcu_struct_nodes(sp, is_static);
@@ -708,8 +707,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,
                if (likely(srcu_init_done))
                        queue_delayed_work(rcu_gp_wq, &sp->work,
                                           srcu_get_delay(sp));
-               else if (list_empty(&sp->srcu_boot_entry))
-                       list_add(&sp->srcu_boot_entry, &srcu_boot_list);
+               else if (list_empty(&sp->work.work.entry))
+                       list_add(&sp->work.work.entry, &srcu_boot_list);
        }
        spin_unlock_irqrestore_rcu_node(sp, flags);
 }
@@ -1323,10 +1322,10 @@ void __init srcu_init(void)
 
        srcu_init_done = true;
        while (!list_empty(&srcu_boot_list)) {
-               sp = list_first_entry(&srcu_boot_list,
-                                     struct srcu_struct, srcu_boot_entry);
+               sp = list_first_entry(&srcu_boot_list, struct srcu_struct,
+                                     work.work.entry);
                check_init_srcu_struct(sp);
-               list_del_init(&sp->srcu_boot_entry);
+               list_del_init(&sp->work.work.entry);
                queue_work(rcu_gp_wq, &sp->work.work);
        }
 }