rcu: Move rcu_cpu_has_work to rcu_data structure

Given that RCU has a perfectly good per-CPU rcu_data structure, most
per-CPU quantities should be stored there.

This commit therefore moves the rcu_cpu_has_work per-CPU variable to
the rcu_data structure.  This also makes this variable unconditionally
present, which should be acceptable given the memory reduction due to the
RCU flavor consolidation and also due to simplifications this will enable.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index e50b0a5a94bca7e927ae40869bbdbaf2a95960ff..7ae6774a920e26968e3b78b60de6c9db9b5155ca 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -238,7 +238,7 @@ struct rcu_data {
        struct task_struct *rcu_cpu_kthread_task;
                                        /* rcuc per-CPU kthread or NULL. */
        unsigned int rcu_cpu_kthread_status;
-                                       /* Running status for rcuc. */
+       char rcu_cpu_has_work;
 
        /* 7) Diagnostic data, including RCU CPU stall warnings. */
        unsigned int softirq_snap;      /* Snapshot of softirq activity. */
@@ -410,7 +410,6 @@ int rcu_dynticks_snap(struct rcu_data *rdp);
 
 #ifdef CONFIG_RCU_BOOST
 DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
-DECLARE_PER_CPU(char, rcu_cpu_has_work);
 #endif /* #ifdef CONFIG_RCU_BOOST */
 
 /* Forward declarations for rcutree_plugin.h */

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index d1f32c63c789fcae62ffc926cd0bdefed1fd2d89..b241c4b20549383522cec816ee36891fb6fc8341 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -34,14 +34,7 @@
 #include "../time/tick-internal.h"
 
 #ifdef CONFIG_RCU_BOOST
-
 #include "../locking/rtmutex_common.h"
-
-/*
- * Control variables for per-CPU and per-rcu_node kthreads.
- */
-DEFINE_PER_CPU(char, rcu_cpu_has_work);
-
 #else /* #ifdef CONFIG_RCU_BOOST */
 
 /*
@@ -1304,7 +1297,7 @@ static void invoke_rcu_callbacks_kthread(void)
        unsigned long flags;
 
        local_irq_save(flags);
-       __this_cpu_write(rcu_cpu_has_work, 1);
+       __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
        if (__this_cpu_read(rcu_data.rcu_cpu_kthread_task) != NULL &&
            current != __this_cpu_read(rcu_data.rcu_cpu_kthread_task)) {
                rcu_wake_cond(__this_cpu_read(rcu_data.rcu_cpu_kthread_task),
@@ -1386,7 +1379,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu)
 
 static int rcu_cpu_kthread_should_run(unsigned int cpu)
 {
-       return __this_cpu_read(rcu_cpu_has_work);
+       return __this_cpu_read(rcu_data.rcu_cpu_has_work);
 }
 
 /*
@@ -1397,7 +1390,7 @@ static int rcu_cpu_kthread_should_run(unsigned int cpu)
 static void rcu_cpu_kthread(unsigned int cpu)
 {
        unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
-       char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
+       char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
        int spincnt;
 
        for (spincnt = 0; spincnt < 10; spincnt++) {
@@ -1472,7 +1465,7 @@ static void __init rcu_spawn_boost_kthreads(void)
        int cpu;
 
        for_each_possible_cpu(cpu)
-               per_cpu(rcu_cpu_has_work, cpu) = 0;
+               per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
        if (WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec), "%s: Could not start rcub kthread, OOM is now expected behavior\n", __func__))
                return;
        rcu_for_each_leaf_node(rnp)