rcu_sync: Add CONFIG_PROVE_RCU checks

This commit validates that the caller of rcu_sync_is_idle() holds the
corresponding type of RCU read-side lock, but only in kernels built
with CONFIG_PROVE_RCU=y.  This validation is carried out via a new
rcu_sync_ops->held() method that is checked within rcu_sync_is_idle().

Note that although this does add code to the fast path, it only does so
in kernels built with CONFIG_PROVE_RCU=y.

Suggested-by: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

diff --git a/include/linux/rcu_sync.h b/include/linux/rcu_sync.h
index c6d2272c4459b8530e4aabef96e5b2a8aa17793e..1f2d4fc30b04a598c57d202443f59f7e09a8e8e9 100644 (file)
--- a/include/linux/rcu_sync.h
+++ b/include/linux/rcu_sync.h
@@ -40,6 +40,8 @@ struct rcu_sync {
        enum rcu_sync_type      gp_type;
 };
 
+extern bool __rcu_sync_is_idle(struct rcu_sync *);
+
 /**
  * rcu_sync_is_idle() - Are readers permitted to use their fastpaths?
  * @rsp: Pointer to rcu_sync structure to use for synchronization
@@ -50,7 +52,11 @@ struct rcu_sync {
  */
 static inline bool rcu_sync_is_idle(struct rcu_sync *rsp)
 {
+#ifdef CONFIG_PROVE_RCU
+       return __rcu_sync_is_idle(rsp);
+#else
        return !rsp->gp_state; /* GP_IDLE */
+#endif
 }
 
 extern void rcu_sync_init(struct rcu_sync *, enum rcu_sync_type);

diff --git a/kernel/rcu/sync.c b/kernel/rcu/sync.c
index 5a9aa4c394f13ac7c74949b5eb482d9e1d3b63fa..01c9807a7f730c5a5981e0d1742059857a4b6ab9 100644 (file)
--- a/kernel/rcu/sync.c
+++ b/kernel/rcu/sync.c
@@ -23,21 +23,33 @@
 #include <linux/rcu_sync.h>
 #include <linux/sched.h>
 
+#ifdef CONFIG_PROVE_RCU
+#define __INIT_HELD(func)      .held = func,
+#else
+#define __INIT_HELD(func)
+#endif
+
 static const struct {
        void (*sync)(void);
        void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+#ifdef CONFIG_PROVE_RCU
+       int  (*held)(void);
+#endif
 } gp_ops[] = {
        [RCU_SYNC] = {
                .sync = synchronize_rcu,
                .call = call_rcu,
+               __INIT_HELD(rcu_read_lock_held)
        },
        [RCU_SCHED_SYNC] = {
                .sync = synchronize_sched,
                .call = call_rcu_sched,
+               __INIT_HELD(rcu_read_lock_sched_held)
        },
        [RCU_BH_SYNC] = {
                .sync = synchronize_rcu_bh,
                .call = call_rcu_bh,
+               __INIT_HELD(rcu_read_lock_bh_held)
        },
 };
 
@@ -46,6 +58,14 @@ enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
 
 #define        rss_lock        gp_wait.lock
 
+#ifdef CONFIG_PROVE_RCU
+bool __rcu_sync_is_idle(struct rcu_sync *rsp)
+{
+       WARN_ON(!gp_ops[rsp->gp_type].held());
+       return rsp->gp_state == GP_IDLE;
+}
+#endif
+
 /**
  * rcu_sync_init() - Initialize an rcu_sync structure
  * @rsp: Pointer to rcu_sync structure to be initialized