rcutorture: Correctly handle grace-period sequence wrap

The new ->gq_seq grace-period sequence numbers must be shifted down,
which give artifacts when these numbers wrap.  This commit therefore
enables rcutorture and rcuperf to handle grace-period sequence numbers
even if they do wrap.  It does this by allowing a special subtraction
function to be specified, and this function subtracts before shifting.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 1c5cbd9d7c97073fc4207233dab56c6190f4c2ed..aa215d6355f87cdc940da1805d3d62537bd1cdd8 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -142,6 +142,15 @@ static inline bool rcu_seq_new_gp(unsigned long old, unsigned long new)
                            new);
 }
 
+/*
+ * Roughly how many full grace periods have elapsed between the collection
+ * of the two specified grace periods?
+ */
+static inline unsigned long rcu_seq_diff(unsigned long new, unsigned long old)
+{
+       return (new - old) >> RCU_SEQ_CTR_SHIFT;
+}
+
 /*
  * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
  * by call_rcu() and rcu callback execution, and are therefore not part of the

diff --git a/kernel/rcu/rcuperf.c b/kernel/rcu/rcuperf.c
index 2b5a613afcf3b8c90472b6ad0495895b6aeb6be8..b080bc4a4f456ac94e9289525dd1159517db4f3c 100644 (file)
--- a/kernel/rcu/rcuperf.c
+++ b/kernel/rcu/rcuperf.c
@@ -139,6 +139,7 @@ struct rcu_perf_ops {
        int (*readlock)(void);
        void (*readunlock)(int idx);
        unsigned long (*get_gp_seq)(void);
+       unsigned long (*gp_diff)(unsigned long new, unsigned long old);
        unsigned long (*exp_completed)(void);
        void (*async)(struct rcu_head *head, rcu_callback_t func);
        void (*gp_barrier)(void);
@@ -179,6 +180,7 @@ static struct rcu_perf_ops rcu_ops = {
        .readlock       = rcu_perf_read_lock,
        .readunlock     = rcu_perf_read_unlock,
        .get_gp_seq     = rcu_get_gp_seq,
+       .gp_diff        = rcu_seq_diff,
        .exp_completed  = rcu_exp_batches_completed,
        .async          = call_rcu,
        .gp_barrier     = rcu_barrier,
@@ -208,6 +210,7 @@ static struct rcu_perf_ops rcu_bh_ops = {
        .readlock       = rcu_bh_perf_read_lock,
        .readunlock     = rcu_bh_perf_read_unlock,
        .get_gp_seq     = rcu_bh_get_gp_seq,
+       .gp_diff        = rcu_seq_diff,
        .exp_completed  = rcu_exp_batches_completed_sched,
        .async          = call_rcu_bh,
        .gp_barrier     = rcu_barrier_bh,
@@ -264,6 +267,7 @@ static struct rcu_perf_ops srcu_ops = {
        .readlock       = srcu_perf_read_lock,
        .readunlock     = srcu_perf_read_unlock,
        .get_gp_seq     = srcu_perf_completed,
+       .gp_diff        = rcu_seq_diff,
        .exp_completed  = srcu_perf_completed,
        .async          = srcu_call_rcu,
        .gp_barrier     = srcu_rcu_barrier,
@@ -292,6 +296,7 @@ static struct rcu_perf_ops srcud_ops = {
        .readlock       = srcu_perf_read_lock,
        .readunlock     = srcu_perf_read_unlock,
        .get_gp_seq     = srcu_perf_completed,
+       .gp_diff        = rcu_seq_diff,
        .exp_completed  = srcu_perf_completed,
        .async          = srcu_call_rcu,
        .gp_barrier     = srcu_rcu_barrier,
@@ -321,6 +326,7 @@ static struct rcu_perf_ops sched_ops = {
        .readlock       = sched_perf_read_lock,
        .readunlock     = sched_perf_read_unlock,
        .get_gp_seq     = rcu_sched_get_gp_seq,
+       .gp_diff        = rcu_seq_diff,
        .exp_completed  = rcu_exp_batches_completed_sched,
        .async          = call_rcu_sched,
        .gp_barrier     = rcu_barrier_sched,
@@ -348,6 +354,7 @@ static struct rcu_perf_ops tasks_ops = {
        .readlock       = tasks_perf_read_lock,
        .readunlock     = tasks_perf_read_unlock,
        .get_gp_seq     = rcu_no_completed,
+       .gp_diff        = rcu_seq_diff,
        .async          = call_rcu_tasks,
        .gp_barrier     = rcu_barrier_tasks,
        .sync           = synchronize_rcu_tasks,
@@ -355,6 +362,13 @@ static struct rcu_perf_ops tasks_ops = {
        .name           = "tasks"
 };
 
+static unsigned long rcuperf_seq_diff(unsigned long new, unsigned long old)
+{
+       if (!cur_ops->gp_diff)
+               return new - old;
+       return cur_ops->gp_diff(new, old);
+}
+
 static bool __maybe_unused torturing_tasks(void)
 {
        return cur_ops == &tasks_ops;
@@ -577,8 +591,8 @@ rcu_perf_cleanup(void)
                         t_rcu_perf_writer_finished -
                         t_rcu_perf_writer_started,
                         ngps,
-                        b_rcu_perf_writer_finished -
-                        b_rcu_perf_writer_started);
+                        rcuperf_seq_diff(b_rcu_perf_writer_finished,
+                                         b_rcu_perf_writer_started));
                for (i = 0; i < nrealwriters; i++) {
                        if (!writer_durations)
                                break;

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 81fb43530d646d81d917afaeb754d3e1b0de023d..0481c728687535a1460a930a537c3207891d5aaf 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -265,6 +265,7 @@ struct rcu_torture_ops {
        void (*read_delay)(struct torture_random_state *rrsp);
        void (*readunlock)(int idx);
        unsigned long (*get_gp_seq)(void);
+       unsigned long (*gp_diff)(unsigned long new, unsigned long old);
        void (*deferred_free)(struct rcu_torture *p);
        void (*sync)(void);
        void (*exp_sync)(void);
@@ -400,6 +401,7 @@ static struct rcu_torture_ops rcu_ops = {
        .read_delay     = rcu_read_delay,
        .readunlock     = rcu_torture_read_unlock,
        .get_gp_seq     = rcu_get_gp_seq,
+       .gp_diff        = rcu_seq_diff,
        .deferred_free  = rcu_torture_deferred_free,
        .sync           = synchronize_rcu,
        .exp_sync       = synchronize_rcu_expedited,
@@ -441,6 +443,7 @@ static struct rcu_torture_ops rcu_bh_ops = {
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = rcu_bh_torture_read_unlock,
        .get_gp_seq     = rcu_bh_get_gp_seq,
+       .gp_diff        = rcu_seq_diff,
        .deferred_free  = rcu_bh_torture_deferred_free,
        .sync           = synchronize_rcu_bh,
        .exp_sync       = synchronize_rcu_bh_expedited,
@@ -646,6 +649,7 @@ static struct rcu_torture_ops sched_ops = {
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
        .get_gp_seq     = rcu_sched_get_gp_seq,
+       .gp_diff        = rcu_seq_diff,
        .deferred_free  = rcu_sched_torture_deferred_free,
        .sync           = synchronize_sched,
        .exp_sync       = synchronize_sched_expedited,
@@ -695,6 +699,13 @@ static struct rcu_torture_ops tasks_ops = {
        .name           = "tasks"
 };
 
+static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old)
+{
+       if (!cur_ops->gp_diff)
+               return new - old;
+       return cur_ops->gp_diff(new, old);
+}
+
 static bool __maybe_unused torturing_tasks(void)
 {
        return cur_ops == &tasks_ops;
@@ -1127,9 +1138,7 @@ static void rcu_torture_timer(struct timer_list *unused)
                rcu_ftrace_dump(DUMP_ALL);
        }
        __this_cpu_inc(rcu_torture_count[pipe_count]);
-       completed = completed - started;
-       if (completed > ULONG_MAX >> 1)
-               completed = 0; /* Not all gp_seq have full range. */
+       completed = rcutorture_seq_diff(completed, started);
        if (completed > RCU_TORTURE_PIPE_LEN) {
                /* Should not happen, but... */
                completed = RCU_TORTURE_PIPE_LEN;
@@ -1205,9 +1214,7 @@ rcu_torture_reader(void *arg)
                        rcu_ftrace_dump(DUMP_ALL);
                }
                __this_cpu_inc(rcu_torture_count[pipe_count]);
-               completed = completed - started;
-               if (completed > ULONG_MAX >> 1)
-                       completed = 0; /* Not all gp_seq have full range. */
+               completed = rcutorture_seq_diff(completed, started);
                if (completed > RCU_TORTURE_PIPE_LEN) {
                        /* Should not happen, but... */
                        completed = RCU_TORTURE_PIPE_LEN;

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cbf2bcde5e601887fe4ff910e413720b0252957a..fa219eea0ae73ba3f1df3f9345d6a0efacdd3644 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -532,7 +532,7 @@ static int rcu_pending(void);
  */
 unsigned long rcu_get_gp_seq(void)
 {
-       return rcu_seq_ctr(READ_ONCE(rcu_state_p->gp_seq));
+       return READ_ONCE(rcu_state_p->gp_seq);
 }
 EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
 
@@ -541,7 +541,7 @@ EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
  */
 unsigned long rcu_sched_get_gp_seq(void)
 {
-       return rcu_seq_ctr(READ_ONCE(rcu_sched_state.gp_seq));
+       return READ_ONCE(rcu_sched_state.gp_seq);
 }
 EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
 
@@ -550,7 +550,7 @@ EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
  */
 unsigned long rcu_bh_get_gp_seq(void)
 {
-       return rcu_seq_ctr(READ_ONCE(rcu_bh_state.gp_seq));
+       return READ_ONCE(rcu_bh_state.gp_seq);
 }
 EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);