Merge branches 'pm-core', 'pm-qos', 'pm-domains' and 'pm-opp'

[linux.git] / kernel / rcu / srcu.c

diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index c9a0015e1c2e2e53a72ffa7fd255e02bbb7103b4..e773129c8b08d29d0ce81cbdbc195bed905b2562 100644 (file)
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -320,7 +320,16 @@ static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount)
  */
 static void srcu_flip(struct srcu_struct *sp)
 {
-       sp->completed++;
+       WRITE_ONCE(sp->completed, sp->completed + 1);
+
+       /*
+        * Ensure that if the updater misses an __srcu_read_unlock()
+        * increment, that task's next __srcu_read_lock() will see the
+        * above counter update.  Note that both this memory barrier
+        * and the one in srcu_readers_active_idx_check() provide the
+        * guarantee for __srcu_read_lock().
+        */
+       smp_mb(); /* D */  /* Pairs with C. */
 }
 
 /*
@@ -358,6 +367,7 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *head,
        head->next = NULL;
        head->func = func;
        spin_lock_irqsave(&sp->queue_lock, flags);
+       smp_mb__after_unlock_lock(); /* Caller's prior accesses before GP. */
        rcu_batch_queue(&sp->batch_queue, head);
        if (!sp->running) {
                sp->running = true;
@@ -391,6 +401,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
        head->next = NULL;
        head->func = wakeme_after_rcu;
        spin_lock_irq(&sp->queue_lock);
+       smp_mb__after_unlock_lock(); /* Caller's prior accesses before GP. */
        if (!sp->running) {
                /* steal the processing owner */
                sp->running = true;
@@ -410,8 +421,11 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
                spin_unlock_irq(&sp->queue_lock);
        }
 
-       if (!done)
+       if (!done) {
                wait_for_completion(&rcu.completion);
+               smp_mb(); /* Caller's later accesses after GP. */
+       }
+
 }
 
 /**
@@ -579,7 +593,8 @@ static void srcu_advance_batches(struct srcu_struct *sp, int trycount)
 /*
  * Invoke a limited number of SRCU callbacks that have passed through
  * their grace period.  If there are more to do, SRCU will reschedule
- * the workqueue.
+ * the workqueue.  Note that needed memory barriers have been executed
+ * in this task's context by srcu_readers_active_idx_check().
  */
 static void srcu_invoke_callbacks(struct srcu_struct *sp)
 {