for (i = 0; i < WBT_NUM_RWQ; i++) {
struct rq_wait *rqw = &rwb->rq_wait[i];
- if (waitqueue_active(&rqw->wait))
+ if (wq_has_sleeper(&rqw->wait))
wake_up_all(&rqw->wait);
}
}
if (inflight && inflight >= limit)
return;
- if (waitqueue_active(&rqw->wait)) {
+ if (wq_has_sleeper(&rqw->wait)) {
int diff = limit - inflight;
if (!inflight || diff >= rwb->wb_background / 2)
{
unsigned int limit;
+ /*
+ * If we got disabled, just return UINT_MAX. This ensures that
+ * we'll properly inc a new IO, and dec+wakeup at the end.
+ */
+ if (!rwb_enabled(rwb))
+ return UINT_MAX;
+
if ((rw & REQ_OP_MASK) == REQ_OP_DISCARD)
return rwb->wb_background;
{
struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
DECLARE_WAITQUEUE(wait, current);
+ bool has_sleeper;
- /*
- * inc it here even if disabled, since we'll dec it at completion.
- * this only happens if the task was sleeping in __wbt_wait(),
- * and someone turned it off at the same time.
- */
- if (!rwb_enabled(rwb)) {
- atomic_inc(&rqw->inflight);
- return;
- }
-
- if (!waitqueue_active(&rqw->wait)
- && rq_wait_inc_below(rqw, get_limit(rwb, rw)))
+ has_sleeper = wq_has_sleeper(&rqw->wait);
+ if (!has_sleeper && rq_wait_inc_below(rqw, get_limit(rwb, rw)))
return;
add_wait_queue_exclusive(&rqw->wait, &wait);
do {
set_current_state(TASK_UNINTERRUPTIBLE);
- if (!rwb_enabled(rwb)) {
- atomic_inc(&rqw->inflight);
- break;
- }
-
- if (rq_wait_inc_below(rqw, get_limit(rwb, rw)))
+ if (!has_sleeper && rq_wait_inc_below(rqw, get_limit(rwb, rw)))
break;
if (lock) {
spin_lock_irq(lock);
} else
io_schedule();
+ has_sleeper = false;
} while (1);
__set_current_state(TASK_RUNNING);
{
enum wbt_flags flags = 0;
+ if (!rwb_enabled(rwb))
+ return 0;
+
if (bio_op(bio) == REQ_OP_READ) {
flags = WBT_READ;
} else if (wbt_should_throttle(rwb, bio)) {