Merge branch 'pm-cpufreq'

[linux.git] / fs / io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index d542f1cf4428ed79af62c76273d32885490f72ff..cfb48bd088e12a0bbd8847e2b78e7ba14cb26fd8 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -679,6 +679,13 @@ static void io_put_req(struct io_kiocb *req)
                io_free_req(req);
 }
 
+static unsigned io_cqring_events(struct io_cq_ring *ring)
+{
+       /* See comment at the top of this file */
+       smp_rmb();
+       return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head);
+}
+
 /*
  * Find and free completed poll iocbs
  */
@@ -771,7 +778,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
 static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
                                long min)
 {
-       while (!list_empty(&ctx->poll_list)) {
+       while (!list_empty(&ctx->poll_list) && !need_resched()) {
                int ret;
 
                ret = io_do_iopoll(ctx, nr_events, min);
@@ -798,6 +805,12 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
                unsigned int nr_events = 0;
 
                io_iopoll_getevents(ctx, &nr_events, 1);
+
+               /*
+                * Ensure we allow local-to-the-cpu processing to take place,
+                * in this case we need to ensure that we reap all events.
+                */
+               cond_resched();
        }
        mutex_unlock(&ctx->uring_lock);
 }
@@ -805,11 +818,42 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
 static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
                           long min)
 {
-       int ret = 0;
+       int iters, ret = 0;
 
+       /*
+        * We disallow the app entering submit/complete with polling, but we
+        * still need to lock the ring to prevent racing with polled issue
+        * that got punted to a workqueue.
+        */
+       mutex_lock(&ctx->uring_lock);
+
+       iters = 0;
        do {
                int tmin = 0;
 
+               /*
+                * Don't enter poll loop if we already have events pending.
+                * If we do, we can potentially be spinning for commands that
+                * already triggered a CQE (eg in error).
+                */
+               if (io_cqring_events(ctx->cq_ring))
+                       break;
+
+               /*
+                * If a submit got punted to a workqueue, we can have the
+                * application entering polling for a command before it gets
+                * issued. That app will hold the uring_lock for the duration
+                * of the poll right here, so we need to take a breather every
+                * now and then to ensure that the issue has a chance to add
+                * the poll to the issued list. Otherwise we can spin here
+                * forever, while the workqueue is stuck trying to acquire the
+                * very same mutex.
+                */
+               if (!(++iters & 7)) {
+                       mutex_unlock(&ctx->uring_lock);
+                       mutex_lock(&ctx->uring_lock);
+               }
+
                if (*nr_events < min)
                        tmin = min - *nr_events;
 
@@ -819,6 +863,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
                ret = 0;
        } while (min && !*nr_events && !need_resched());
 
+       mutex_unlock(&ctx->uring_lock);
        return ret;
 }
 
@@ -1097,10 +1142,8 @@ static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
 
                        iter->bvec = bvec + seg_skip;
                        iter->nr_segs -= seg_skip;
-                       iter->count -= (seg_skip << PAGE_SHIFT);
+                       iter->count -= bvec->bv_len + offset;
                        iter->iov_offset = offset & ~PAGE_MASK;
-                       if (iter->iov_offset)
-                               iter->count -= iter->iov_offset;
                }
        }
 
@@ -2025,6 +2068,15 @@ static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 {
        int ret;
 
+       ret = io_req_defer(ctx, req, s->sqe);
+       if (ret) {
+               if (ret != -EIOCBQUEUED) {
+                       io_free_req(req);
+                       io_cqring_add_event(ctx, s->sqe->user_data, ret);
+               }
+               return 0;
+       }
+
        ret = __io_submit_sqe(ctx, req, s, true);
        if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
                struct io_uring_sqe *sqe_copy;
@@ -2097,13 +2149,6 @@ static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
                return;
        }
 
-       ret = io_req_defer(ctx, req, s->sqe);
-       if (ret) {
-               if (ret != -EIOCBQUEUED)
-                       goto err_req;
-               return;
-       }
-
        /*
         * If we already have a head request, queue this one for async
         * submittal once the head completes. If we don't have a head but
@@ -2280,15 +2325,7 @@ static int io_sq_thread(void *data)
                        unsigned nr_events = 0;
 
                        if (ctx->flags & IORING_SETUP_IOPOLL) {
-                               /*
-                                * We disallow the app entering submit/complete
-                                * with polling, but we still need to lock the
-                                * ring to prevent racing with polled issue
-                                * that got punted to a workqueue.
-                                */
-                               mutex_lock(&ctx->uring_lock);
                                io_iopoll_check(ctx, &nr_events, 0);
-                               mutex_unlock(&ctx->uring_lock);
                        } else {
                                /*
                                 * Normal IO, just pretend everything completed.
@@ -2433,13 +2470,6 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
        return submit;
 }
 
-static unsigned io_cqring_events(struct io_cq_ring *ring)
-{
-       /* See comment at the top of this file */
-       smp_rmb();
-       return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head);
-}
-
 /*
  * Wait until events become available, if we don't already have some. The
  * application must reap them itself, as they reside on the shared cq ring.
@@ -3190,9 +3220,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
                min_complete = min(min_complete, ctx->cq_entries);
 
                if (ctx->flags & IORING_SETUP_IOPOLL) {
-                       mutex_lock(&ctx->uring_lock);
                        ret = io_iopoll_check(ctx, &nr_events, min_complete);
-                       mutex_unlock(&ctx->uring_lock);
                } else {
                        ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
                }