Merge tag 'f2fs-for-5.4' of git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs

[linux.git] / fs / io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 03fcd974fd1dc35ebe1114299880cdffb61f1282..0dadbdbead0fbfef8b0f2373756b3254832ada53 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -75,7 +75,7 @@
 
 #include "internal.h"
 
-#define IORING_MAX_ENTRIES     4096
+#define IORING_MAX_ENTRIES     32768
 #define IORING_MAX_FIXED_FILES 1024
 
 struct io_uring {
@@ -167,7 +167,7 @@ struct async_list {
        struct list_head        list;
 
        struct file             *file;
-       off_t                   io_end;
+       off_t                   io_start;
        size_t                  io_len;
 };
 
@@ -1189,6 +1189,28 @@ static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
        return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
 }
 
+static inline bool io_should_merge(struct async_list *al, struct kiocb *kiocb)
+{
+       if (al->file == kiocb->ki_filp) {
+               off_t start, end;
+
+               /*
+                * Allow merging if we're anywhere in the range of the same
+                * page. Generally this happens for sub-page reads or writes,
+                * and it's beneficial to allow the first worker to bring the
+                * page in and the piggy backed work can then work on the
+                * cached page.
+                */
+               start = al->io_start & PAGE_MASK;
+               end = (al->io_start + al->io_len + PAGE_SIZE - 1) & PAGE_MASK;
+               if (kiocb->ki_pos >= start && kiocb->ki_pos <= end)
+                       return true;
+       }
+
+       al->file = NULL;
+       return false;
+}
+
 /*
  * Make a note of the last file/offset/direction we punted to async
  * context. We'll use this information to see if we can piggy back a
@@ -1200,9 +1222,8 @@ static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
        struct async_list *async_list = &req->ctx->pending_async[rw];
        struct kiocb *kiocb = &req->rw;
        struct file *filp = kiocb->ki_filp;
-       off_t io_end = kiocb->ki_pos + len;
 
-       if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) {
+       if (io_should_merge(async_list, kiocb)) {
                unsigned long max_bytes;
 
                /* Use 8x RA size as a decent limiter for both reads/writes */
@@ -1215,17 +1236,16 @@ static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
                        req->flags |= REQ_F_SEQ_PREV;
                        async_list->io_len += len;
                } else {
-                       io_end = 0;
-                       async_list->io_len = 0;
+                       async_list->file = NULL;
                }
        }
 
        /* New file? Reset state. */
        if (async_list->file != filp) {
-               async_list->io_len = 0;
+               async_list->io_start = kiocb->ki_pos;
+               async_list->io_len = len;
                async_list->file = filp;
        }
-       async_list->io_end = io_end;
 }
 
 static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
@@ -1994,7 +2014,7 @@ static void io_sq_wq_submit_work(struct work_struct *work)
  */
 static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
 {
-       bool ret = false;
+       bool ret;
 
        if (!list)
                return false;
@@ -3336,15 +3356,12 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
         * Just return the requested submit count, and wake the thread if
         * we were asked to.
         */
+       ret = 0;
        if (ctx->flags & IORING_SETUP_SQPOLL) {
                if (flags & IORING_ENTER_SQ_WAKEUP)
                        wake_up(&ctx->sqo_wait);
                submitted = to_submit;
-               goto out_ctx;
-       }
-
-       ret = 0;
-       if (to_submit) {
+       } else if (to_submit) {
                bool block_for_last = false;
 
                to_submit = min(to_submit, ctx->sq_entries);
@@ -3374,7 +3391,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
                }
        }
 
-out_ctx:
        io_ring_drop_ctx_refs(ctx, 1);
 out_fput:
        fdput(f);