Merge tag 'devdax-for-5.1' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm

[linux.git] / fs / block_dev.c

diff --git a/fs/block_dev.c b/fs/block_dev.c
index 58a4c1217fa87d60376771a2c315b8b02d0bff1c..e9faa52bb489c424775af69ccf9362c1c4b07c2f 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -211,6 +211,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
        ssize_t ret;
        blk_qc_t qc;
        int i;
+       struct bvec_iter_all iter_all;
 
        if ((pos | iov_iter_alignment(iter)) &
            (bdev_logical_block_size(bdev) - 1))
@@ -247,7 +248,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
                task_io_account_write(ret);
        }
        if (iocb->ki_flags & IOCB_HIPRI)
-               bio.bi_opf |= REQ_HIPRI;
+               bio_set_polled(&bio, iocb);
 
        qc = submit_bio(&bio);
        for (;;) {
@@ -260,7 +261,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
        }
        __set_current_state(TASK_RUNNING);
 
-       bio_for_each_segment_all(bvec, &bio, i) {
+       bio_for_each_segment_all(bvec, &bio, i, iter_all) {
                if (should_dirty && !PageCompound(bvec->bv_page))
                        set_page_dirty_lock(bvec->bv_page);
                put_page(bvec->bv_page);
@@ -293,6 +294,14 @@ struct blkdev_dio {
 
 static struct bio_set blkdev_dio_pool;
 
+static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
+{
+       struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
+       struct request_queue *q = bdev_get_queue(bdev);
+
+       return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
+}
+
 static void blkdev_bio_end_io(struct bio *bio)
 {
        struct blkdev_dio *dio = bio->bi_private;
@@ -329,8 +338,9 @@ static void blkdev_bio_end_io(struct bio *bio)
        } else {
                struct bio_vec *bvec;
                int i;
+               struct bvec_iter_all iter_all;
 
-               bio_for_each_segment_all(bvec, bio, i)
+               bio_for_each_segment_all(bvec, bio, i, iter_all)
                        put_page(bvec->bv_page);
                bio_put(bio);
        }
@@ -406,10 +416,17 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
 
                nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
                if (!nr_pages) {
-                       if (iocb->ki_flags & IOCB_HIPRI)
-                               bio->bi_opf |= REQ_HIPRI;
+                       bool polled = false;
+
+                       if (iocb->ki_flags & IOCB_HIPRI) {
+                               bio_set_polled(bio, iocb);
+                               polled = true;
+                       }
 
                        qc = submit_bio(bio);
+
+                       if (polled)
+                               WRITE_ONCE(iocb->ki_cookie, qc);
                        break;
                }
 
@@ -2076,6 +2093,7 @@ const struct file_operations def_blk_fops = {
        .llseek         = block_llseek,
        .read_iter      = blkdev_read_iter,
        .write_iter     = blkdev_write_iter,
+       .iopoll         = blkdev_iopoll,
        .mmap           = generic_file_mmap,
        .fsync          = blkdev_fsync,
        .unlocked_ioctl = block_ioctl,