Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[linux.git] / lib / iov_iter.c

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 42d39116a5564daf057b6248eb95e433c4c101db..54c248526b55fc498c996f2a5ea651262fcc7f61 100644 (file)
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -83,6 +83,7 @@
                        const struct kvec *kvec;                \
                        struct kvec v;                          \
                        iterate_kvec(i, n, v, kvec, skip, (K))  \
+               } else if (unlikely(i->type & ITER_DISCARD)) {  \
                } else {                                        \
                        const struct iovec *iov;                \
                        struct iovec v;                         \
@@ -114,6 +115,8 @@
                        }                                       \
                        i->nr_segs -= kvec - i->kvec;           \
                        i->kvec = kvec;                         \
+               } else if (unlikely(i->type & ITER_DISCARD)) {  \
+                       skip += n;                              \
                } else {                                        \
                        const struct iovec *iov;                \
                        struct iovec v;                         \
@@ -428,17 +431,19 @@ int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 }
 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 
-void iov_iter_init(struct iov_iter *i, int direction,
+void iov_iter_init(struct iov_iter *i, unsigned int direction,
                        const struct iovec *iov, unsigned long nr_segs,
                        size_t count)
 {
+       WARN_ON(direction & ~(READ | WRITE));
+       direction &= READ | WRITE;
+
        /* It will get better.  Eventually... */
        if (uaccess_kernel()) {
-               direction |= ITER_KVEC;
-               i->type = direction;
+               i->type = ITER_KVEC | direction;
                i->kvec = (struct kvec *)iov;
        } else {
-               i->type = direction;
+               i->type = ITER_IOVEC | direction;
                i->iov = iov;
        }
        i->nr_segs = nr_segs;
@@ -555,6 +560,38 @@ static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
        return bytes;
 }
 
+static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
+                               __wsum *csum, struct iov_iter *i)
+{
+       struct pipe_inode_info *pipe = i->pipe;
+       size_t n, r;
+       size_t off = 0;
+       __wsum sum = *csum, next;
+       int idx;
+
+       if (!sanity(i))
+               return 0;
+
+       bytes = n = push_pipe(i, bytes, &idx, &r);
+       if (unlikely(!n))
+               return 0;
+       for ( ; n; idx = next_idx(idx, pipe), r = 0) {
+               size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
+               char *p = kmap_atomic(pipe->bufs[idx].page);
+               next = csum_partial_copy_nocheck(addr, p + r, chunk, 0);
+               sum = csum_block_add(sum, next, off);
+               kunmap_atomic(p);
+               i->idx = idx;
+               i->iov_offset = r + chunk;
+               n -= chunk;
+               off += chunk;
+               addr += chunk;
+       }
+       i->count -= bytes;
+       *csum = sum;
+       return bytes;
+}
+
 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 {
        const char *from = addr;
@@ -836,7 +873,9 @@ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
                size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
                kunmap_atomic(kaddr);
                return wanted;
-       } else if (likely(!iov_iter_is_pipe(i)))
+       } else if (unlikely(iov_iter_is_discard(i)))
+               return bytes;
+       else if (likely(!iov_iter_is_pipe(i)))
                return copy_page_to_iter_iovec(page, offset, bytes, i);
        else
                return copy_page_to_iter_pipe(page, offset, bytes, i);
@@ -848,7 +887,7 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 {
        if (unlikely(!page_copy_sane(page, offset, bytes)))
                return 0;
-       if (unlikely(iov_iter_is_pipe(i))) {
+       if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
                WARN_ON(1);
                return 0;
        }
@@ -908,7 +947,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page,
                kunmap_atomic(kaddr);
                return 0;
        }
-       if (unlikely(iov_iter_is_pipe(i))) {
+       if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
                kunmap_atomic(kaddr);
                WARN_ON(1);
                return 0;
@@ -976,6 +1015,10 @@ void iov_iter_advance(struct iov_iter *i, size_t size)
                pipe_advance(i, size);
                return;
        }
+       if (unlikely(iov_iter_is_discard(i))) {
+               i->count -= size;
+               return;
+       }
        iterate_and_advance(i, size, v, 0, 0, 0)
 }
 EXPORT_SYMBOL(iov_iter_advance);
@@ -1011,6 +1054,8 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll)
                pipe_truncate(i);
                return;
        }
+       if (unlikely(iov_iter_is_discard(i)))
+               return;
        if (unroll <= i->iov_offset) {
                i->iov_offset -= unroll;
                return;
@@ -1053,6 +1098,8 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i)
                return i->count;        // it is a silly place, anyway
        if (i->nr_segs == 1)
                return i->count;
+       if (unlikely(iov_iter_is_discard(i)))
+               return i->count;
        else if (iov_iter_is_bvec(i))
                return min(i->count, i->bvec->bv_len - i->iov_offset);
        else
@@ -1060,12 +1107,12 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i)
 }
 EXPORT_SYMBOL(iov_iter_single_seg_count);
 
-void iov_iter_kvec(struct iov_iter *i, int direction,
+void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
                        const struct kvec *kvec, unsigned long nr_segs,
                        size_t count)
 {
-       BUG_ON(!(direction & ITER_KVEC));
-       i->type = direction;
+       WARN_ON(direction & ~(READ | WRITE));
+       i->type = ITER_KVEC | (direction & (READ | WRITE));
        i->kvec = kvec;
        i->nr_segs = nr_segs;
        i->iov_offset = 0;
@@ -1073,12 +1120,12 @@ void iov_iter_kvec(struct iov_iter *i, int direction,
 }
 EXPORT_SYMBOL(iov_iter_kvec);
 
-void iov_iter_bvec(struct iov_iter *i, int direction,
+void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
                        const struct bio_vec *bvec, unsigned long nr_segs,
                        size_t count)
 {
-       BUG_ON(!(direction & ITER_BVEC));
-       i->type = direction;
+       WARN_ON(direction & ~(READ | WRITE));
+       i->type = ITER_BVEC | (direction & (READ | WRITE));
        i->bvec = bvec;
        i->nr_segs = nr_segs;
        i->iov_offset = 0;
@@ -1086,13 +1133,13 @@ void iov_iter_bvec(struct iov_iter *i, int direction,
 }
 EXPORT_SYMBOL(iov_iter_bvec);
 
-void iov_iter_pipe(struct iov_iter *i, int direction,
+void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
                        struct pipe_inode_info *pipe,
                        size_t count)
 {
-       BUG_ON(direction != ITER_PIPE);
+       BUG_ON(direction != READ);
        WARN_ON(pipe->nrbufs == pipe->buffers);
-       i->type = direction;
+       i->type = ITER_PIPE | READ;
        i->pipe = pipe;
        i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
        i->iov_offset = 0;
@@ -1101,6 +1148,24 @@ void iov_iter_pipe(struct iov_iter *i, int direction,
 }
 EXPORT_SYMBOL(iov_iter_pipe);
 
+/**
+ * iov_iter_discard - Initialise an I/O iterator that discards data
+ * @i: The iterator to initialise.
+ * @direction: The direction of the transfer.
+ * @count: The size of the I/O buffer in bytes.
+ *
+ * Set up an I/O iterator that just discards everything that's written to it.
+ * It's only available as a READ iterator.
+ */
+void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
+{
+       BUG_ON(direction != READ);
+       i->type = ITER_DISCARD | READ;
+       i->count = count;
+       i->iov_offset = 0;
+}
+EXPORT_SYMBOL(iov_iter_discard);
+
 unsigned long iov_iter_alignment(const struct iov_iter *i)
 {
        unsigned long res = 0;
@@ -1125,7 +1190,7 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
        unsigned long res = 0;
        size_t size = i->count;
 
-       if (unlikely(iov_iter_is_pipe(i))) {
+       if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
                WARN_ON(1);
                return ~0U;
        }
@@ -1195,6 +1260,9 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
 
        if (unlikely(iov_iter_is_pipe(i)))
                return pipe_get_pages(i, pages, maxsize, maxpages, start);
+       if (unlikely(iov_iter_is_discard(i)))
+               return -EFAULT;
+
        iterate_all_kinds(i, maxsize, v, ({
                unsigned long addr = (unsigned long)v.iov_base;
                size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
@@ -1272,6 +1340,9 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
 
        if (unlikely(iov_iter_is_pipe(i)))
                return pipe_get_pages_alloc(i, pages, maxsize, start);
+       if (unlikely(iov_iter_is_discard(i)))
+               return -EFAULT;
+
        iterate_all_kinds(i, maxsize, v, ({
                unsigned long addr = (unsigned long)v.iov_base;
                size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
@@ -1313,7 +1384,7 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
        __wsum sum, next;
        size_t off = 0;
        sum = *csum;
-       if (unlikely(iov_iter_is_pipe(i))) {
+       if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
                WARN_ON(1);
                return 0;
        }
@@ -1355,7 +1426,7 @@ bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
        __wsum sum, next;
        size_t off = 0;
        sum = *csum;
-       if (unlikely(iov_iter_is_pipe(i))) {
+       if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
                WARN_ON(1);
                return false;
        }
@@ -1399,8 +1470,12 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
        const char *from = addr;
        __wsum sum, next;
        size_t off = 0;
+
+       if (unlikely(iov_iter_is_pipe(i)))
+               return csum_and_copy_to_pipe_iter(addr, bytes, csum, i);
+
        sum = *csum;
-       if (unlikely(iov_iter_is_pipe(i))) {
+       if (unlikely(iov_iter_is_discard(i))) {
                WARN_ON(1);     /* for now */
                return 0;
        }
@@ -1442,6 +1517,8 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages)
 
        if (!size)
                return 0;
+       if (unlikely(iov_iter_is_discard(i)))
+               return 0;
 
        if (unlikely(iov_iter_is_pipe(i))) {
                struct pipe_inode_info *pipe = i->pipe;
@@ -1485,6 +1562,8 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
                WARN_ON(1);
                return NULL;
        }
+       if (unlikely(iov_iter_is_discard(new)))
+               return NULL;
        if (iov_iter_is_bvec(new))
                return new->bvec = kmemdup(new->bvec,
                                    new->nr_segs * sizeof(struct bio_vec),