Merge tag 'random_for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso...

[linux.git] / drivers / md / dm-writecache.c

diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index 7d727a72aa139422695984d8f4fa1fb298860bd2..b9e27e37a94373c1cbc8ca52f3b64c0d72d46228 100644 (file)
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -442,7 +442,13 @@ static void writecache_notify_io(unsigned long error, void *context)
                complete(&endio->c);
 }
 
-static void ssd_commit_flushed(struct dm_writecache *wc)
+static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
+{
+       wait_event(wc->bio_in_progress_wait[direction],
+                  !atomic_read(&wc->bio_in_progress[direction]));
+}
+
+static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
 {
        struct dm_io_region region;
        struct dm_io_request req;
@@ -488,17 +494,20 @@ static void ssd_commit_flushed(struct dm_writecache *wc)
        writecache_notify_io(0, &endio);
        wait_for_completion_io(&endio.c);
 
+       if (wait_for_ios)
+               writecache_wait_for_ios(wc, WRITE);
+
        writecache_disk_flush(wc, wc->ssd_dev);
 
        memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
 }
 
-static void writecache_commit_flushed(struct dm_writecache *wc)
+static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
 {
        if (WC_MODE_PMEM(wc))
                wmb();
        else
-               ssd_commit_flushed(wc);
+               ssd_commit_flushed(wc, wait_for_ios);
 }
 
 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
@@ -522,12 +531,6 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
                writecache_error(wc, r, "error flushing metadata: %d", r);
 }
 
-static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
-{
-       wait_event(wc->bio_in_progress_wait[direction],
-                  !atomic_read(&wc->bio_in_progress[direction]));
-}
-
 #define WFE_RETURN_FOLLOWING   1
 #define WFE_LOWEST_SEQ         2
 
@@ -622,7 +625,7 @@ static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry
        wc->freelist_size++;
 }
 
-static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc)
+static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
 {
        struct wc_entry *e;
 
@@ -631,6 +634,8 @@ static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc)
                if (unlikely(!wc->current_free))
                        return NULL;
                e = wc->current_free;
+               if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
+                       return NULL;
                next = rb_next(&e->rb_node);
                rb_erase(&e->rb_node, &wc->freetree);
                if (unlikely(!next))
@@ -640,6 +645,8 @@ static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc)
                if (unlikely(list_empty(&wc->freelist)))
                        return NULL;
                e = container_of(wc->freelist.next, struct wc_entry, lru);
+               if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
+                       return NULL;
                list_del(&e->lru);
        }
        wc->freelist_size--;
@@ -724,15 +731,12 @@ static void writecache_flush(struct dm_writecache *wc)
                e = e2;
                cond_resched();
        }
-       writecache_commit_flushed(wc);
-
-       if (!WC_MODE_PMEM(wc))
-               writecache_wait_for_ios(wc, WRITE);
+       writecache_commit_flushed(wc, true);
 
        wc->seq_count++;
        pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
        writecache_flush_region(wc, &sb(wc)->seq_count, sizeof sb(wc)->seq_count);
-       writecache_commit_flushed(wc);
+       writecache_commit_flushed(wc, false);
 
        wc->overwrote_committed = false;
 
@@ -756,7 +760,7 @@ static void writecache_flush(struct dm_writecache *wc)
        }
 
        if (need_flush_after_free)
-               writecache_commit_flushed(wc);
+               writecache_commit_flushed(wc, false);
 }
 
 static void writecache_flush_work(struct work_struct *work)
@@ -809,7 +813,7 @@ static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_
        }
 
        if (discarded_something)
-               writecache_commit_flushed(wc);
+               writecache_commit_flushed(wc, false);
 }
 
 static bool writecache_wait_for_writeback(struct dm_writecache *wc)
@@ -958,7 +962,7 @@ static void writecache_resume(struct dm_target *ti)
 
        if (need_flush) {
                writecache_flush_all_metadata(wc);
-               writecache_commit_flushed(wc);
+               writecache_commit_flushed(wc, false);
        }
 
        wc_unlock(wc);
@@ -1193,7 +1197,7 @@ static int writecache_map(struct dm_target *ti, struct bio *bio)
                                        goto bio_copy;
                                }
                        }
-                       e = writecache_pop_from_freelist(wc);
+                       e = writecache_pop_from_freelist(wc, (sector_t)-1);
                        if (unlikely(!e)) {
                                writecache_wait_on_freelist(wc);
                                continue;
@@ -1205,9 +1209,26 @@ static int writecache_map(struct dm_target *ti, struct bio *bio)
                        if (WC_MODE_PMEM(wc)) {
                                bio_copy_block(wc, bio, memory_data(wc, e));
                        } else {
-                               dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
+                               unsigned bio_size = wc->block_size;
+                               sector_t start_cache_sec = cache_sector(wc, e);
+                               sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
+
+                               while (bio_size < bio->bi_iter.bi_size) {
+                                       struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
+                                       if (!f)
+                                               break;
+                                       write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
+                                                                       (bio_size >> SECTOR_SHIFT), wc->seq_count);
+                                       writecache_insert_entry(wc, f);
+                                       wc->uncommitted_blocks++;
+                                       bio_size += wc->block_size;
+                                       current_cache_sec += wc->block_size >> SECTOR_SHIFT;
+                               }
+
                                bio_set_dev(bio, wc->ssd_dev->bdev);
-                               bio->bi_iter.bi_sector = cache_sector(wc, e);
+                               bio->bi_iter.bi_sector = start_cache_sec;
+                               dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
+
                                if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
                                        wc->uncommitted_blocks = 0;
                                        queue_work(wc->writeback_wq, &wc->flush_work);
@@ -1342,7 +1363,7 @@ static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *
                        wc->writeback_size--;
                        n_walked++;
                        if (unlikely(n_walked >= ENDIO_LATENCY)) {
-                               writecache_commit_flushed(wc);
+                               writecache_commit_flushed(wc, false);
                                wc_unlock(wc);
                                wc_lock(wc);
                                n_walked = 0;
@@ -1423,7 +1444,7 @@ static int writecache_endio_thread(void *data)
                        writecache_wait_for_ios(wc, READ);
                }
 
-               writecache_commit_flushed(wc);
+               writecache_commit_flushed(wc, false);
 
                wc_unlock(wc);
        }
@@ -1766,10 +1787,10 @@ static int init_memory(struct dm_writecache *wc)
                write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
 
        writecache_flush_all_metadata(wc);
-       writecache_commit_flushed(wc);
+       writecache_commit_flushed(wc, false);
        pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
        writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
-       writecache_commit_flushed(wc);
+       writecache_commit_flushed(wc, false);
 
        return 0;
 }