drivers/lightnvm/pblk-rb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2016 CNEX Labs
   4  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
   5  *
   6  * Based upon the circular ringbuffer.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License version
  10  * 2 as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License for more details.
  16  *
  17  * pblk-rb.c - pblk's write buffer
  18  */
  19
  20 #include <linux/circ_buf.h>
  21
  22 #include "pblk.h"
  23
  24 static DECLARE_RWSEM(pblk_rb_lock);
  25
  26 static void pblk_rb_data_free(struct pblk_rb *rb)
  27 {
  28         struct pblk_rb_pages *p, *t;
  29
  30         down_write(&pblk_rb_lock);
  31         list_for_each_entry_safe(p, t, &rb->pages, list) {
  32                 free_pages((unsigned long)page_address(p->pages), p->order);
  33                 list_del(&p->list);
  34                 kfree(p);
  35         }
  36         up_write(&pblk_rb_lock);
  37 }
  38
  39 void pblk_rb_free(struct pblk_rb *rb)
  40 {
  41         pblk_rb_data_free(rb);
  42         vfree(rb->entries);
  43 }
  44
  45 /*
  46  * pblk_rb_calculate_size -- calculate the size of the write buffer
  47  */
  48 static unsigned int pblk_rb_calculate_size(unsigned int nr_entries)
  49 {
  50         /* Alloc a write buffer that can at least fit 128 entries */
  51         return (1 << max(get_count_order(nr_entries), 7));
  52 }
  53
  54 /*
  55  * Initialize ring buffer. The data and metadata buffers must be previously
  56  * allocated and their size must be a power of two
  57  * (Documentation/core-api/circular-buffers.rst)
  58  */
  59 int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int seg_size)
  60 {
  61         struct pblk *pblk = container_of(rb, struct pblk, rwb);
  62         struct pblk_rb_entry *entries;
  63         unsigned int init_entry = 0;
  64         unsigned int max_order = MAX_ORDER - 1;
  65         unsigned int power_size, power_seg_sz;
  66         unsigned int alloc_order, order, iter;
  67         unsigned int nr_entries;
  68
  69         nr_entries = pblk_rb_calculate_size(size);
  70         entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry)));
  71         if (!entries)
  72                 return -ENOMEM;
  73
  74         power_size = get_count_order(size);
  75         power_seg_sz = get_count_order(seg_size);
  76
  77         down_write(&pblk_rb_lock);
  78         rb->entries = entries;
  79         rb->seg_size = (1 << power_seg_sz);
  80         rb->nr_entries = (1 << power_size);
  81         rb->mem = rb->subm = rb->sync = rb->l2p_update = 0;
  82         rb->flush_point = EMPTY_ENTRY;
  83
  84         spin_lock_init(&rb->w_lock);
  85         spin_lock_init(&rb->s_lock);
  86
  87         INIT_LIST_HEAD(&rb->pages);
  88
  89         alloc_order = power_size;
  90         if (alloc_order >= max_order) {
  91                 order = max_order;
  92                 iter = (1 << (alloc_order - max_order));
  93         } else {
  94                 order = alloc_order;
  95                 iter = 1;
  96         }
  97
  98         do {
  99                 struct pblk_rb_entry *entry;
 100                 struct pblk_rb_pages *page_set;
 101                 void *kaddr;
 102                 unsigned long set_size;
 103                 int i;
 104
 105                 page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL);
 106                 if (!page_set) {
 107                         up_write(&pblk_rb_lock);
 108                         vfree(entries);
 109                         return -ENOMEM;
 110                 }
 111
 112                 page_set->order = order;
 113                 page_set->pages = alloc_pages(GFP_KERNEL, order);
 114                 if (!page_set->pages) {
 115                         kfree(page_set);
 116                         pblk_rb_data_free(rb);
 117                         up_write(&pblk_rb_lock);
 118                         vfree(entries);
 119                         return -ENOMEM;
 120                 }
 121                 kaddr = page_address(page_set->pages);
 122
 123                 entry = &rb->entries[init_entry];
 124                 entry->data = kaddr;
 125                 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
 126                 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
 127
 128                 set_size = (1 << order);
 129                 for (i = 1; i < set_size; i++) {
 130                         entry = &rb->entries[init_entry];
 131                         entry->cacheline = pblk_cacheline_to_addr(init_entry++);
 132                         entry->data = kaddr + (i * rb->seg_size);
 133                         entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
 134                         bio_list_init(&entry->w_ctx.bios);
 135                 }
 136
 137                 list_add_tail(&page_set->list, &rb->pages);
 138                 iter--;
 139         } while (iter > 0);
 140         up_write(&pblk_rb_lock);
 141
 142 #ifdef CONFIG_NVM_PBLK_DEBUG
 143         atomic_set(&rb->inflight_flush_point, 0);
 144 #endif
 145
 146         /*
 147          * Initialize rate-limiter, which controls access to the write buffer
 148          * but user and GC I/O
 149          */
 150         pblk_rl_init(&pblk->rl, rb->nr_entries);
 151
 152         return 0;
 153 }
 154
 155 static void clean_wctx(struct pblk_w_ctx *w_ctx)
 156 {
 157         int flags;
 158
 159         flags = READ_ONCE(w_ctx->flags);
 160         WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY),
 161                         "pblk: overwriting unsubmitted data\n");
 162
 163         /* Release flags on context. Protect from writes and reads */
 164         smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY);
 165         pblk_ppa_set_empty(&w_ctx->ppa);
 166         w_ctx->lba = ADDR_EMPTY;
 167 }
 168
 169 #define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size)
 170 #define pblk_rb_ring_space(rb, head, tail, size) \
 171                                         (CIRC_SPACE(head, tail, size))
 172
 173 /*
 174  * Buffer space is calculated with respect to the back pointer signaling
 175  * synchronized entries to the media.
 176  */
 177 static unsigned int pblk_rb_space(struct pblk_rb *rb)
 178 {
 179         unsigned int mem = READ_ONCE(rb->mem);
 180         unsigned int sync = READ_ONCE(rb->sync);
 181
 182         return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries);
 183 }
 184
 185 unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p,
 186                               unsigned int nr_entries)
 187 {
 188         return (p + nr_entries) & (rb->nr_entries - 1);
 189 }
 190
 191 /*
 192  * Buffer count is calculated with respect to the submission entry signaling the
 193  * entries that are available to send to the media
 194  */
 195 unsigned int pblk_rb_read_count(struct pblk_rb *rb)
 196 {
 197         unsigned int mem = READ_ONCE(rb->mem);
 198         unsigned int subm = READ_ONCE(rb->subm);
 199
 200         return pblk_rb_ring_count(mem, subm, rb->nr_entries);
 201 }
 202
 203 unsigned int pblk_rb_sync_count(struct pblk_rb *rb)
 204 {
 205         unsigned int mem = READ_ONCE(rb->mem);
 206         unsigned int sync = READ_ONCE(rb->sync);
 207
 208         return pblk_rb_ring_count(mem, sync, rb->nr_entries);
 209 }
 210
 211 unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries)
 212 {
 213         unsigned int subm;
 214
 215         subm = READ_ONCE(rb->subm);
 216         /* Commit read means updating submission pointer */
 217         smp_store_release(&rb->subm, pblk_rb_ptr_wrap(rb, subm, nr_entries));
 218
 219         return subm;
 220 }
 221
 222 static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update)
 223 {
 224         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 225         struct pblk_line *line;
 226         struct pblk_rb_entry *entry;
 227         struct pblk_w_ctx *w_ctx;
 228         unsigned int user_io = 0, gc_io = 0;
 229         unsigned int i;
 230         int flags;
 231
 232         for (i = 0; i < to_update; i++) {
 233                 entry = &rb->entries[rb->l2p_update];
 234                 w_ctx = &entry->w_ctx;
 235
 236                 flags = READ_ONCE(entry->w_ctx.flags);
 237                 if (flags & PBLK_IOTYPE_USER)
 238                         user_io++;
 239                 else if (flags & PBLK_IOTYPE_GC)
 240                         gc_io++;
 241                 else
 242                         WARN(1, "pblk: unknown IO type\n");
 243
 244                 pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa,
 245                                                         entry->cacheline);
 246
 247                 line = pblk_ppa_to_line(pblk, w_ctx->ppa);
 248                 kref_put(&line->ref, pblk_line_put);
 249                 clean_wctx(w_ctx);
 250                 rb->l2p_update = pblk_rb_ptr_wrap(rb, rb->l2p_update, 1);
 251         }
 252
 253         pblk_rl_out(&pblk->rl, user_io, gc_io);
 254
 255         return 0;
 256 }
 257
 258 /*
 259  * When we move the l2p_update pointer, we update the l2p table - lookups will
 260  * point to the physical address instead of to the cacheline in the write buffer
 261  * from this moment on.
 262  */
 263 static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries,
 264                               unsigned int mem, unsigned int sync)
 265 {
 266         unsigned int space, count;
 267         int ret = 0;
 268
 269         lockdep_assert_held(&rb->w_lock);
 270
 271         /* Update l2p only as buffer entries are being overwritten */
 272         space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries);
 273         if (space > nr_entries)
 274                 goto out;
 275
 276         count = nr_entries - space;
 277         /* l2p_update used exclusively under rb->w_lock */
 278         ret = __pblk_rb_update_l2p(rb, count);
 279
 280 out:
 281         return ret;
 282 }
 283
 284 /*
 285  * Update the l2p entry for all sectors stored on the write buffer. This means
 286  * that all future lookups to the l2p table will point to a device address, not
 287  * to the cacheline in the write buffer.
 288  */
 289 void pblk_rb_sync_l2p(struct pblk_rb *rb)
 290 {
 291         unsigned int sync;
 292         unsigned int to_update;
 293
 294         spin_lock(&rb->w_lock);
 295
 296         /* Protect from reads and writes */
 297         sync = smp_load_acquire(&rb->sync);
 298
 299         to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries);
 300         __pblk_rb_update_l2p(rb, to_update);
 301
 302         spin_unlock(&rb->w_lock);
 303 }
 304
 305 /*
 306  * Write @nr_entries to ring buffer from @data buffer if there is enough space.
 307  * Typically, 4KB data chunks coming from a bio will be copied to the ring
 308  * buffer, thus the write will fail if not all incoming data can be copied.
 309  *
 310  */
 311 static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data,
 312                                   struct pblk_w_ctx w_ctx,
 313                                   struct pblk_rb_entry *entry)
 314 {
 315         memcpy(entry->data, data, rb->seg_size);
 316
 317         entry->w_ctx.lba = w_ctx.lba;
 318         entry->w_ctx.ppa = w_ctx.ppa;
 319 }
 320
 321 void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data,
 322                               struct pblk_w_ctx w_ctx, unsigned int ring_pos)
 323 {
 324         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 325         struct pblk_rb_entry *entry;
 326         int flags;
 327
 328         entry = &rb->entries[ring_pos];
 329         flags = READ_ONCE(entry->w_ctx.flags);
 330 #ifdef CONFIG_NVM_PBLK_DEBUG
 331         /* Caller must guarantee that the entry is free */
 332         BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
 333 #endif
 334
 335         __pblk_rb_write_entry(rb, data, w_ctx, entry);
 336
 337         pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline);
 338         flags = w_ctx.flags | PBLK_WRITTEN_DATA;
 339
 340         /* Release flags on write context. Protect from writes */
 341         smp_store_release(&entry->w_ctx.flags, flags);
 342 }
 343
 344 void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
 345                             struct pblk_w_ctx w_ctx, struct pblk_line *line,
 346                             u64 paddr, unsigned int ring_pos)
 347 {
 348         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 349         struct pblk_rb_entry *entry;
 350         int flags;
 351
 352         entry = &rb->entries[ring_pos];
 353         flags = READ_ONCE(entry->w_ctx.flags);
 354 #ifdef CONFIG_NVM_PBLK_DEBUG
 355         /* Caller must guarantee that the entry is free */
 356         BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
 357 #endif
 358
 359         __pblk_rb_write_entry(rb, data, w_ctx, entry);
 360
 361         if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr))
 362                 entry->w_ctx.lba = ADDR_EMPTY;
 363
 364         flags = w_ctx.flags | PBLK_WRITTEN_DATA;
 365
 366         /* Release flags on write context. Protect from writes */
 367         smp_store_release(&entry->w_ctx.flags, flags);
 368 }
 369
 370 static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
 371                                    unsigned int pos)
 372 {
 373         struct pblk_rb_entry *entry;
 374         unsigned int sync, flush_point;
 375
 376         pblk_rb_sync_init(rb, NULL);
 377         sync = READ_ONCE(rb->sync);
 378
 379         if (pos == sync) {
 380                 pblk_rb_sync_end(rb, NULL);
 381                 return 0;
 382         }
 383
 384 #ifdef CONFIG_NVM_PBLK_DEBUG
 385         atomic_inc(&rb->inflight_flush_point);
 386 #endif
 387
 388         flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
 389         entry = &rb->entries[flush_point];
 390
 391         /* Protect flush points */
 392         smp_store_release(&rb->flush_point, flush_point);
 393
 394         if (bio)
 395                 bio_list_add(&entry->w_ctx.bios, bio);
 396
 397         pblk_rb_sync_end(rb, NULL);
 398
 399         return bio ? 1 : 0;
 400 }
 401
 402 static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
 403                                unsigned int *pos)
 404 {
 405         unsigned int mem;
 406         unsigned int sync;
 407
 408         sync = READ_ONCE(rb->sync);
 409         mem = READ_ONCE(rb->mem);
 410
 411         if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < nr_entries)
 412                 return 0;
 413
 414         if (pblk_rb_update_l2p(rb, nr_entries, mem, sync))
 415                 return 0;
 416
 417         *pos = mem;
 418
 419         return 1;
 420 }
 421
 422 static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
 423                              unsigned int *pos)
 424 {
 425         if (!__pblk_rb_may_write(rb, nr_entries, pos))
 426                 return 0;
 427
 428         /* Protect from read count */
 429         smp_store_release(&rb->mem, pblk_rb_ptr_wrap(rb, *pos, nr_entries));
 430         return 1;
 431 }
 432
 433 void pblk_rb_flush(struct pblk_rb *rb)
 434 {
 435         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 436         unsigned int mem = READ_ONCE(rb->mem);
 437
 438         if (pblk_rb_flush_point_set(rb, NULL, mem))
 439                 return;
 440
 441         pblk_write_kick(pblk);
 442 }
 443
 444 static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries,
 445                                    unsigned int *pos, struct bio *bio,
 446                                    int *io_ret)
 447 {
 448         unsigned int mem;
 449
 450         if (!__pblk_rb_may_write(rb, nr_entries, pos))
 451                 return 0;
 452
 453         mem = pblk_rb_ptr_wrap(rb, *pos, nr_entries);
 454         *io_ret = NVM_IO_DONE;
 455
 456         if (bio->bi_opf & REQ_PREFLUSH) {
 457                 struct pblk *pblk = container_of(rb, struct pblk, rwb);
 458
 459                 atomic64_inc(&pblk->nr_flush);
 460                 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem))
 461                         *io_ret = NVM_IO_OK;
 462         }
 463
 464         /* Protect from read count */
 465         smp_store_release(&rb->mem, mem);
 466
 467         return 1;
 468 }
 469
 470 /*
 471  * Atomically check that (i) there is space on the write buffer for the
 472  * incoming I/O, and (ii) the current I/O type has enough budget in the write
 473  * buffer (rate-limiter).
 474  */
 475 int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio,
 476                            unsigned int nr_entries, unsigned int *pos)
 477 {
 478         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 479         int io_ret;
 480
 481         spin_lock(&rb->w_lock);
 482         io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries);
 483         if (io_ret) {
 484                 spin_unlock(&rb->w_lock);
 485                 return io_ret;
 486         }
 487
 488         if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) {
 489                 spin_unlock(&rb->w_lock);
 490                 return NVM_IO_REQUEUE;
 491         }
 492
 493         pblk_rl_user_in(&pblk->rl, nr_entries);
 494         spin_unlock(&rb->w_lock);
 495
 496         return io_ret;
 497 }
 498
 499 /*
 500  * Look at pblk_rb_may_write_user comment
 501  */
 502 int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries,
 503                          unsigned int *pos)
 504 {
 505         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 506
 507         spin_lock(&rb->w_lock);
 508         if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) {
 509                 spin_unlock(&rb->w_lock);
 510                 return 0;
 511         }
 512
 513         if (!pblk_rb_may_write(rb, nr_entries, pos)) {
 514                 spin_unlock(&rb->w_lock);
 515                 return 0;
 516         }
 517
 518         pblk_rl_gc_in(&pblk->rl, nr_entries);
 519         spin_unlock(&rb->w_lock);
 520
 521         return 1;
 522 }
 523
 524 /*
 525  * Read available entries on rb and add them to the given bio. To avoid a memory
 526  * copy, a page reference to the write buffer is used to be added to the bio.
 527  *
 528  * This function is used by the write thread to form the write bio that will
 529  * persist data on the write buffer to the media.
 530  */
 531 unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
 532                                  unsigned int pos, unsigned int nr_entries,
 533                                  unsigned int count)
 534 {
 535         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 536         struct request_queue *q = pblk->dev->q;
 537         struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
 538         struct bio *bio = rqd->bio;
 539         struct pblk_rb_entry *entry;
 540         struct page *page;
 541         unsigned int pad = 0, to_read = nr_entries;
 542         unsigned int i;
 543         int flags;
 544
 545         if (count < nr_entries) {
 546                 pad = nr_entries - count;
 547                 to_read = count;
 548         }
 549
 550         c_ctx->sentry = pos;
 551         c_ctx->nr_valid = to_read;
 552         c_ctx->nr_padded = pad;
 553
 554         for (i = 0; i < to_read; i++) {
 555                 entry = &rb->entries[pos];
 556
 557                 /* A write has been allowed into the buffer, but data is still
 558                  * being copied to it. It is ok to busy wait.
 559                  */
 560 try:
 561                 flags = READ_ONCE(entry->w_ctx.flags);
 562                 if (!(flags & PBLK_WRITTEN_DATA)) {
 563                         io_schedule();
 564                         goto try;
 565                 }
 566
 567                 page = virt_to_page(entry->data);
 568                 if (!page) {
 569                         pblk_err(pblk, "could not allocate write bio page\n");
 570                         flags &= ~PBLK_WRITTEN_DATA;
 571                         flags |= PBLK_SUBMITTED_ENTRY;
 572                         /* Release flags on context. Protect from writes */
 573                         smp_store_release(&entry->w_ctx.flags, flags);
 574                         return NVM_IO_ERR;
 575                 }
 576
 577                 if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) !=
 578                                                                 rb->seg_size) {
 579                         pblk_err(pblk, "could not add page to write bio\n");
 580                         flags &= ~PBLK_WRITTEN_DATA;
 581                         flags |= PBLK_SUBMITTED_ENTRY;
 582                         /* Release flags on context. Protect from writes */
 583                         smp_store_release(&entry->w_ctx.flags, flags);
 584                         return NVM_IO_ERR;
 585                 }
 586
 587                 flags &= ~PBLK_WRITTEN_DATA;
 588                 flags |= PBLK_SUBMITTED_ENTRY;
 589
 590                 /* Release flags on context. Protect from writes */
 591                 smp_store_release(&entry->w_ctx.flags, flags);
 592
 593                 pos = pblk_rb_ptr_wrap(rb, pos, 1);
 594         }
 595
 596         if (pad) {
 597                 if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) {
 598                         pblk_err(pblk, "could not pad page in write bio\n");
 599                         return NVM_IO_ERR;
 600                 }
 601
 602                 if (pad < pblk->min_write_pgs)
 603                         atomic64_inc(&pblk->pad_dist[pad - 1]);
 604                 else
 605                         pblk_warn(pblk, "padding more than min. sectors\n");
 606
 607                 atomic64_add(pad, &pblk->pad_wa);
 608         }
 609
 610 #ifdef CONFIG_NVM_PBLK_DEBUG
 611         atomic_long_add(pad, &pblk->padded_writes);
 612 #endif
 613
 614         return NVM_IO_OK;
 615 }
 616
 617 /*
 618  * Copy to bio only if the lba matches the one on the given cache entry.
 619  * Otherwise, it means that the entry has been overwritten, and the bio should
 620  * be directed to disk.
 621  */
 622 int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba,
 623                         struct ppa_addr ppa, int bio_iter, bool advanced_bio)
 624 {
 625         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 626         struct pblk_rb_entry *entry;
 627         struct pblk_w_ctx *w_ctx;
 628         struct ppa_addr l2p_ppa;
 629         u64 pos = pblk_addr_to_cacheline(ppa);
 630         void *data;
 631         int flags;
 632         int ret = 1;
 633
 634
 635 #ifdef CONFIG_NVM_PBLK_DEBUG
 636         /* Caller must ensure that the access will not cause an overflow */
 637         BUG_ON(pos >= rb->nr_entries);
 638 #endif
 639         entry = &rb->entries[pos];
 640         w_ctx = &entry->w_ctx;
 641         flags = READ_ONCE(w_ctx->flags);
 642
 643         spin_lock(&rb->w_lock);
 644         spin_lock(&pblk->trans_lock);
 645         l2p_ppa = pblk_trans_map_get(pblk, lba);
 646         spin_unlock(&pblk->trans_lock);
 647
 648         /* Check if the entry has been overwritten or is scheduled to be */
 649         if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba ||
 650                                                 flags & PBLK_WRITABLE_ENTRY) {
 651                 ret = 0;
 652                 goto out;
 653         }
 654
 655         /* Only advance the bio if it hasn't been advanced already. If advanced,
 656          * this bio is at least a partial bio (i.e., it has partially been
 657          * filled with data from the cache). If part of the data resides on the
 658          * media, we will read later on
 659          */
 660         if (unlikely(!advanced_bio))
 661                 bio_advance(bio, bio_iter * PBLK_EXPOSED_PAGE_SIZE);
 662
 663         data = bio_data(bio);
 664         memcpy(data, entry->data, rb->seg_size);
 665
 666 out:
 667         spin_unlock(&rb->w_lock);
 668         return ret;
 669 }
 670
 671 struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos)
 672 {
 673         unsigned int entry = pblk_rb_ptr_wrap(rb, pos, 0);
 674
 675         return &rb->entries[entry].w_ctx;
 676 }
 677
 678 unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags)
 679         __acquires(&rb->s_lock)
 680 {
 681         if (flags)
 682                 spin_lock_irqsave(&rb->s_lock, *flags);
 683         else
 684                 spin_lock_irq(&rb->s_lock);
 685
 686         return rb->sync;
 687 }
 688
 689 void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
 690         __releases(&rb->s_lock)
 691 {
 692         lockdep_assert_held(&rb->s_lock);
 693
 694         if (flags)
 695                 spin_unlock_irqrestore(&rb->s_lock, *flags);
 696         else
 697                 spin_unlock_irq(&rb->s_lock);
 698 }
 699
 700 unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
 701 {
 702         unsigned int sync, flush_point;
 703         lockdep_assert_held(&rb->s_lock);
 704
 705         sync = READ_ONCE(rb->sync);
 706         flush_point = READ_ONCE(rb->flush_point);
 707
 708         if (flush_point != EMPTY_ENTRY) {
 709                 unsigned int secs_to_flush;
 710
 711                 secs_to_flush = pblk_rb_ring_count(flush_point, sync,
 712                                         rb->nr_entries);
 713                 if (secs_to_flush < nr_entries) {
 714                         /* Protect flush points */
 715                         smp_store_release(&rb->flush_point, EMPTY_ENTRY);
 716                 }
 717         }
 718
 719         sync = pblk_rb_ptr_wrap(rb, sync, nr_entries);
 720
 721         /* Protect from counts */
 722         smp_store_release(&rb->sync, sync);
 723
 724         return sync;
 725 }
 726
 727 /* Calculate how many sectors to submit up to the current flush point. */
 728 unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
 729 {
 730         unsigned int subm, sync, flush_point;
 731         unsigned int submitted, to_flush;
 732
 733         /* Protect flush points */
 734         flush_point = smp_load_acquire(&rb->flush_point);
 735         if (flush_point == EMPTY_ENTRY)
 736                 return 0;
 737
 738         /* Protect syncs */
 739         sync = smp_load_acquire(&rb->sync);
 740
 741         subm = READ_ONCE(rb->subm);
 742         submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
 743
 744         /* The sync point itself counts as a sector to sync */
 745         to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
 746
 747         return (submitted < to_flush) ? (to_flush - submitted) : 0;
 748 }
 749
 750 int pblk_rb_tear_down_check(struct pblk_rb *rb)
 751 {
 752         struct pblk_rb_entry *entry;
 753         int i;
 754         int ret = 0;
 755
 756         spin_lock(&rb->w_lock);
 757         spin_lock_irq(&rb->s_lock);
 758
 759         if ((rb->mem == rb->subm) && (rb->subm == rb->sync) &&
 760                                 (rb->sync == rb->l2p_update) &&
 761                                 (rb->flush_point == EMPTY_ENTRY)) {
 762                 goto out;
 763         }
 764
 765         if (!rb->entries) {
 766                 ret = 1;
 767                 goto out;
 768         }
 769
 770         for (i = 0; i < rb->nr_entries; i++) {
 771                 entry = &rb->entries[i];
 772
 773                 if (!entry->data) {
 774                         ret = 1;
 775                         goto out;
 776                 }
 777         }
 778
 779 out:
 780         spin_unlock(&rb->w_lock);
 781         spin_unlock_irq(&rb->s_lock);
 782
 783         return ret;
 784 }
 785
 786 unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos)
 787 {
 788         return (pos & (rb->nr_entries - 1));
 789 }
 790
 791 int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos)
 792 {
 793         return (pos >= rb->nr_entries);
 794 }
 795
 796 ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
 797 {
 798         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 799         struct pblk_c_ctx *c;
 800         ssize_t offset;
 801         int queued_entries = 0;
 802
 803         spin_lock_irq(&rb->s_lock);
 804         list_for_each_entry(c, &pblk->compl_list, list)
 805                 queued_entries++;
 806         spin_unlock_irq(&rb->s_lock);
 807
 808         if (rb->flush_point != EMPTY_ENTRY)
 809                 offset = scnprintf(buf, PAGE_SIZE,
 810                         "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
 811                         rb->nr_entries,
 812                         rb->mem,
 813                         rb->subm,
 814                         rb->sync,
 815                         rb->l2p_update,
 816 #ifdef CONFIG_NVM_PBLK_DEBUG
 817                         atomic_read(&rb->inflight_flush_point),
 818 #else
 819                         0,
 820 #endif
 821                         rb->flush_point,
 822                         pblk_rb_read_count(rb),
 823                         pblk_rb_space(rb),
 824                         pblk_rb_flush_point_count(rb),
 825                         queued_entries);
 826         else
 827                 offset = scnprintf(buf, PAGE_SIZE,
 828                         "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n",
 829                         rb->nr_entries,
 830                         rb->mem,
 831                         rb->subm,
 832                         rb->sync,
 833                         rb->l2p_update,
 834 #ifdef CONFIG_NVM_PBLK_DEBUG
 835                         atomic_read(&rb->inflight_flush_point),
 836 #else
 837                         0,
 838 #endif
 839                         pblk_rb_read_count(rb),
 840                         pblk_rb_space(rb),
 841                         pblk_rb_flush_point_count(rb),
 842                         queued_entries);
 843
 844         return offset;
 845 }