1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2016 CNEX Labs
4 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
6 * Based upon the circular ringbuffer.
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License version
10 * 2 as published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
17 * pblk-rb.c - pblk's write buffer
20 #include <linux/circ_buf.h>
24 static DECLARE_RWSEM(pblk_rb_lock);
26 void pblk_rb_data_free(struct pblk_rb *rb)
28 struct pblk_rb_pages *p, *t;
30 down_write(&pblk_rb_lock);
31 list_for_each_entry_safe(p, t, &rb->pages, list) {
32 free_pages((unsigned long)page_address(p->pages), p->order);
36 up_write(&pblk_rb_lock);
40 * Initialize ring buffer. The data and metadata buffers must be previously
41 * allocated and their size must be a power of two
42 * (Documentation/core-api/circular-buffers.rst)
44 int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base,
45 unsigned int power_size, unsigned int power_seg_sz)
47 struct pblk *pblk = container_of(rb, struct pblk, rwb);
48 unsigned int init_entry = 0;
49 unsigned int alloc_order = power_size;
50 unsigned int max_order = MAX_ORDER - 1;
51 unsigned int order, iter;
53 down_write(&pblk_rb_lock);
54 rb->entries = rb_entry_base;
55 rb->seg_size = (1 << power_seg_sz);
56 rb->nr_entries = (1 << power_size);
57 rb->mem = rb->subm = rb->sync = rb->l2p_update = 0;
58 rb->flush_point = EMPTY_ENTRY;
60 spin_lock_init(&rb->w_lock);
61 spin_lock_init(&rb->s_lock);
63 INIT_LIST_HEAD(&rb->pages);
65 if (alloc_order >= max_order) {
67 iter = (1 << (alloc_order - max_order));
74 struct pblk_rb_entry *entry;
75 struct pblk_rb_pages *page_set;
77 unsigned long set_size;
80 page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL);
82 up_write(&pblk_rb_lock);
86 page_set->order = order;
87 page_set->pages = alloc_pages(GFP_KERNEL, order);
88 if (!page_set->pages) {
90 pblk_rb_data_free(rb);
91 up_write(&pblk_rb_lock);
94 kaddr = page_address(page_set->pages);
96 entry = &rb->entries[init_entry];
98 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
99 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
101 set_size = (1 << order);
102 for (i = 1; i < set_size; i++) {
103 entry = &rb->entries[init_entry];
104 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
105 entry->data = kaddr + (i * rb->seg_size);
106 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
107 bio_list_init(&entry->w_ctx.bios);
110 list_add_tail(&page_set->list, &rb->pages);
113 up_write(&pblk_rb_lock);
115 #ifdef CONFIG_NVM_PBLK_DEBUG
116 atomic_set(&rb->inflight_flush_point, 0);
120 * Initialize rate-limiter, which controls access to the write buffer
121 * but user and GC I/O
123 pblk_rl_init(&pblk->rl, rb->nr_entries);
129 * pblk_rb_calculate_size -- calculate the size of the write buffer
131 unsigned int pblk_rb_calculate_size(unsigned int nr_entries)
133 /* Alloc a write buffer that can at least fit 128 entries */
134 return (1 << max(get_count_order(nr_entries), 7));
137 void *pblk_rb_entries_ref(struct pblk_rb *rb)
142 static void clean_wctx(struct pblk_w_ctx *w_ctx)
146 flags = READ_ONCE(w_ctx->flags);
147 WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY),
148 "pblk: overwriting unsubmitted data\n");
150 /* Release flags on context. Protect from writes and reads */
151 smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY);
152 pblk_ppa_set_empty(&w_ctx->ppa);
153 w_ctx->lba = ADDR_EMPTY;
156 #define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size)
157 #define pblk_rb_ring_space(rb, head, tail, size) \
158 (CIRC_SPACE(head, tail, size))
161 * Buffer space is calculated with respect to the back pointer signaling
162 * synchronized entries to the media.
164 static unsigned int pblk_rb_space(struct pblk_rb *rb)
166 unsigned int mem = READ_ONCE(rb->mem);
167 unsigned int sync = READ_ONCE(rb->sync);
169 return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries);
173 * Buffer count is calculated with respect to the submission entry signaling the
174 * entries that are available to send to the media
176 unsigned int pblk_rb_read_count(struct pblk_rb *rb)
178 unsigned int mem = READ_ONCE(rb->mem);
179 unsigned int subm = READ_ONCE(rb->subm);
181 return pblk_rb_ring_count(mem, subm, rb->nr_entries);
184 unsigned int pblk_rb_sync_count(struct pblk_rb *rb)
186 unsigned int mem = READ_ONCE(rb->mem);
187 unsigned int sync = READ_ONCE(rb->sync);
189 return pblk_rb_ring_count(mem, sync, rb->nr_entries);
192 unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries)
196 subm = READ_ONCE(rb->subm);
197 /* Commit read means updating submission pointer */
198 smp_store_release(&rb->subm,
199 (subm + nr_entries) & (rb->nr_entries - 1));
204 static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update)
206 struct pblk *pblk = container_of(rb, struct pblk, rwb);
207 struct pblk_line *line;
208 struct pblk_rb_entry *entry;
209 struct pblk_w_ctx *w_ctx;
210 unsigned int user_io = 0, gc_io = 0;
214 for (i = 0; i < to_update; i++) {
215 entry = &rb->entries[rb->l2p_update];
216 w_ctx = &entry->w_ctx;
218 flags = READ_ONCE(entry->w_ctx.flags);
219 if (flags & PBLK_IOTYPE_USER)
221 else if (flags & PBLK_IOTYPE_GC)
224 WARN(1, "pblk: unknown IO type\n");
226 pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa,
229 line = pblk_ppa_to_line(pblk, w_ctx->ppa);
230 kref_put(&line->ref, pblk_line_put);
232 rb->l2p_update = (rb->l2p_update + 1) & (rb->nr_entries - 1);
235 pblk_rl_out(&pblk->rl, user_io, gc_io);
241 * When we move the l2p_update pointer, we update the l2p table - lookups will
242 * point to the physical address instead of to the cacheline in the write buffer
243 * from this moment on.
245 static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries,
246 unsigned int mem, unsigned int sync)
248 unsigned int space, count;
251 lockdep_assert_held(&rb->w_lock);
253 /* Update l2p only as buffer entries are being overwritten */
254 space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries);
255 if (space > nr_entries)
258 count = nr_entries - space;
259 /* l2p_update used exclusively under rb->w_lock */
260 ret = __pblk_rb_update_l2p(rb, count);
267 * Update the l2p entry for all sectors stored on the write buffer. This means
268 * that all future lookups to the l2p table will point to a device address, not
269 * to the cacheline in the write buffer.
271 void pblk_rb_sync_l2p(struct pblk_rb *rb)
274 unsigned int to_update;
276 spin_lock(&rb->w_lock);
278 /* Protect from reads and writes */
279 sync = smp_load_acquire(&rb->sync);
281 to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries);
282 __pblk_rb_update_l2p(rb, to_update);
284 spin_unlock(&rb->w_lock);
288 * Write @nr_entries to ring buffer from @data buffer if there is enough space.
289 * Typically, 4KB data chunks coming from a bio will be copied to the ring
290 * buffer, thus the write will fail if not all incoming data can be copied.
293 static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data,
294 struct pblk_w_ctx w_ctx,
295 struct pblk_rb_entry *entry)
297 memcpy(entry->data, data, rb->seg_size);
299 entry->w_ctx.lba = w_ctx.lba;
300 entry->w_ctx.ppa = w_ctx.ppa;
303 void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data,
304 struct pblk_w_ctx w_ctx, unsigned int ring_pos)
306 struct pblk *pblk = container_of(rb, struct pblk, rwb);
307 struct pblk_rb_entry *entry;
310 entry = &rb->entries[ring_pos];
311 flags = READ_ONCE(entry->w_ctx.flags);
312 #ifdef CONFIG_NVM_PBLK_DEBUG
313 /* Caller must guarantee that the entry is free */
314 BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
317 __pblk_rb_write_entry(rb, data, w_ctx, entry);
319 pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline);
320 flags = w_ctx.flags | PBLK_WRITTEN_DATA;
322 /* Release flags on write context. Protect from writes */
323 smp_store_release(&entry->w_ctx.flags, flags);
326 void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
327 struct pblk_w_ctx w_ctx, struct pblk_line *line,
328 u64 paddr, unsigned int ring_pos)
330 struct pblk *pblk = container_of(rb, struct pblk, rwb);
331 struct pblk_rb_entry *entry;
334 entry = &rb->entries[ring_pos];
335 flags = READ_ONCE(entry->w_ctx.flags);
336 #ifdef CONFIG_NVM_PBLK_DEBUG
337 /* Caller must guarantee that the entry is free */
338 BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
341 __pblk_rb_write_entry(rb, data, w_ctx, entry);
343 if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr))
344 entry->w_ctx.lba = ADDR_EMPTY;
346 flags = w_ctx.flags | PBLK_WRITTEN_DATA;
348 /* Release flags on write context. Protect from writes */
349 smp_store_release(&entry->w_ctx.flags, flags);
352 static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
355 struct pblk_rb_entry *entry;
356 unsigned int sync, flush_point;
358 pblk_rb_sync_init(rb, NULL);
359 sync = READ_ONCE(rb->sync);
362 pblk_rb_sync_end(rb, NULL);
366 #ifdef CONFIG_NVM_PBLK_DEBUG
367 atomic_inc(&rb->inflight_flush_point);
370 flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
371 entry = &rb->entries[flush_point];
373 /* Protect flush points */
374 smp_store_release(&rb->flush_point, flush_point);
377 bio_list_add(&entry->w_ctx.bios, bio);
379 pblk_rb_sync_end(rb, NULL);
384 static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
390 sync = READ_ONCE(rb->sync);
391 mem = READ_ONCE(rb->mem);
393 if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < nr_entries)
396 if (pblk_rb_update_l2p(rb, nr_entries, mem, sync))
404 static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
407 if (!__pblk_rb_may_write(rb, nr_entries, pos))
410 /* Protect from read count */
411 smp_store_release(&rb->mem, (*pos + nr_entries) & (rb->nr_entries - 1));
415 void pblk_rb_flush(struct pblk_rb *rb)
417 struct pblk *pblk = container_of(rb, struct pblk, rwb);
418 unsigned int mem = READ_ONCE(rb->mem);
420 if (pblk_rb_flush_point_set(rb, NULL, mem))
423 pblk_write_kick(pblk);
426 static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries,
427 unsigned int *pos, struct bio *bio,
432 if (!__pblk_rb_may_write(rb, nr_entries, pos))
435 mem = (*pos + nr_entries) & (rb->nr_entries - 1);
436 *io_ret = NVM_IO_DONE;
438 if (bio->bi_opf & REQ_PREFLUSH) {
439 struct pblk *pblk = container_of(rb, struct pblk, rwb);
441 atomic64_inc(&pblk->nr_flush);
442 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem))
446 /* Protect from read count */
447 smp_store_release(&rb->mem, mem);
453 * Atomically check that (i) there is space on the write buffer for the
454 * incoming I/O, and (ii) the current I/O type has enough budget in the write
455 * buffer (rate-limiter).
457 int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio,
458 unsigned int nr_entries, unsigned int *pos)
460 struct pblk *pblk = container_of(rb, struct pblk, rwb);
463 spin_lock(&rb->w_lock);
464 io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries);
466 spin_unlock(&rb->w_lock);
470 if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) {
471 spin_unlock(&rb->w_lock);
472 return NVM_IO_REQUEUE;
475 pblk_rl_user_in(&pblk->rl, nr_entries);
476 spin_unlock(&rb->w_lock);
482 * Look at pblk_rb_may_write_user comment
484 int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries,
487 struct pblk *pblk = container_of(rb, struct pblk, rwb);
489 spin_lock(&rb->w_lock);
490 if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) {
491 spin_unlock(&rb->w_lock);
495 if (!pblk_rb_may_write(rb, nr_entries, pos)) {
496 spin_unlock(&rb->w_lock);
500 pblk_rl_gc_in(&pblk->rl, nr_entries);
501 spin_unlock(&rb->w_lock);
507 * Read available entries on rb and add them to the given bio. To avoid a memory
508 * copy, a page reference to the write buffer is used to be added to the bio.
510 * This function is used by the write thread to form the write bio that will
511 * persist data on the write buffer to the media.
513 unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
514 unsigned int pos, unsigned int nr_entries,
517 struct pblk *pblk = container_of(rb, struct pblk, rwb);
518 struct request_queue *q = pblk->dev->q;
519 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
520 struct bio *bio = rqd->bio;
521 struct pblk_rb_entry *entry;
523 unsigned int pad = 0, to_read = nr_entries;
527 if (count < nr_entries) {
528 pad = nr_entries - count;
533 c_ctx->nr_valid = to_read;
534 c_ctx->nr_padded = pad;
536 for (i = 0; i < to_read; i++) {
537 entry = &rb->entries[pos];
539 /* A write has been allowed into the buffer, but data is still
540 * being copied to it. It is ok to busy wait.
543 flags = READ_ONCE(entry->w_ctx.flags);
544 if (!(flags & PBLK_WRITTEN_DATA)) {
549 page = virt_to_page(entry->data);
551 pblk_err(pblk, "could not allocate write bio page\n");
552 flags &= ~PBLK_WRITTEN_DATA;
553 flags |= PBLK_SUBMITTED_ENTRY;
554 /* Release flags on context. Protect from writes */
555 smp_store_release(&entry->w_ctx.flags, flags);
559 if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) !=
561 pblk_err(pblk, "could not add page to write bio\n");
562 flags &= ~PBLK_WRITTEN_DATA;
563 flags |= PBLK_SUBMITTED_ENTRY;
564 /* Release flags on context. Protect from writes */
565 smp_store_release(&entry->w_ctx.flags, flags);
569 flags &= ~PBLK_WRITTEN_DATA;
570 flags |= PBLK_SUBMITTED_ENTRY;
572 /* Release flags on context. Protect from writes */
573 smp_store_release(&entry->w_ctx.flags, flags);
575 pos = (pos + 1) & (rb->nr_entries - 1);
579 if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) {
580 pblk_err(pblk, "could not pad page in write bio\n");
584 if (pad < pblk->min_write_pgs)
585 atomic64_inc(&pblk->pad_dist[pad - 1]);
587 pblk_warn(pblk, "padding more than min. sectors\n");
589 atomic64_add(pad, &pblk->pad_wa);
592 #ifdef CONFIG_NVM_PBLK_DEBUG
593 atomic_long_add(pad, &pblk->padded_writes);
600 * Copy to bio only if the lba matches the one on the given cache entry.
601 * Otherwise, it means that the entry has been overwritten, and the bio should
602 * be directed to disk.
604 int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba,
605 struct ppa_addr ppa, int bio_iter, bool advanced_bio)
607 struct pblk *pblk = container_of(rb, struct pblk, rwb);
608 struct pblk_rb_entry *entry;
609 struct pblk_w_ctx *w_ctx;
610 struct ppa_addr l2p_ppa;
611 u64 pos = pblk_addr_to_cacheline(ppa);
617 #ifdef CONFIG_NVM_PBLK_DEBUG
618 /* Caller must ensure that the access will not cause an overflow */
619 BUG_ON(pos >= rb->nr_entries);
621 entry = &rb->entries[pos];
622 w_ctx = &entry->w_ctx;
623 flags = READ_ONCE(w_ctx->flags);
625 spin_lock(&rb->w_lock);
626 spin_lock(&pblk->trans_lock);
627 l2p_ppa = pblk_trans_map_get(pblk, lba);
628 spin_unlock(&pblk->trans_lock);
630 /* Check if the entry has been overwritten or is scheduled to be */
631 if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba ||
632 flags & PBLK_WRITABLE_ENTRY) {
637 /* Only advance the bio if it hasn't been advanced already. If advanced,
638 * this bio is at least a partial bio (i.e., it has partially been
639 * filled with data from the cache). If part of the data resides on the
640 * media, we will read later on
642 if (unlikely(!advanced_bio))
643 bio_advance(bio, bio_iter * PBLK_EXPOSED_PAGE_SIZE);
645 data = bio_data(bio);
646 memcpy(data, entry->data, rb->seg_size);
649 spin_unlock(&rb->w_lock);
653 struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos)
655 unsigned int entry = pos & (rb->nr_entries - 1);
657 return &rb->entries[entry].w_ctx;
660 unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags)
661 __acquires(&rb->s_lock)
664 spin_lock_irqsave(&rb->s_lock, *flags);
666 spin_lock_irq(&rb->s_lock);
671 void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
672 __releases(&rb->s_lock)
674 lockdep_assert_held(&rb->s_lock);
677 spin_unlock_irqrestore(&rb->s_lock, *flags);
679 spin_unlock_irq(&rb->s_lock);
682 unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
684 unsigned int sync, flush_point;
685 lockdep_assert_held(&rb->s_lock);
687 sync = READ_ONCE(rb->sync);
688 flush_point = READ_ONCE(rb->flush_point);
690 if (flush_point != EMPTY_ENTRY) {
691 unsigned int secs_to_flush;
693 secs_to_flush = pblk_rb_ring_count(flush_point, sync,
695 if (secs_to_flush < nr_entries) {
696 /* Protect flush points */
697 smp_store_release(&rb->flush_point, EMPTY_ENTRY);
701 sync = (sync + nr_entries) & (rb->nr_entries - 1);
703 /* Protect from counts */
704 smp_store_release(&rb->sync, sync);
709 /* Calculate how many sectors to submit up to the current flush point. */
710 unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
712 unsigned int subm, sync, flush_point;
713 unsigned int submitted, to_flush;
715 /* Protect flush points */
716 flush_point = smp_load_acquire(&rb->flush_point);
717 if (flush_point == EMPTY_ENTRY)
721 sync = smp_load_acquire(&rb->sync);
723 subm = READ_ONCE(rb->subm);
724 submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
726 /* The sync point itself counts as a sector to sync */
727 to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
729 return (submitted < to_flush) ? (to_flush - submitted) : 0;
733 * Scan from the current position of the sync pointer to find the entry that
734 * corresponds to the given ppa. This is necessary since write requests can be
735 * completed out of order. The assumption is that the ppa is close to the sync
736 * pointer thus the search will not take long.
738 * The caller of this function must guarantee that the sync pointer will no
739 * reach the entry while it is using the metadata associated with it. With this
740 * assumption in mind, there is no need to take the sync lock.
742 struct pblk_rb_entry *pblk_rb_sync_scan_entry(struct pblk_rb *rb,
743 struct ppa_addr *ppa)
745 unsigned int sync, subm, count;
748 sync = READ_ONCE(rb->sync);
749 subm = READ_ONCE(rb->subm);
750 count = pblk_rb_ring_count(subm, sync, rb->nr_entries);
752 for (i = 0; i < count; i++)
753 sync = (sync + 1) & (rb->nr_entries - 1);
758 int pblk_rb_tear_down_check(struct pblk_rb *rb)
760 struct pblk_rb_entry *entry;
764 spin_lock(&rb->w_lock);
765 spin_lock_irq(&rb->s_lock);
767 if ((rb->mem == rb->subm) && (rb->subm == rb->sync) &&
768 (rb->sync == rb->l2p_update) &&
769 (rb->flush_point == EMPTY_ENTRY)) {
778 for (i = 0; i < rb->nr_entries; i++) {
779 entry = &rb->entries[i];
788 spin_unlock(&rb->w_lock);
789 spin_unlock_irq(&rb->s_lock);
794 unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos)
796 return (pos & (rb->nr_entries - 1));
799 int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos)
801 return (pos >= rb->nr_entries);
804 ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
806 struct pblk *pblk = container_of(rb, struct pblk, rwb);
807 struct pblk_c_ctx *c;
809 int queued_entries = 0;
811 spin_lock_irq(&rb->s_lock);
812 list_for_each_entry(c, &pblk->compl_list, list)
814 spin_unlock_irq(&rb->s_lock);
816 if (rb->flush_point != EMPTY_ENTRY)
817 offset = scnprintf(buf, PAGE_SIZE,
818 "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
824 #ifdef CONFIG_NVM_PBLK_DEBUG
825 atomic_read(&rb->inflight_flush_point),
830 pblk_rb_read_count(rb),
832 pblk_rb_flush_point_count(rb),
835 offset = scnprintf(buf, PAGE_SIZE,
836 "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n",
842 #ifdef CONFIG_NVM_PBLK_DEBUG
843 atomic_read(&rb->inflight_flush_point),
847 pblk_rb_read_count(rb),
849 pblk_rb_flush_point_count(rb),