1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2016 CNEX Labs
4 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
5 * Matias Bjorling <matias@cnexlabs.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * pblk-gc.c - pblk's garbage collector
20 #include "pblk-trace.h"
21 #include <linux/delay.h>
24 static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq)
31 static int pblk_gc_write(struct pblk *pblk)
33 struct pblk_gc *gc = &pblk->gc;
34 struct pblk_gc_rq *gc_rq, *tgc_rq;
37 spin_lock(&gc->w_lock);
38 if (list_empty(&gc->w_list)) {
39 spin_unlock(&gc->w_lock);
43 list_cut_position(&w_list, &gc->w_list, gc->w_list.prev);
45 spin_unlock(&gc->w_lock);
47 list_for_each_entry_safe(gc_rq, tgc_rq, &w_list, list) {
48 pblk_write_gc_to_cache(pblk, gc_rq);
49 list_del(&gc_rq->list);
50 kref_put(&gc_rq->line->ref, pblk_line_put);
51 pblk_gc_free_gc_rq(gc_rq);
57 static void pblk_gc_writer_kick(struct pblk_gc *gc)
59 wake_up_process(gc->gc_writer_ts);
62 static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
64 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
65 struct list_head *move_list;
67 spin_lock(&line->lock);
68 WARN_ON(line->state != PBLK_LINESTATE_GC);
69 line->state = PBLK_LINESTATE_CLOSED;
70 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
72 move_list = pblk_line_gc_list(pblk, line);
73 spin_unlock(&line->lock);
76 spin_lock(&l_mg->gc_lock);
77 list_add_tail(&line->list, move_list);
78 spin_unlock(&l_mg->gc_lock);
82 static void pblk_gc_line_ws(struct work_struct *work)
84 struct pblk_line_ws *gc_rq_ws = container_of(work,
85 struct pblk_line_ws, ws);
86 struct pblk *pblk = gc_rq_ws->pblk;
87 struct nvm_tgt_dev *dev = pblk->dev;
88 struct nvm_geo *geo = &dev->geo;
89 struct pblk_gc *gc = &pblk->gc;
90 struct pblk_line *line = gc_rq_ws->line;
91 struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
96 gc_rq->data = vmalloc(array_size(gc_rq->nr_secs, geo->csecs));
98 pblk_err(pblk, "could not GC line:%d (%d/%d)\n",
99 line->id, *line->vsc, gc_rq->nr_secs);
103 /* Read from GC victim block */
104 ret = pblk_submit_read_gc(pblk, gc_rq);
106 pblk_err(pblk, "failed GC read in line:%d (err:%d)\n",
111 if (!gc_rq->secs_to_gc)
115 spin_lock(&gc->w_lock);
116 if (gc->w_entries >= PBLK_GC_RQ_QD) {
117 spin_unlock(&gc->w_lock);
118 pblk_gc_writer_kick(&pblk->gc);
119 usleep_range(128, 256);
123 list_add_tail(&gc_rq->list, &gc->w_list);
124 spin_unlock(&gc->w_lock);
126 pblk_gc_writer_kick(&pblk->gc);
132 pblk_gc_free_gc_rq(gc_rq);
133 kref_put(&line->ref, pblk_line_put);
137 static __le64 *get_lba_list_from_emeta(struct pblk *pblk,
138 struct pblk_line *line)
140 struct line_emeta *emeta_buf;
141 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
142 struct pblk_line_meta *lm = &pblk->lm;
143 unsigned int lba_list_size = lm->emeta_len[2];
147 emeta_buf = pblk_malloc(lm->emeta_len[0],
148 l_mg->emeta_alloc_type, GFP_KERNEL);
152 ret = pblk_line_emeta_read(pblk, line, emeta_buf);
154 pblk_err(pblk, "line %d read emeta failed (%d)\n",
156 pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
160 /* If this read fails, it means that emeta is corrupted.
161 * For now, leave the line untouched.
162 * TODO: Implement a recovery routine that scans and moves
163 * all sectors on the line.
166 ret = pblk_recov_check_emeta(pblk, emeta_buf);
168 pblk_err(pblk, "inconsistent emeta (line %d)\n",
170 pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
174 lba_list = pblk_malloc(lba_list_size,
175 l_mg->emeta_alloc_type, GFP_KERNEL);
177 memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size);
179 pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
184 static void pblk_gc_line_prepare_ws(struct work_struct *work)
186 struct pblk_line_ws *line_ws = container_of(work, struct pblk_line_ws,
188 struct pblk *pblk = line_ws->pblk;
189 struct pblk_line *line = line_ws->line;
190 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
191 struct pblk_line_meta *lm = &pblk->lm;
192 struct pblk_gc *gc = &pblk->gc;
193 struct pblk_line_ws *gc_rq_ws;
194 struct pblk_gc_rq *gc_rq;
196 unsigned long *invalid_bitmap;
197 int sec_left, nr_secs, bit;
199 invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL);
203 if (line->w_err_gc->has_write_err) {
204 lba_list = line->w_err_gc->lba_list;
205 line->w_err_gc->lba_list = NULL;
207 lba_list = get_lba_list_from_emeta(pblk, line);
209 pblk_err(pblk, "could not interpret emeta (line %d)\n",
211 goto fail_free_invalid_bitmap;
215 spin_lock(&line->lock);
216 bitmap_copy(invalid_bitmap, line->invalid_bitmap, lm->sec_per_line);
217 sec_left = pblk_line_vsc(line);
218 spin_unlock(&line->lock);
221 pblk_err(pblk, "corrupted GC line (%d)\n", line->id);
222 goto fail_free_lba_list;
227 gc_rq = kmalloc(sizeof(struct pblk_gc_rq), GFP_KERNEL);
229 goto fail_free_lba_list;
233 bit = find_next_zero_bit(invalid_bitmap, lm->sec_per_line,
235 if (bit > line->emeta_ssec)
238 gc_rq->paddr_list[nr_secs] = bit;
239 gc_rq->lba_list[nr_secs++] = le64_to_cpu(lba_list[bit]);
240 } while (nr_secs < pblk->max_write_pgs);
242 if (unlikely(!nr_secs)) {
247 gc_rq->nr_secs = nr_secs;
250 gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
252 goto fail_free_gc_rq;
254 gc_rq_ws->pblk = pblk;
255 gc_rq_ws->line = line;
256 gc_rq_ws->priv = gc_rq;
258 /* The write GC path can be much slower than the read GC one due to
259 * the budget imposed by the rate-limiter. Balance in case that we get
260 * back pressure from the write GC path.
262 while (down_timeout(&gc->gc_sem, msecs_to_jiffies(30000)))
265 kref_get(&line->ref);
267 INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
268 queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
275 pblk_mfree(lba_list, l_mg->emeta_alloc_type);
277 kfree(invalid_bitmap);
279 kref_put(&line->ref, pblk_line_put);
280 atomic_dec(&gc->read_inflight_gc);
287 pblk_mfree(lba_list, l_mg->emeta_alloc_type);
288 fail_free_invalid_bitmap:
289 kfree(invalid_bitmap);
293 pblk_put_line_back(pblk, line);
294 kref_put(&line->ref, pblk_line_put);
295 atomic_dec(&gc->read_inflight_gc);
297 pblk_err(pblk, "failed to GC line %d\n", line->id);
300 static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
302 struct pblk_gc *gc = &pblk->gc;
303 struct pblk_line_ws *line_ws;
305 pblk_debug(pblk, "line '%d' being reclaimed for GC\n", line->id);
307 line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
311 line_ws->pblk = pblk;
312 line_ws->line = line;
314 atomic_inc(&gc->pipeline_gc);
315 INIT_WORK(&line_ws->ws, pblk_gc_line_prepare_ws);
316 queue_work(gc->gc_reader_wq, &line_ws->ws);
321 static void pblk_gc_reader_kick(struct pblk_gc *gc)
323 wake_up_process(gc->gc_reader_ts);
326 static void pblk_gc_kick(struct pblk *pblk)
328 struct pblk_gc *gc = &pblk->gc;
330 pblk_gc_writer_kick(gc);
331 pblk_gc_reader_kick(gc);
333 /* If we're shutting down GC, let's not start it up again */
334 if (gc->gc_enabled) {
335 wake_up_process(gc->gc_ts);
336 mod_timer(&gc->gc_timer,
337 jiffies + msecs_to_jiffies(GC_TIME_MSECS));
341 static int pblk_gc_read(struct pblk *pblk)
343 struct pblk_gc *gc = &pblk->gc;
344 struct pblk_line *line;
346 spin_lock(&gc->r_lock);
347 if (list_empty(&gc->r_list)) {
348 spin_unlock(&gc->r_lock);
352 line = list_first_entry(&gc->r_list, struct pblk_line, list);
353 list_del(&line->list);
354 spin_unlock(&gc->r_lock);
358 if (pblk_gc_line(pblk, line))
359 pblk_err(pblk, "failed to GC line %d\n", line->id);
364 static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk,
365 struct list_head *group_list)
367 struct pblk_line *line, *victim;
368 unsigned int line_vsc = ~0x0L, victim_vsc = ~0x0L;
370 victim = list_first_entry(group_list, struct pblk_line, list);
372 list_for_each_entry(line, group_list, list) {
373 if (!atomic_read(&line->sec_to_update))
374 line_vsc = le32_to_cpu(*line->vsc);
375 if (line_vsc < victim_vsc) {
377 victim_vsc = le32_to_cpu(*victim->vsc);
381 if (victim_vsc == ~0x0)
387 static bool pblk_gc_should_run(struct pblk_gc *gc, struct pblk_rl *rl)
389 unsigned int nr_blocks_free, nr_blocks_need;
390 unsigned int werr_lines = atomic_read(&rl->werr_lines);
392 nr_blocks_need = pblk_rl_high_thrs(rl);
393 nr_blocks_free = pblk_rl_nr_free_blks(rl);
395 /* This is not critical, no need to take lock here */
396 return ((werr_lines > 0) ||
397 ((gc->gc_active) && (nr_blocks_need > nr_blocks_free)));
400 void pblk_gc_free_full_lines(struct pblk *pblk)
402 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
403 struct pblk_gc *gc = &pblk->gc;
404 struct pblk_line *line;
407 spin_lock(&l_mg->gc_lock);
408 if (list_empty(&l_mg->gc_full_list)) {
409 spin_unlock(&l_mg->gc_lock);
413 line = list_first_entry(&l_mg->gc_full_list,
414 struct pblk_line, list);
416 spin_lock(&line->lock);
417 WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
418 line->state = PBLK_LINESTATE_GC;
419 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
421 spin_unlock(&line->lock);
423 list_del(&line->list);
424 spin_unlock(&l_mg->gc_lock);
426 atomic_inc(&gc->pipeline_gc);
427 kref_put(&line->ref, pblk_line_put);
432 * Lines with no valid sectors will be returned to the free list immediately. If
433 * GC is activated - either because the free block count is under the determined
434 * threshold, or because it is being forced from user space - only lines with a
435 * high count of invalid sectors will be recycled.
437 static void pblk_gc_run(struct pblk *pblk)
439 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
440 struct pblk_gc *gc = &pblk->gc;
441 struct pblk_line *line;
442 struct list_head *group_list;
444 int read_inflight_gc, gc_group = 0, prev_group = 0;
446 pblk_gc_free_full_lines(pblk);
448 run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
449 if (!run_gc || (atomic_read(&gc->read_inflight_gc) >= PBLK_GC_L_QD))
453 group_list = l_mg->gc_lists[gc_group++];
456 spin_lock(&l_mg->gc_lock);
458 line = pblk_gc_get_victim_line(pblk, group_list);
460 spin_unlock(&l_mg->gc_lock);
464 spin_lock(&line->lock);
465 WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
466 line->state = PBLK_LINESTATE_GC;
467 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
469 spin_unlock(&line->lock);
471 list_del(&line->list);
472 spin_unlock(&l_mg->gc_lock);
474 spin_lock(&gc->r_lock);
475 list_add_tail(&line->list, &gc->r_list);
476 spin_unlock(&gc->r_lock);
478 read_inflight_gc = atomic_inc_return(&gc->read_inflight_gc);
479 pblk_gc_reader_kick(gc);
483 /* No need to queue up more GC lines than we can handle */
484 run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
485 if (!run_gc || read_inflight_gc >= PBLK_GC_L_QD)
489 if (!prev_group && pblk->rl.rb_state > gc_group &&
490 gc_group < PBLK_GC_NR_LISTS)
494 static void pblk_gc_timer(struct timer_list *t)
496 struct pblk *pblk = from_timer(pblk, t, gc.gc_timer);
501 static int pblk_gc_ts(void *data)
503 struct pblk *pblk = data;
505 while (!kthread_should_stop()) {
507 set_current_state(TASK_INTERRUPTIBLE);
514 static int pblk_gc_writer_ts(void *data)
516 struct pblk *pblk = data;
518 while (!kthread_should_stop()) {
519 if (!pblk_gc_write(pblk))
521 set_current_state(TASK_INTERRUPTIBLE);
528 static int pblk_gc_reader_ts(void *data)
530 struct pblk *pblk = data;
531 struct pblk_gc *gc = &pblk->gc;
533 while (!kthread_should_stop()) {
534 if (!pblk_gc_read(pblk))
536 set_current_state(TASK_INTERRUPTIBLE);
540 #ifdef CONFIG_NVM_PBLK_DEBUG
541 pblk_info(pblk, "flushing gc pipeline, %d lines left\n",
542 atomic_read(&gc->pipeline_gc));
546 if (!atomic_read(&gc->pipeline_gc))
555 static void pblk_gc_start(struct pblk *pblk)
557 pblk->gc.gc_active = 1;
558 pblk_debug(pblk, "gc start\n");
561 void pblk_gc_should_start(struct pblk *pblk)
563 struct pblk_gc *gc = &pblk->gc;
565 if (gc->gc_enabled && !gc->gc_active) {
571 void pblk_gc_should_stop(struct pblk *pblk)
573 struct pblk_gc *gc = &pblk->gc;
575 if (gc->gc_active && !gc->gc_forced)
579 void pblk_gc_should_kick(struct pblk *pblk)
581 pblk_rl_update_rates(&pblk->rl);
584 void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled,
587 struct pblk_gc *gc = &pblk->gc;
589 spin_lock(&gc->lock);
590 *gc_enabled = gc->gc_enabled;
591 *gc_active = gc->gc_active;
592 spin_unlock(&gc->lock);
595 int pblk_gc_sysfs_force(struct pblk *pblk, int force)
597 struct pblk_gc *gc = &pblk->gc;
599 if (force < 0 || force > 1)
602 spin_lock(&gc->lock);
603 gc->gc_forced = force;
609 spin_unlock(&gc->lock);
611 pblk_gc_should_start(pblk);
616 int pblk_gc_init(struct pblk *pblk)
618 struct pblk_gc *gc = &pblk->gc;
621 gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts");
622 if (IS_ERR(gc->gc_ts)) {
623 pblk_err(pblk, "could not allocate GC main kthread\n");
624 return PTR_ERR(gc->gc_ts);
627 gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk,
628 "pblk-gc-writer-ts");
629 if (IS_ERR(gc->gc_writer_ts)) {
630 pblk_err(pblk, "could not allocate GC writer kthread\n");
631 ret = PTR_ERR(gc->gc_writer_ts);
632 goto fail_free_main_kthread;
635 gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk,
636 "pblk-gc-reader-ts");
637 if (IS_ERR(gc->gc_reader_ts)) {
638 pblk_err(pblk, "could not allocate GC reader kthread\n");
639 ret = PTR_ERR(gc->gc_reader_ts);
640 goto fail_free_writer_kthread;
643 timer_setup(&gc->gc_timer, pblk_gc_timer, 0);
644 mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS));
650 atomic_set(&gc->read_inflight_gc, 0);
651 atomic_set(&gc->pipeline_gc, 0);
653 /* Workqueue that reads valid sectors from a line and submit them to the
654 * GC writer to be recycled.
656 gc->gc_line_reader_wq = alloc_workqueue("pblk-gc-line-reader-wq",
657 WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_GC_MAX_READERS);
658 if (!gc->gc_line_reader_wq) {
659 pblk_err(pblk, "could not allocate GC line reader workqueue\n");
661 goto fail_free_reader_kthread;
664 /* Workqueue that prepare lines for GC */
665 gc->gc_reader_wq = alloc_workqueue("pblk-gc-line_wq",
666 WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
667 if (!gc->gc_reader_wq) {
668 pblk_err(pblk, "could not allocate GC reader workqueue\n");
670 goto fail_free_reader_line_wq;
673 spin_lock_init(&gc->lock);
674 spin_lock_init(&gc->w_lock);
675 spin_lock_init(&gc->r_lock);
677 sema_init(&gc->gc_sem, PBLK_GC_RQ_QD);
679 INIT_LIST_HEAD(&gc->w_list);
680 INIT_LIST_HEAD(&gc->r_list);
684 fail_free_reader_line_wq:
685 destroy_workqueue(gc->gc_line_reader_wq);
686 fail_free_reader_kthread:
687 kthread_stop(gc->gc_reader_ts);
688 fail_free_writer_kthread:
689 kthread_stop(gc->gc_writer_ts);
690 fail_free_main_kthread:
691 kthread_stop(gc->gc_ts);
696 void pblk_gc_exit(struct pblk *pblk, bool graceful)
698 struct pblk_gc *gc = &pblk->gc;
701 del_timer_sync(&gc->gc_timer);
705 kthread_stop(gc->gc_ts);
707 if (gc->gc_reader_ts)
708 kthread_stop(gc->gc_reader_ts);
711 flush_workqueue(gc->gc_reader_wq);
712 flush_workqueue(gc->gc_line_reader_wq);
715 destroy_workqueue(gc->gc_reader_wq);
716 destroy_workqueue(gc->gc_line_reader_wq);
718 if (gc->gc_writer_ts)
719 kthread_stop(gc->gc_writer_ts);