drivers/lightnvm/pblk-recovery.c

   1 /*
   2  * Copyright (C) 2016 CNEX Labs
   3  * Initial: Javier Gonzalez <javier@cnexlabs.com>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version
   7  * 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * pblk-recovery.c - pblk's recovery path
  15  */
  16
  17 #include "pblk.h"
  18 #include "pblk-trace.h"
  19
  20 int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta_buf)
  21 {
  22         u32 crc;
  23
  24         crc = pblk_calc_emeta_crc(pblk, emeta_buf);
  25         if (le32_to_cpu(emeta_buf->crc) != crc)
  26                 return 1;
  27
  28         if (le32_to_cpu(emeta_buf->header.identifier) != PBLK_MAGIC)
  29                 return 1;
  30
  31         return 0;
  32 }
  33
  34 static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line)
  35 {
  36         struct nvm_tgt_dev *dev = pblk->dev;
  37         struct nvm_geo *geo = &dev->geo;
  38         struct pblk_line_meta *lm = &pblk->lm;
  39         struct pblk_emeta *emeta = line->emeta;
  40         struct line_emeta *emeta_buf = emeta->buf;
  41         __le64 *lba_list;
  42         u64 data_start, data_end;
  43         u64 nr_valid_lbas, nr_lbas = 0;
  44         u64 i;
  45
  46         lba_list = emeta_to_lbas(pblk, emeta_buf);
  47         if (!lba_list)
  48                 return 1;
  49
  50         data_start = pblk_line_smeta_start(pblk, line) + lm->smeta_sec;
  51         data_end = line->emeta_ssec;
  52         nr_valid_lbas = le64_to_cpu(emeta_buf->nr_valid_lbas);
  53
  54         for (i = data_start; i < data_end; i++) {
  55                 struct ppa_addr ppa;
  56                 int pos;
  57
  58                 ppa = addr_to_gen_ppa(pblk, i, line->id);
  59                 pos = pblk_ppa_to_pos(geo, ppa);
  60
  61                 /* Do not update bad blocks */
  62                 if (test_bit(pos, line->blk_bitmap))
  63                         continue;
  64
  65                 if (le64_to_cpu(lba_list[i]) == ADDR_EMPTY) {
  66                         spin_lock(&line->lock);
  67                         if (test_and_set_bit(i, line->invalid_bitmap))
  68                                 WARN_ONCE(1, "pblk: rec. double invalidate:\n");
  69                         else
  70                                 le32_add_cpu(line->vsc, -1);
  71                         spin_unlock(&line->lock);
  72
  73                         continue;
  74                 }
  75
  76                 pblk_update_map(pblk, le64_to_cpu(lba_list[i]), ppa);
  77                 nr_lbas++;
  78         }
  79
  80         if (nr_valid_lbas != nr_lbas)
  81                 pblk_err(pblk, "line %d - inconsistent lba list(%llu/%llu)\n",
  82                                 line->id, nr_valid_lbas, nr_lbas);
  83
  84         line->left_msecs = 0;
  85
  86         return 0;
  87 }
  88
  89 static void pblk_update_line_wp(struct pblk *pblk, struct pblk_line *line,
  90                                 u64 written_secs)
  91 {
  92         int i;
  93
  94         for (i = 0; i < written_secs; i += pblk->min_write_pgs)
  95                 pblk_alloc_page(pblk, line, pblk->min_write_pgs);
  96 }
  97
  98 static u64 pblk_sec_in_open_line(struct pblk *pblk, struct pblk_line *line)
  99 {
 100         struct pblk_line_meta *lm = &pblk->lm;
 101         int nr_bb = bitmap_weight(line->blk_bitmap, lm->blk_per_line);
 102         u64 written_secs = 0;
 103         int valid_chunks = 0;
 104         int i;
 105
 106         for (i = 0; i < lm->blk_per_line; i++) {
 107                 struct nvm_chk_meta *chunk = &line->chks[i];
 108
 109                 if (chunk->state & NVM_CHK_ST_OFFLINE)
 110                         continue;
 111
 112                 written_secs += chunk->wp;
 113                 valid_chunks++;
 114         }
 115
 116         if (lm->blk_per_line - nr_bb != valid_chunks)
 117                 pblk_err(pblk, "recovery line %d is bad\n", line->id);
 118
 119         pblk_update_line_wp(pblk, line, written_secs - lm->smeta_sec);
 120
 121         return written_secs;
 122 }
 123
 124 struct pblk_recov_alloc {
 125         struct ppa_addr *ppa_list;
 126         struct pblk_sec_meta *meta_list;
 127         struct nvm_rq *rqd;
 128         void *data;
 129         dma_addr_t dma_ppa_list;
 130         dma_addr_t dma_meta_list;
 131 };
 132
 133 static void pblk_recov_complete(struct kref *ref)
 134 {
 135         struct pblk_pad_rq *pad_rq = container_of(ref, struct pblk_pad_rq, ref);
 136
 137         complete(&pad_rq->wait);
 138 }
 139
 140 static void pblk_end_io_recov(struct nvm_rq *rqd)
 141 {
 142         struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
 143         struct pblk_pad_rq *pad_rq = rqd->private;
 144         struct pblk *pblk = pad_rq->pblk;
 145
 146         pblk_up_chunk(pblk, ppa_list[0]);
 147
 148         pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);
 149
 150         atomic_dec(&pblk->inflight_io);
 151         kref_put(&pad_rq->ref, pblk_recov_complete);
 152 }
 153
 154 /* pad line using line bitmap.  */
 155 static int pblk_recov_pad_line(struct pblk *pblk, struct pblk_line *line,
 156                                int left_ppas)
 157 {
 158         struct nvm_tgt_dev *dev = pblk->dev;
 159         struct nvm_geo *geo = &dev->geo;
 160         struct pblk_sec_meta *meta_list;
 161         struct pblk_pad_rq *pad_rq;
 162         struct nvm_rq *rqd;
 163         struct bio *bio;
 164         void *data;
 165         __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf);
 166         u64 w_ptr = line->cur_sec;
 167         int left_line_ppas, rq_ppas, rq_len;
 168         int i, j;
 169         int ret = 0;
 170
 171         spin_lock(&line->lock);
 172         left_line_ppas = line->left_msecs;
 173         spin_unlock(&line->lock);
 174
 175         pad_rq = kmalloc(sizeof(struct pblk_pad_rq), GFP_KERNEL);
 176         if (!pad_rq)
 177                 return -ENOMEM;
 178
 179         data = vzalloc(array_size(pblk->max_write_pgs, geo->csecs));
 180         if (!data) {
 181                 ret = -ENOMEM;
 182                 goto free_rq;
 183         }
 184
 185         pad_rq->pblk = pblk;
 186         init_completion(&pad_rq->wait);
 187         kref_init(&pad_rq->ref);
 188
 189 next_pad_rq:
 190         rq_ppas = pblk_calc_secs(pblk, left_ppas, 0);
 191         if (rq_ppas < pblk->min_write_pgs) {
 192                 pblk_err(pblk, "corrupted pad line %d\n", line->id);
 193                 goto fail_free_pad;
 194         }
 195
 196         rq_len = rq_ppas * geo->csecs;
 197
 198         bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len,
 199                                                 PBLK_VMALLOC_META, GFP_KERNEL);
 200         if (IS_ERR(bio)) {
 201                 ret = PTR_ERR(bio);
 202                 goto fail_free_pad;
 203         }
 204
 205         bio->bi_iter.bi_sector = 0; /* internal bio */
 206         bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 207
 208         rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT);
 209
 210         ret = pblk_alloc_rqd_meta(pblk, rqd);
 211         if (ret)
 212                 goto fail_free_rqd;
 213
 214         rqd->bio = bio;
 215         rqd->opcode = NVM_OP_PWRITE;
 216         rqd->is_seq = 1;
 217         rqd->nr_ppas = rq_ppas;
 218         rqd->end_io = pblk_end_io_recov;
 219         rqd->private = pad_rq;
 220
 221         meta_list = rqd->meta_list;
 222
 223         for (i = 0; i < rqd->nr_ppas; ) {
 224                 struct ppa_addr ppa;
 225                 int pos;
 226
 227                 w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs);
 228                 ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
 229                 pos = pblk_ppa_to_pos(geo, ppa);
 230
 231                 while (test_bit(pos, line->blk_bitmap)) {
 232                         w_ptr += pblk->min_write_pgs;
 233                         ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
 234                         pos = pblk_ppa_to_pos(geo, ppa);
 235                 }
 236
 237                 for (j = 0; j < pblk->min_write_pgs; j++, i++, w_ptr++) {
 238                         struct ppa_addr dev_ppa;
 239                         __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
 240
 241                         dev_ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
 242
 243                         pblk_map_invalidate(pblk, dev_ppa);
 244                         lba_list[w_ptr] = meta_list[i].lba = addr_empty;
 245                         rqd->ppa_list[i] = dev_ppa;
 246                 }
 247         }
 248
 249         kref_get(&pad_rq->ref);
 250         pblk_down_chunk(pblk, rqd->ppa_list[0]);
 251
 252         ret = pblk_submit_io(pblk, rqd);
 253         if (ret) {
 254                 pblk_err(pblk, "I/O submission failed: %d\n", ret);
 255                 pblk_up_chunk(pblk, rqd->ppa_list[0]);
 256                 goto fail_free_rqd;
 257         }
 258
 259         left_line_ppas -= rq_ppas;
 260         left_ppas -= rq_ppas;
 261         if (left_ppas && left_line_ppas)
 262                 goto next_pad_rq;
 263
 264         kref_put(&pad_rq->ref, pblk_recov_complete);
 265
 266         if (!wait_for_completion_io_timeout(&pad_rq->wait,
 267                                 msecs_to_jiffies(PBLK_COMMAND_TIMEOUT_MS))) {
 268                 pblk_err(pblk, "pad write timed out\n");
 269                 ret = -ETIME;
 270         }
 271
 272         if (!pblk_line_is_full(line))
 273                 pblk_err(pblk, "corrupted padded line: %d\n", line->id);
 274
 275         vfree(data);
 276 free_rq:
 277         kfree(pad_rq);
 278         return ret;
 279
 280 fail_free_rqd:
 281         pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);
 282         bio_put(bio);
 283 fail_free_pad:
 284         kfree(pad_rq);
 285         vfree(data);
 286         return ret;
 287 }
 288
 289 static int pblk_pad_distance(struct pblk *pblk, struct pblk_line *line)
 290 {
 291         struct nvm_tgt_dev *dev = pblk->dev;
 292         struct nvm_geo *geo = &dev->geo;
 293         int distance = geo->mw_cunits * geo->all_luns * geo->ws_opt;
 294
 295         return (distance > line->left_msecs) ? line->left_msecs : distance;
 296 }
 297
 298 static int pblk_line_wp_is_unbalanced(struct pblk *pblk,
 299                                       struct pblk_line *line)
 300 {
 301         struct nvm_tgt_dev *dev = pblk->dev;
 302         struct nvm_geo *geo = &dev->geo;
 303         struct pblk_line_meta *lm = &pblk->lm;
 304         struct pblk_lun *rlun;
 305         struct nvm_chk_meta *chunk;
 306         struct ppa_addr ppa;
 307         u64 line_wp;
 308         int pos, i;
 309
 310         rlun = &pblk->luns[0];
 311         ppa = rlun->bppa;
 312         pos = pblk_ppa_to_pos(geo, ppa);
 313         chunk = &line->chks[pos];
 314
 315         line_wp = chunk->wp;
 316
 317         for (i = 1; i < lm->blk_per_line; i++) {
 318                 rlun = &pblk->luns[i];
 319                 ppa = rlun->bppa;
 320                 pos = pblk_ppa_to_pos(geo, ppa);
 321                 chunk = &line->chks[pos];
 322
 323                 if (chunk->wp > line_wp)
 324                         return 1;
 325                 else if (chunk->wp < line_wp)
 326                         line_wp = chunk->wp;
 327         }
 328
 329         return 0;
 330 }
 331
 332 static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line,
 333                                struct pblk_recov_alloc p)
 334 {
 335         struct nvm_tgt_dev *dev = pblk->dev;
 336         struct nvm_geo *geo = &dev->geo;
 337         struct ppa_addr *ppa_list;
 338         struct pblk_sec_meta *meta_list;
 339         struct nvm_rq *rqd;
 340         struct bio *bio;
 341         void *data;
 342         dma_addr_t dma_ppa_list, dma_meta_list;
 343         __le64 *lba_list;
 344         u64 paddr = 0;
 345         bool padded = false;
 346         int rq_ppas, rq_len;
 347         int i, j;
 348         int ret;
 349         u64 left_ppas = pblk_sec_in_open_line(pblk, line);
 350
 351         if (pblk_line_wp_is_unbalanced(pblk, line))
 352                 pblk_warn(pblk, "recovering unbalanced line (%d)\n", line->id);
 353
 354         ppa_list = p.ppa_list;
 355         meta_list = p.meta_list;
 356         rqd = p.rqd;
 357         data = p.data;
 358         dma_ppa_list = p.dma_ppa_list;
 359         dma_meta_list = p.dma_meta_list;
 360
 361         lba_list = emeta_to_lbas(pblk, line->emeta->buf);
 362
 363 next_rq:
 364         memset(rqd, 0, pblk_g_rq_size);
 365
 366         rq_ppas = pblk_calc_secs(pblk, left_ppas, 0);
 367         if (!rq_ppas)
 368                 rq_ppas = pblk->min_write_pgs;
 369         rq_len = rq_ppas * geo->csecs;
 370
 371         bio = bio_map_kern(dev->q, data, rq_len, GFP_KERNEL);
 372         if (IS_ERR(bio))
 373                 return PTR_ERR(bio);
 374
 375         bio->bi_iter.bi_sector = 0; /* internal bio */
 376         bio_set_op_attrs(bio, REQ_OP_READ, 0);
 377
 378         rqd->bio = bio;
 379         rqd->opcode = NVM_OP_PREAD;
 380         rqd->meta_list = meta_list;
 381         rqd->nr_ppas = rq_ppas;
 382         rqd->ppa_list = ppa_list;
 383         rqd->dma_ppa_list = dma_ppa_list;
 384         rqd->dma_meta_list = dma_meta_list;
 385
 386         if (pblk_io_aligned(pblk, rq_ppas))
 387                 rqd->is_seq = 1;
 388
 389 retry_rq:
 390         for (i = 0; i < rqd->nr_ppas; ) {
 391                 struct ppa_addr ppa;
 392                 int pos;
 393
 394                 ppa = addr_to_gen_ppa(pblk, paddr, line->id);
 395                 pos = pblk_ppa_to_pos(geo, ppa);
 396
 397                 while (test_bit(pos, line->blk_bitmap)) {
 398                         paddr += pblk->min_write_pgs;
 399                         ppa = addr_to_gen_ppa(pblk, paddr, line->id);
 400                         pos = pblk_ppa_to_pos(geo, ppa);
 401                 }
 402
 403                 for (j = 0; j < pblk->min_write_pgs; j++, i++)
 404                         rqd->ppa_list[i] =
 405                                 addr_to_gen_ppa(pblk, paddr + j, line->id);
 406         }
 407
 408         ret = pblk_submit_io_sync(pblk, rqd);
 409         if (ret) {
 410                 pblk_err(pblk, "I/O submission failed: %d\n", ret);
 411                 bio_put(bio);
 412                 return ret;
 413         }
 414
 415         atomic_dec(&pblk->inflight_io);
 416
 417         /* If a read fails, do a best effort by padding the line and retrying */
 418         if (rqd->error) {
 419                 int pad_distance, ret;
 420
 421                 if (padded) {
 422                         pblk_log_read_err(pblk, rqd);
 423                         return -EINTR;
 424                 }
 425
 426                 pad_distance = pblk_pad_distance(pblk, line);
 427                 ret = pblk_recov_pad_line(pblk, line, pad_distance);
 428                 if (ret)
 429                         return ret;
 430
 431                 padded = true;
 432                 goto retry_rq;
 433         }
 434
 435         for (i = 0; i < rqd->nr_ppas; i++) {
 436                 u64 lba = le64_to_cpu(meta_list[i].lba);
 437
 438                 lba_list[paddr++] = cpu_to_le64(lba);
 439
 440                 if (lba == ADDR_EMPTY || lba > pblk->rl.nr_secs)
 441                         continue;
 442
 443                 line->nr_valid_lbas++;
 444                 pblk_update_map(pblk, lba, rqd->ppa_list[i]);
 445         }
 446
 447         left_ppas -= rq_ppas;
 448         if (left_ppas > 0)
 449                 goto next_rq;
 450
 451 #ifdef CONFIG_NVM_PBLK_DEBUG
 452         WARN_ON(padded && !pblk_line_is_full(line));
 453 #endif
 454
 455         return 0;
 456 }
 457
 458 /* Scan line for lbas on out of bound area */
 459 static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line)
 460 {
 461         struct nvm_tgt_dev *dev = pblk->dev;
 462         struct nvm_geo *geo = &dev->geo;
 463         struct nvm_rq *rqd;
 464         struct ppa_addr *ppa_list;
 465         struct pblk_sec_meta *meta_list;
 466         struct pblk_recov_alloc p;
 467         void *data;
 468         dma_addr_t dma_ppa_list, dma_meta_list;
 469         int ret = 0;
 470
 471         meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list);
 472         if (!meta_list)
 473                 return -ENOMEM;
 474
 475         ppa_list = (void *)(meta_list) + pblk_dma_meta_size;
 476         dma_ppa_list = dma_meta_list + pblk_dma_meta_size;
 477
 478         data = kcalloc(pblk->max_write_pgs, geo->csecs, GFP_KERNEL);
 479         if (!data) {
 480                 ret = -ENOMEM;
 481                 goto free_meta_list;
 482         }
 483
 484         rqd = mempool_alloc(&pblk->r_rq_pool, GFP_KERNEL);
 485         memset(rqd, 0, pblk_g_rq_size);
 486
 487         p.ppa_list = ppa_list;
 488         p.meta_list = meta_list;
 489         p.rqd = rqd;
 490         p.data = data;
 491         p.dma_ppa_list = dma_ppa_list;
 492         p.dma_meta_list = dma_meta_list;
 493
 494         ret = pblk_recov_scan_oob(pblk, line, p);
 495         if (ret) {
 496                 pblk_err(pblk, "could not recover L2P form OOB\n");
 497                 goto out;
 498         }
 499
 500         if (pblk_line_is_full(line))
 501                 pblk_line_recov_close(pblk, line);
 502
 503 out:
 504         mempool_free(rqd, &pblk->r_rq_pool);
 505         kfree(data);
 506 free_meta_list:
 507         nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list);
 508
 509         return ret;
 510 }
 511
 512 /* Insert lines ordered by sequence number (seq_num) on list */
 513 static void pblk_recov_line_add_ordered(struct list_head *head,
 514                                         struct pblk_line *line)
 515 {
 516         struct pblk_line *t = NULL;
 517
 518         list_for_each_entry(t, head, list)
 519                 if (t->seq_nr > line->seq_nr)
 520                         break;
 521
 522         __list_add(&line->list, t->list.prev, &t->list);
 523 }
 524
 525 static u64 pblk_line_emeta_start(struct pblk *pblk, struct pblk_line *line)
 526 {
 527         struct nvm_tgt_dev *dev = pblk->dev;
 528         struct nvm_geo *geo = &dev->geo;
 529         struct pblk_line_meta *lm = &pblk->lm;
 530         unsigned int emeta_secs;
 531         u64 emeta_start;
 532         struct ppa_addr ppa;
 533         int pos;
 534
 535         emeta_secs = lm->emeta_sec[0];
 536         emeta_start = lm->sec_per_line;
 537
 538         while (emeta_secs) {
 539                 emeta_start--;
 540                 ppa = addr_to_gen_ppa(pblk, emeta_start, line->id);
 541                 pos = pblk_ppa_to_pos(geo, ppa);
 542                 if (!test_bit(pos, line->blk_bitmap))
 543                         emeta_secs--;
 544         }
 545
 546         return emeta_start;
 547 }
 548
 549 static int pblk_recov_check_line_version(struct pblk *pblk,
 550                                          struct line_emeta *emeta)
 551 {
 552         struct line_header *header = &emeta->header;
 553
 554         if (header->version_major != EMETA_VERSION_MAJOR) {
 555                 pblk_err(pblk, "line major version mismatch: %d, expected: %d\n",
 556                          header->version_major, EMETA_VERSION_MAJOR);
 557                 return 1;
 558         }
 559
 560 #ifdef CONFIG_NVM_PBLK_DEBUG
 561         if (header->version_minor > EMETA_VERSION_MINOR)
 562                 pblk_info(pblk, "newer line minor version found: %d\n",
 563                                 header->version_minor);
 564 #endif
 565
 566         return 0;
 567 }
 568
 569 static void pblk_recov_wa_counters(struct pblk *pblk,
 570                                    struct line_emeta *emeta)
 571 {
 572         struct pblk_line_meta *lm = &pblk->lm;
 573         struct line_header *header = &emeta->header;
 574         struct wa_counters *wa = emeta_to_wa(lm, emeta);
 575
 576         /* WA counters were introduced in emeta version 0.2 */
 577         if (header->version_major > 0 || header->version_minor >= 2) {
 578                 u64 user = le64_to_cpu(wa->user);
 579                 u64 pad = le64_to_cpu(wa->pad);
 580                 u64 gc = le64_to_cpu(wa->gc);
 581
 582                 atomic64_set(&pblk->user_wa, user);
 583                 atomic64_set(&pblk->pad_wa, pad);
 584                 atomic64_set(&pblk->gc_wa, gc);
 585
 586                 pblk->user_rst_wa = user;
 587                 pblk->pad_rst_wa = pad;
 588                 pblk->gc_rst_wa = gc;
 589         }
 590 }
 591
 592 static int pblk_line_was_written(struct pblk_line *line,
 593                                  struct pblk *pblk)
 594 {
 595
 596         struct pblk_line_meta *lm = &pblk->lm;
 597         struct nvm_tgt_dev *dev = pblk->dev;
 598         struct nvm_geo *geo = &dev->geo;
 599         struct nvm_chk_meta *chunk;
 600         struct ppa_addr bppa;
 601         int smeta_blk;
 602
 603         if (line->state == PBLK_LINESTATE_BAD)
 604                 return 0;
 605
 606         smeta_blk = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line);
 607         if (smeta_blk >= lm->blk_per_line)
 608                 return 0;
 609
 610         bppa = pblk->luns[smeta_blk].bppa;
 611         chunk = &line->chks[pblk_ppa_to_pos(geo, bppa)];
 612
 613         if (chunk->state & NVM_CHK_ST_FREE)
 614                 return 0;
 615
 616         return 1;
 617 }
 618
 619 static bool pblk_line_is_open(struct pblk *pblk, struct pblk_line *line)
 620 {
 621         struct pblk_line_meta *lm = &pblk->lm;
 622         int i;
 623
 624         for (i = 0; i < lm->blk_per_line; i++)
 625                 if (line->chks[i].state & NVM_CHK_ST_OPEN)
 626                         return true;
 627
 628         return false;
 629 }
 630
 631 struct pblk_line *pblk_recov_l2p(struct pblk *pblk)
 632 {
 633         struct pblk_line_meta *lm = &pblk->lm;
 634         struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 635         struct pblk_line *line, *tline, *data_line = NULL;
 636         struct pblk_smeta *smeta;
 637         struct pblk_emeta *emeta;
 638         struct line_smeta *smeta_buf;
 639         int found_lines = 0, recovered_lines = 0, open_lines = 0;
 640         int is_next = 0;
 641         int meta_line;
 642         int i, valid_uuid = 0;
 643         LIST_HEAD(recov_list);
 644
 645         /* TODO: Implement FTL snapshot */
 646
 647         /* Scan recovery - takes place when FTL snapshot fails */
 648         spin_lock(&l_mg->free_lock);
 649         meta_line = find_first_zero_bit(&l_mg->meta_bitmap, PBLK_DATA_LINES);
 650         set_bit(meta_line, &l_mg->meta_bitmap);
 651         smeta = l_mg->sline_meta[meta_line];
 652         emeta = l_mg->eline_meta[meta_line];
 653         smeta_buf = (struct line_smeta *)smeta;
 654         spin_unlock(&l_mg->free_lock);
 655
 656         /* Order data lines using their sequence number */
 657         for (i = 0; i < l_mg->nr_lines; i++) {
 658                 u32 crc;
 659
 660                 line = &pblk->lines[i];
 661
 662                 memset(smeta, 0, lm->smeta_len);
 663                 line->smeta = smeta;
 664                 line->lun_bitmap = ((void *)(smeta_buf)) +
 665                                                 sizeof(struct line_smeta);
 666
 667                 if (!pblk_line_was_written(line, pblk))
 668                         continue;
 669
 670                 /* Lines that cannot be read are assumed as not written here */
 671                 if (pblk_line_smeta_read(pblk, line))
 672                         continue;
 673
 674                 crc = pblk_calc_smeta_crc(pblk, smeta_buf);
 675                 if (le32_to_cpu(smeta_buf->crc) != crc)
 676                         continue;
 677
 678                 if (le32_to_cpu(smeta_buf->header.identifier) != PBLK_MAGIC)
 679                         continue;
 680
 681                 if (smeta_buf->header.version_major != SMETA_VERSION_MAJOR) {
 682                         pblk_err(pblk, "found incompatible line version %u\n",
 683                                         smeta_buf->header.version_major);
 684                         return ERR_PTR(-EINVAL);
 685                 }
 686
 687                 /* The first valid instance uuid is used for initialization */
 688                 if (!valid_uuid) {
 689                         memcpy(pblk->instance_uuid, smeta_buf->header.uuid, 16);
 690                         valid_uuid = 1;
 691                 }
 692
 693                 if (memcmp(pblk->instance_uuid, smeta_buf->header.uuid, 16)) {
 694                         pblk_debug(pblk, "ignore line %u due to uuid mismatch\n",
 695                                         i);
 696                         continue;
 697                 }
 698
 699                 /* Update line metadata */
 700                 spin_lock(&line->lock);
 701                 line->id = le32_to_cpu(smeta_buf->header.id);
 702                 line->type = le16_to_cpu(smeta_buf->header.type);
 703                 line->seq_nr = le64_to_cpu(smeta_buf->seq_nr);
 704                 spin_unlock(&line->lock);
 705
 706                 /* Update general metadata */
 707                 spin_lock(&l_mg->free_lock);
 708                 if (line->seq_nr >= l_mg->d_seq_nr)
 709                         l_mg->d_seq_nr = line->seq_nr + 1;
 710                 l_mg->nr_free_lines--;
 711                 spin_unlock(&l_mg->free_lock);
 712
 713                 if (pblk_line_recov_alloc(pblk, line))
 714                         goto out;
 715
 716                 pblk_recov_line_add_ordered(&recov_list, line);
 717                 found_lines++;
 718                 pblk_debug(pblk, "recovering data line %d, seq:%llu\n",
 719                                                 line->id, smeta_buf->seq_nr);
 720         }
 721
 722         if (!found_lines) {
 723                 pblk_setup_uuid(pblk);
 724
 725                 spin_lock(&l_mg->free_lock);
 726                 WARN_ON_ONCE(!test_and_clear_bit(meta_line,
 727                                                         &l_mg->meta_bitmap));
 728                 spin_unlock(&l_mg->free_lock);
 729
 730                 goto out;
 731         }
 732
 733         /* Verify closed blocks and recover this portion of L2P table*/
 734         list_for_each_entry_safe(line, tline, &recov_list, list) {
 735                 recovered_lines++;
 736
 737                 line->emeta_ssec = pblk_line_emeta_start(pblk, line);
 738                 line->emeta = emeta;
 739                 memset(line->emeta->buf, 0, lm->emeta_len[0]);
 740
 741                 if (pblk_line_is_open(pblk, line)) {
 742                         pblk_recov_l2p_from_oob(pblk, line);
 743                         goto next;
 744                 }
 745
 746                 if (pblk_line_emeta_read(pblk, line, line->emeta->buf)) {
 747                         pblk_recov_l2p_from_oob(pblk, line);
 748                         goto next;
 749                 }
 750
 751                 if (pblk_recov_check_emeta(pblk, line->emeta->buf)) {
 752                         pblk_recov_l2p_from_oob(pblk, line);
 753                         goto next;
 754                 }
 755
 756                 if (pblk_recov_check_line_version(pblk, line->emeta->buf))
 757                         return ERR_PTR(-EINVAL);
 758
 759                 pblk_recov_wa_counters(pblk, line->emeta->buf);
 760
 761                 if (pblk_recov_l2p_from_emeta(pblk, line))
 762                         pblk_recov_l2p_from_oob(pblk, line);
 763
 764 next:
 765                 if (pblk_line_is_full(line)) {
 766                         struct list_head *move_list;
 767
 768                         spin_lock(&line->lock);
 769                         line->state = PBLK_LINESTATE_CLOSED;
 770                         trace_pblk_line_state(pblk_disk_name(pblk), line->id,
 771                                         line->state);
 772                         move_list = pblk_line_gc_list(pblk, line);
 773                         spin_unlock(&line->lock);
 774
 775                         spin_lock(&l_mg->gc_lock);
 776                         list_move_tail(&line->list, move_list);
 777                         spin_unlock(&l_mg->gc_lock);
 778
 779                         mempool_free(line->map_bitmap, l_mg->bitmap_pool);
 780                         line->map_bitmap = NULL;
 781                         line->smeta = NULL;
 782                         line->emeta = NULL;
 783                 } else {
 784                         spin_lock(&line->lock);
 785                         line->state = PBLK_LINESTATE_OPEN;
 786                         spin_unlock(&line->lock);
 787
 788                         line->emeta->mem = 0;
 789                         atomic_set(&line->emeta->sync, 0);
 790
 791                         trace_pblk_line_state(pblk_disk_name(pblk), line->id,
 792                                         line->state);
 793
 794                         data_line = line;
 795                         line->meta_line = meta_line;
 796
 797                         open_lines++;
 798                 }
 799         }
 800
 801         if (!open_lines) {
 802                 spin_lock(&l_mg->free_lock);
 803                 WARN_ON_ONCE(!test_and_clear_bit(meta_line,
 804                                                         &l_mg->meta_bitmap));
 805                 spin_unlock(&l_mg->free_lock);
 806                 pblk_line_replace_data(pblk);
 807         } else {
 808                 spin_lock(&l_mg->free_lock);
 809                 /* Allocate next line for preparation */
 810                 l_mg->data_next = pblk_line_get(pblk);
 811                 if (l_mg->data_next) {
 812                         l_mg->data_next->seq_nr = l_mg->d_seq_nr++;
 813                         l_mg->data_next->type = PBLK_LINETYPE_DATA;
 814                         is_next = 1;
 815                 }
 816                 spin_unlock(&l_mg->free_lock);
 817         }
 818
 819         if (is_next)
 820                 pblk_line_erase(pblk, l_mg->data_next);
 821
 822 out:
 823         if (found_lines != recovered_lines)
 824                 pblk_err(pblk, "failed to recover all found lines %d/%d\n",
 825                                                 found_lines, recovered_lines);
 826
 827         return data_line;
 828 }
 829
 830 /*
 831  * Pad current line
 832  */
 833 int pblk_recov_pad(struct pblk *pblk)
 834 {
 835         struct pblk_line *line;
 836         struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 837         int left_msecs;
 838         int ret = 0;
 839
 840         spin_lock(&l_mg->free_lock);
 841         line = l_mg->data_line;
 842         left_msecs = line->left_msecs;
 843         spin_unlock(&l_mg->free_lock);
 844
 845         ret = pblk_recov_pad_line(pblk, line, left_msecs);
 846         if (ret) {
 847                 pblk_err(pblk, "tear down padding failed (%d)\n", ret);
 848                 return ret;
 849         }
 850
 851         pblk_line_close_meta(pblk, line);
 852         return ret;
 853 }