drivers/lightnvm/core.c

   1 /*
   2  * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
   3  * Initial release: Matias Bjorling <m@bjorling.me>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version
   7  * 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; see the file COPYING.  If not, write to
  16  * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
  17  * USA.
  18  *
  19  */
  20
  21 #include <linux/list.h>
  22 #include <linux/types.h>
  23 #include <linux/sem.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/module.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/lightnvm.h>
  29 #include <linux/sched/sysctl.h>
  30
  31 static LIST_HEAD(nvm_tgt_types);
  32 static DECLARE_RWSEM(nvm_tgtt_lock);
  33 static LIST_HEAD(nvm_devices);
  34 static DECLARE_RWSEM(nvm_lock);
  35
  36 /* Map between virtual and physical channel and lun */
  37 struct nvm_ch_map {
  38         int ch_off;
  39         int nr_luns;
  40         int *lun_offs;
  41 };
  42
  43 struct nvm_dev_map {
  44         struct nvm_ch_map *chnls;
  45         int nr_chnls;
  46 };
  47
  48 static struct nvm_target *nvm_find_target(struct nvm_dev *dev, const char *name)
  49 {
  50         struct nvm_target *tgt;
  51
  52         list_for_each_entry(tgt, &dev->targets, list)
  53                 if (!strcmp(name, tgt->disk->disk_name))
  54                         return tgt;
  55
  56         return NULL;
  57 }
  58
  59 static bool nvm_target_exists(const char *name)
  60 {
  61         struct nvm_dev *dev;
  62         struct nvm_target *tgt;
  63         bool ret = false;
  64
  65         down_write(&nvm_lock);
  66         list_for_each_entry(dev, &nvm_devices, devices) {
  67                 mutex_lock(&dev->mlock);
  68                 list_for_each_entry(tgt, &dev->targets, list) {
  69                         if (!strcmp(name, tgt->disk->disk_name)) {
  70                                 ret = true;
  71                                 mutex_unlock(&dev->mlock);
  72                                 goto out;
  73                         }
  74                 }
  75                 mutex_unlock(&dev->mlock);
  76         }
  77
  78 out:
  79         up_write(&nvm_lock);
  80         return ret;
  81 }
  82
  83 static int nvm_reserve_luns(struct nvm_dev *dev, int lun_begin, int lun_end)
  84 {
  85         int i;
  86
  87         for (i = lun_begin; i <= lun_end; i++) {
  88                 if (test_and_set_bit(i, dev->lun_map)) {
  89                         pr_err("nvm: lun %d already allocated\n", i);
  90                         goto err;
  91                 }
  92         }
  93
  94         return 0;
  95 err:
  96         while (--i >= lun_begin)
  97                 clear_bit(i, dev->lun_map);
  98
  99         return -EBUSY;
 100 }
 101
 102 static void nvm_release_luns_err(struct nvm_dev *dev, int lun_begin,
 103                                  int lun_end)
 104 {
 105         int i;
 106
 107         for (i = lun_begin; i <= lun_end; i++)
 108                 WARN_ON(!test_and_clear_bit(i, dev->lun_map));
 109 }
 110
 111 static void nvm_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev, int clear)
 112 {
 113         struct nvm_dev *dev = tgt_dev->parent;
 114         struct nvm_dev_map *dev_map = tgt_dev->map;
 115         int i, j;
 116
 117         for (i = 0; i < dev_map->nr_chnls; i++) {
 118                 struct nvm_ch_map *ch_map = &dev_map->chnls[i];
 119                 int *lun_offs = ch_map->lun_offs;
 120                 int ch = i + ch_map->ch_off;
 121
 122                 if (clear) {
 123                         for (j = 0; j < ch_map->nr_luns; j++) {
 124                                 int lun = j + lun_offs[j];
 125                                 int lunid = (ch * dev->geo.nr_luns) + lun;
 126
 127                                 WARN_ON(!test_and_clear_bit(lunid,
 128                                                         dev->lun_map));
 129                         }
 130                 }
 131
 132                 kfree(ch_map->lun_offs);
 133         }
 134
 135         kfree(dev_map->chnls);
 136         kfree(dev_map);
 137
 138         kfree(tgt_dev->luns);
 139         kfree(tgt_dev);
 140 }
 141
 142 static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev,
 143                                               int lun_begin, int lun_end)
 144 {
 145         struct nvm_tgt_dev *tgt_dev = NULL;
 146         struct nvm_dev_map *dev_rmap = dev->rmap;
 147         struct nvm_dev_map *dev_map;
 148         struct ppa_addr *luns;
 149         int nr_luns = lun_end - lun_begin + 1;
 150         int luns_left = nr_luns;
 151         int nr_chnls = nr_luns / dev->geo.nr_luns;
 152         int nr_chnls_mod = nr_luns % dev->geo.nr_luns;
 153         int bch = lun_begin / dev->geo.nr_luns;
 154         int blun = lun_begin % dev->geo.nr_luns;
 155         int lunid = 0;
 156         int lun_balanced = 1;
 157         int prev_nr_luns;
 158         int i, j;
 159
 160         nr_chnls = (nr_chnls_mod == 0) ? nr_chnls : nr_chnls + 1;
 161
 162         dev_map = kmalloc(sizeof(struct nvm_dev_map), GFP_KERNEL);
 163         if (!dev_map)
 164                 goto err_dev;
 165
 166         dev_map->chnls = kcalloc(nr_chnls, sizeof(struct nvm_ch_map),
 167                                                                 GFP_KERNEL);
 168         if (!dev_map->chnls)
 169                 goto err_chnls;
 170
 171         luns = kcalloc(nr_luns, sizeof(struct ppa_addr), GFP_KERNEL);
 172         if (!luns)
 173                 goto err_luns;
 174
 175         prev_nr_luns = (luns_left > dev->geo.nr_luns) ?
 176                                         dev->geo.nr_luns : luns_left;
 177         for (i = 0; i < nr_chnls; i++) {
 178                 struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[i + bch];
 179                 int *lun_roffs = ch_rmap->lun_offs;
 180                 struct nvm_ch_map *ch_map = &dev_map->chnls[i];
 181                 int *lun_offs;
 182                 int luns_in_chnl = (luns_left > dev->geo.nr_luns) ?
 183                                         dev->geo.nr_luns : luns_left;
 184
 185                 if (lun_balanced && prev_nr_luns != luns_in_chnl)
 186                         lun_balanced = 0;
 187
 188                 ch_map->ch_off = ch_rmap->ch_off = bch;
 189                 ch_map->nr_luns = luns_in_chnl;
 190
 191                 lun_offs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL);
 192                 if (!lun_offs)
 193                         goto err_ch;
 194
 195                 for (j = 0; j < luns_in_chnl; j++) {
 196                         luns[lunid].ppa = 0;
 197                         luns[lunid].g.ch = i;
 198                         luns[lunid++].g.lun = j;
 199
 200                         lun_offs[j] = blun;
 201                         lun_roffs[j + blun] = blun;
 202                 }
 203
 204                 ch_map->lun_offs = lun_offs;
 205
 206                 /* when starting a new channel, lun offset is reset */
 207                 blun = 0;
 208                 luns_left -= luns_in_chnl;
 209         }
 210
 211         dev_map->nr_chnls = nr_chnls;
 212
 213         tgt_dev = kmalloc(sizeof(struct nvm_tgt_dev), GFP_KERNEL);
 214         if (!tgt_dev)
 215                 goto err_ch;
 216
 217         memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo));
 218         /* Target device only owns a portion of the physical device */
 219         tgt_dev->geo.nr_chnls = nr_chnls;
 220         tgt_dev->geo.all_luns = nr_luns;
 221         tgt_dev->geo.nr_luns = (lun_balanced) ? prev_nr_luns : -1;
 222         tgt_dev->total_secs = nr_luns * tgt_dev->geo.sec_per_lun;
 223         tgt_dev->q = dev->q;
 224         tgt_dev->map = dev_map;
 225         tgt_dev->luns = luns;
 226         memcpy(&tgt_dev->identity, &dev->identity, sizeof(struct nvm_id));
 227
 228         tgt_dev->parent = dev;
 229
 230         return tgt_dev;
 231 err_ch:
 232         while (--i >= 0)
 233                 kfree(dev_map->chnls[i].lun_offs);
 234         kfree(luns);
 235 err_luns:
 236         kfree(dev_map->chnls);
 237 err_chnls:
 238         kfree(dev_map);
 239 err_dev:
 240         return tgt_dev;
 241 }
 242
 243 static const struct block_device_operations nvm_fops = {
 244         .owner          = THIS_MODULE,
 245 };
 246
 247 static struct nvm_tgt_type *__nvm_find_target_type(const char *name)
 248 {
 249         struct nvm_tgt_type *tt;
 250
 251         list_for_each_entry(tt, &nvm_tgt_types, list)
 252                 if (!strcmp(name, tt->name))
 253                         return tt;
 254
 255         return NULL;
 256 }
 257
 258 static struct nvm_tgt_type *nvm_find_target_type(const char *name)
 259 {
 260         struct nvm_tgt_type *tt;
 261
 262         down_write(&nvm_tgtt_lock);
 263         tt = __nvm_find_target_type(name);
 264         up_write(&nvm_tgtt_lock);
 265
 266         return tt;
 267 }
 268
 269 static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 270 {
 271         struct nvm_ioctl_create_simple *s = &create->conf.s;
 272         struct request_queue *tqueue;
 273         struct gendisk *tdisk;
 274         struct nvm_tgt_type *tt;
 275         struct nvm_target *t;
 276         struct nvm_tgt_dev *tgt_dev;
 277         void *targetdata;
 278         int ret;
 279
 280         tt = nvm_find_target_type(create->tgttype);
 281         if (!tt) {
 282                 pr_err("nvm: target type %s not found\n", create->tgttype);
 283                 return -EINVAL;
 284         }
 285
 286         if (nvm_target_exists(create->tgtname)) {
 287                 pr_err("nvm: target name already exists (%s)\n",
 288                                                         create->tgtname);
 289                 return -EINVAL;
 290         }
 291
 292         ret = nvm_reserve_luns(dev, s->lun_begin, s->lun_end);
 293         if (ret)
 294                 return ret;
 295
 296         t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL);
 297         if (!t) {
 298                 ret = -ENOMEM;
 299                 goto err_reserve;
 300         }
 301
 302         tgt_dev = nvm_create_tgt_dev(dev, s->lun_begin, s->lun_end);
 303         if (!tgt_dev) {
 304                 pr_err("nvm: could not create target device\n");
 305                 ret = -ENOMEM;
 306                 goto err_t;
 307         }
 308
 309         tdisk = alloc_disk(0);
 310         if (!tdisk) {
 311                 ret = -ENOMEM;
 312                 goto err_dev;
 313         }
 314
 315         tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
 316         if (!tqueue) {
 317                 ret = -ENOMEM;
 318                 goto err_disk;
 319         }
 320         blk_queue_make_request(tqueue, tt->make_rq);
 321
 322         strlcpy(tdisk->disk_name, create->tgtname, sizeof(tdisk->disk_name));
 323         tdisk->flags = GENHD_FL_EXT_DEVT;
 324         tdisk->major = 0;
 325         tdisk->first_minor = 0;
 326         tdisk->fops = &nvm_fops;
 327         tdisk->queue = tqueue;
 328
 329         targetdata = tt->init(tgt_dev, tdisk, create->flags);
 330         if (IS_ERR(targetdata)) {
 331                 ret = PTR_ERR(targetdata);
 332                 goto err_init;
 333         }
 334
 335         tdisk->private_data = targetdata;
 336         tqueue->queuedata = targetdata;
 337
 338         blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect);
 339
 340         set_capacity(tdisk, tt->capacity(targetdata));
 341         add_disk(tdisk);
 342
 343         if (tt->sysfs_init && tt->sysfs_init(tdisk)) {
 344                 ret = -ENOMEM;
 345                 goto err_sysfs;
 346         }
 347
 348         t->type = tt;
 349         t->disk = tdisk;
 350         t->dev = tgt_dev;
 351
 352         mutex_lock(&dev->mlock);
 353         list_add_tail(&t->list, &dev->targets);
 354         mutex_unlock(&dev->mlock);
 355
 356         __module_get(tt->owner);
 357
 358         return 0;
 359 err_sysfs:
 360         if (tt->exit)
 361                 tt->exit(targetdata);
 362 err_init:
 363         blk_cleanup_queue(tqueue);
 364         tdisk->queue = NULL;
 365 err_disk:
 366         put_disk(tdisk);
 367 err_dev:
 368         nvm_remove_tgt_dev(tgt_dev, 0);
 369 err_t:
 370         kfree(t);
 371 err_reserve:
 372         nvm_release_luns_err(dev, s->lun_begin, s->lun_end);
 373         return ret;
 374 }
 375
 376 static void __nvm_remove_target(struct nvm_target *t)
 377 {
 378         struct nvm_tgt_type *tt = t->type;
 379         struct gendisk *tdisk = t->disk;
 380         struct request_queue *q = tdisk->queue;
 381
 382         del_gendisk(tdisk);
 383         blk_cleanup_queue(q);
 384
 385         if (tt->sysfs_exit)
 386                 tt->sysfs_exit(tdisk);
 387
 388         if (tt->exit)
 389                 tt->exit(tdisk->private_data);
 390
 391         nvm_remove_tgt_dev(t->dev, 1);
 392         put_disk(tdisk);
 393         module_put(t->type->owner);
 394
 395         list_del(&t->list);
 396         kfree(t);
 397 }
 398
 399 /**
 400  * nvm_remove_tgt - Removes a target from the media manager
 401  * @dev:        device
 402  * @remove:     ioctl structure with target name to remove.
 403  *
 404  * Returns:
 405  * 0: on success
 406  * 1: on not found
 407  * <0: on error
 408  */
 409 static int nvm_remove_tgt(struct nvm_dev *dev, struct nvm_ioctl_remove *remove)
 410 {
 411         struct nvm_target *t;
 412
 413         mutex_lock(&dev->mlock);
 414         t = nvm_find_target(dev, remove->tgtname);
 415         if (!t) {
 416                 mutex_unlock(&dev->mlock);
 417                 return 1;
 418         }
 419         __nvm_remove_target(t);
 420         mutex_unlock(&dev->mlock);
 421
 422         return 0;
 423 }
 424
 425 static int nvm_register_map(struct nvm_dev *dev)
 426 {
 427         struct nvm_dev_map *rmap;
 428         int i, j;
 429
 430         rmap = kmalloc(sizeof(struct nvm_dev_map), GFP_KERNEL);
 431         if (!rmap)
 432                 goto err_rmap;
 433
 434         rmap->chnls = kcalloc(dev->geo.nr_chnls, sizeof(struct nvm_ch_map),
 435                                                                 GFP_KERNEL);
 436         if (!rmap->chnls)
 437                 goto err_chnls;
 438
 439         for (i = 0; i < dev->geo.nr_chnls; i++) {
 440                 struct nvm_ch_map *ch_rmap;
 441                 int *lun_roffs;
 442                 int luns_in_chnl = dev->geo.nr_luns;
 443
 444                 ch_rmap = &rmap->chnls[i];
 445
 446                 ch_rmap->ch_off = -1;
 447                 ch_rmap->nr_luns = luns_in_chnl;
 448
 449                 lun_roffs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL);
 450                 if (!lun_roffs)
 451                         goto err_ch;
 452
 453                 for (j = 0; j < luns_in_chnl; j++)
 454                         lun_roffs[j] = -1;
 455
 456                 ch_rmap->lun_offs = lun_roffs;
 457         }
 458
 459         dev->rmap = rmap;
 460
 461         return 0;
 462 err_ch:
 463         while (--i >= 0)
 464                 kfree(rmap->chnls[i].lun_offs);
 465 err_chnls:
 466         kfree(rmap);
 467 err_rmap:
 468         return -ENOMEM;
 469 }
 470
 471 static void nvm_unregister_map(struct nvm_dev *dev)
 472 {
 473         struct nvm_dev_map *rmap = dev->rmap;
 474         int i;
 475
 476         for (i = 0; i < dev->geo.nr_chnls; i++)
 477                 kfree(rmap->chnls[i].lun_offs);
 478
 479         kfree(rmap->chnls);
 480         kfree(rmap);
 481 }
 482
 483 static void nvm_map_to_dev(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p)
 484 {
 485         struct nvm_dev_map *dev_map = tgt_dev->map;
 486         struct nvm_ch_map *ch_map = &dev_map->chnls[p->g.ch];
 487         int lun_off = ch_map->lun_offs[p->g.lun];
 488
 489         p->g.ch += ch_map->ch_off;
 490         p->g.lun += lun_off;
 491 }
 492
 493 static void nvm_map_to_tgt(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p)
 494 {
 495         struct nvm_dev *dev = tgt_dev->parent;
 496         struct nvm_dev_map *dev_rmap = dev->rmap;
 497         struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[p->g.ch];
 498         int lun_roff = ch_rmap->lun_offs[p->g.lun];
 499
 500         p->g.ch -= ch_rmap->ch_off;
 501         p->g.lun -= lun_roff;
 502 }
 503
 504 static void nvm_ppa_tgt_to_dev(struct nvm_tgt_dev *tgt_dev,
 505                                 struct ppa_addr *ppa_list, int nr_ppas)
 506 {
 507         int i;
 508
 509         for (i = 0; i < nr_ppas; i++) {
 510                 nvm_map_to_dev(tgt_dev, &ppa_list[i]);
 511                 ppa_list[i] = generic_to_dev_addr(tgt_dev, ppa_list[i]);
 512         }
 513 }
 514
 515 static void nvm_ppa_dev_to_tgt(struct nvm_tgt_dev *tgt_dev,
 516                                 struct ppa_addr *ppa_list, int nr_ppas)
 517 {
 518         int i;
 519
 520         for (i = 0; i < nr_ppas; i++) {
 521                 ppa_list[i] = dev_to_generic_addr(tgt_dev, ppa_list[i]);
 522                 nvm_map_to_tgt(tgt_dev, &ppa_list[i]);
 523         }
 524 }
 525
 526 static void nvm_rq_tgt_to_dev(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 527 {
 528         if (rqd->nr_ppas == 1) {
 529                 nvm_ppa_tgt_to_dev(tgt_dev, &rqd->ppa_addr, 1);
 530                 return;
 531         }
 532
 533         nvm_ppa_tgt_to_dev(tgt_dev, rqd->ppa_list, rqd->nr_ppas);
 534 }
 535
 536 static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 537 {
 538         if (rqd->nr_ppas == 1) {
 539                 nvm_ppa_dev_to_tgt(tgt_dev, &rqd->ppa_addr, 1);
 540                 return;
 541         }
 542
 543         nvm_ppa_dev_to_tgt(tgt_dev, rqd->ppa_list, rqd->nr_ppas);
 544 }
 545
 546 int nvm_register_tgt_type(struct nvm_tgt_type *tt)
 547 {
 548         int ret = 0;
 549
 550         down_write(&nvm_tgtt_lock);
 551         if (__nvm_find_target_type(tt->name))
 552                 ret = -EEXIST;
 553         else
 554                 list_add(&tt->list, &nvm_tgt_types);
 555         up_write(&nvm_tgtt_lock);
 556
 557         return ret;
 558 }
 559 EXPORT_SYMBOL(nvm_register_tgt_type);
 560
 561 void nvm_unregister_tgt_type(struct nvm_tgt_type *tt)
 562 {
 563         if (!tt)
 564                 return;
 565
 566         down_write(&nvm_tgtt_lock);
 567         list_del(&tt->list);
 568         up_write(&nvm_tgtt_lock);
 569 }
 570 EXPORT_SYMBOL(nvm_unregister_tgt_type);
 571
 572 void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
 573                                                         dma_addr_t *dma_handler)
 574 {
 575         return dev->ops->dev_dma_alloc(dev, dev->dma_pool, mem_flags,
 576                                                                 dma_handler);
 577 }
 578 EXPORT_SYMBOL(nvm_dev_dma_alloc);
 579
 580 void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler)
 581 {
 582         dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler);
 583 }
 584 EXPORT_SYMBOL(nvm_dev_dma_free);
 585
 586 static struct nvm_dev *nvm_find_nvm_dev(const char *name)
 587 {
 588         struct nvm_dev *dev;
 589
 590         list_for_each_entry(dev, &nvm_devices, devices)
 591                 if (!strcmp(name, dev->name))
 592                         return dev;
 593
 594         return NULL;
 595 }
 596
 597 static int nvm_set_rqd_ppalist(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd,
 598                         const struct ppa_addr *ppas, int nr_ppas)
 599 {
 600         struct nvm_dev *dev = tgt_dev->parent;
 601         struct nvm_geo *geo = &tgt_dev->geo;
 602         int i, plane_cnt, pl_idx;
 603         struct ppa_addr ppa;
 604
 605         if (geo->plane_mode == NVM_PLANE_SINGLE && nr_ppas == 1) {
 606                 rqd->nr_ppas = nr_ppas;
 607                 rqd->ppa_addr = ppas[0];
 608
 609                 return 0;
 610         }
 611
 612         rqd->nr_ppas = nr_ppas;
 613         rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
 614         if (!rqd->ppa_list) {
 615                 pr_err("nvm: failed to allocate dma memory\n");
 616                 return -ENOMEM;
 617         }
 618
 619         plane_cnt = geo->plane_mode;
 620         rqd->nr_ppas *= plane_cnt;
 621
 622         for (i = 0; i < nr_ppas; i++) {
 623                 for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
 624                         ppa = ppas[i];
 625                         ppa.g.pl = pl_idx;
 626                         rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppa;
 627                 }
 628         }
 629
 630         return 0;
 631 }
 632
 633 static void nvm_free_rqd_ppalist(struct nvm_tgt_dev *tgt_dev,
 634                         struct nvm_rq *rqd)
 635 {
 636         if (!rqd->ppa_list)
 637                 return;
 638
 639         nvm_dev_dma_free(tgt_dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
 640 }
 641
 642
 643 int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas,
 644                        int nr_ppas, int type)
 645 {
 646         struct nvm_dev *dev = tgt_dev->parent;
 647         struct nvm_rq rqd;
 648         int ret;
 649
 650         if (nr_ppas > dev->ops->max_phys_sect) {
 651                 pr_err("nvm: unable to update all blocks atomically\n");
 652                 return -EINVAL;
 653         }
 654
 655         memset(&rqd, 0, sizeof(struct nvm_rq));
 656
 657         nvm_set_rqd_ppalist(tgt_dev, &rqd, ppas, nr_ppas);
 658         nvm_rq_tgt_to_dev(tgt_dev, &rqd);
 659
 660         ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
 661         nvm_free_rqd_ppalist(tgt_dev, &rqd);
 662         if (ret) {
 663                 pr_err("nvm: failed bb mark\n");
 664                 return -EINVAL;
 665         }
 666
 667         return 0;
 668 }
 669 EXPORT_SYMBOL(nvm_set_tgt_bb_tbl);
 670
 671 int nvm_max_phys_sects(struct nvm_tgt_dev *tgt_dev)
 672 {
 673         struct nvm_dev *dev = tgt_dev->parent;
 674
 675         return dev->ops->max_phys_sect;
 676 }
 677 EXPORT_SYMBOL(nvm_max_phys_sects);
 678
 679 int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 680 {
 681         struct nvm_dev *dev = tgt_dev->parent;
 682         int ret;
 683
 684         if (!dev->ops->submit_io)
 685                 return -ENODEV;
 686
 687         nvm_rq_tgt_to_dev(tgt_dev, rqd);
 688
 689         rqd->dev = tgt_dev;
 690
 691         /* In case of error, fail with right address format */
 692         ret = dev->ops->submit_io(dev, rqd);
 693         if (ret)
 694                 nvm_rq_dev_to_tgt(tgt_dev, rqd);
 695         return ret;
 696 }
 697 EXPORT_SYMBOL(nvm_submit_io);
 698
 699 int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 700 {
 701         struct nvm_dev *dev = tgt_dev->parent;
 702         int ret;
 703
 704         if (!dev->ops->submit_io_sync)
 705                 return -ENODEV;
 706
 707         nvm_rq_tgt_to_dev(tgt_dev, rqd);
 708
 709         rqd->dev = tgt_dev;
 710
 711         /* In case of error, fail with right address format */
 712         ret = dev->ops->submit_io_sync(dev, rqd);
 713         nvm_rq_dev_to_tgt(tgt_dev, rqd);
 714
 715         return ret;
 716 }
 717 EXPORT_SYMBOL(nvm_submit_io_sync);
 718
 719 void nvm_end_io(struct nvm_rq *rqd)
 720 {
 721         struct nvm_tgt_dev *tgt_dev = rqd->dev;
 722
 723         /* Convert address space */
 724         if (tgt_dev)
 725                 nvm_rq_dev_to_tgt(tgt_dev, rqd);
 726
 727         if (rqd->end_io)
 728                 rqd->end_io(rqd);
 729 }
 730 EXPORT_SYMBOL(nvm_end_io);
 731
 732 /*
 733  * folds a bad block list from its plane representation to its virtual
 734  * block representation. The fold is done in place and reduced size is
 735  * returned.
 736  *
 737  * If any of the planes status are bad or grown bad block, the virtual block
 738  * is marked bad. If not bad, the first plane state acts as the block state.
 739  */
 740 int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
 741 {
 742         struct nvm_geo *geo = &dev->geo;
 743         int blk, offset, pl, blktype;
 744
 745         if (nr_blks != geo->nr_chks * geo->plane_mode)
 746                 return -EINVAL;
 747
 748         for (blk = 0; blk < geo->nr_chks; blk++) {
 749                 offset = blk * geo->plane_mode;
 750                 blktype = blks[offset];
 751
 752                 /* Bad blocks on any planes take precedence over other types */
 753                 for (pl = 0; pl < geo->plane_mode; pl++) {
 754                         if (blks[offset + pl] &
 755                                         (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) {
 756                                 blktype = blks[offset + pl];
 757                                 break;
 758                         }
 759                 }
 760
 761                 blks[blk] = blktype;
 762         }
 763
 764         return geo->nr_chks;
 765 }
 766 EXPORT_SYMBOL(nvm_bb_tbl_fold);
 767
 768 int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa,
 769                        u8 *blks)
 770 {
 771         struct nvm_dev *dev = tgt_dev->parent;
 772
 773         nvm_ppa_tgt_to_dev(tgt_dev, &ppa, 1);
 774
 775         return dev->ops->get_bb_tbl(dev, ppa, blks);
 776 }
 777 EXPORT_SYMBOL(nvm_get_tgt_bb_tbl);
 778
 779 static int nvm_core_init(struct nvm_dev *dev)
 780 {
 781         struct nvm_id *id = &dev->identity;
 782         struct nvm_id_group *grp = &id->grp;
 783         struct nvm_geo *geo = &dev->geo;
 784         int ret;
 785
 786         memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
 787
 788         if (grp->mtype != 0) {
 789                 pr_err("nvm: memory type not supported\n");
 790                 return -EINVAL;
 791         }
 792
 793         /* Whole device values */
 794         geo->nr_chnls = grp->num_ch;
 795         geo->nr_luns = grp->num_lun;
 796
 797         /* Generic device geometry values */
 798         geo->ws_min = grp->ws_min;
 799         geo->ws_opt = grp->ws_opt;
 800         geo->ws_seq = grp->ws_seq;
 801         geo->ws_per_chk = grp->ws_per_chk;
 802         geo->nr_chks = grp->num_chk;
 803         geo->sec_size = grp->csecs;
 804         geo->oob_size = grp->sos;
 805         geo->mccap = grp->mccap;
 806         geo->max_rq_size = dev->ops->max_phys_sect * geo->sec_size;
 807
 808         geo->sec_per_chk = grp->clba;
 809         geo->sec_per_lun = geo->sec_per_chk * geo->nr_chks;
 810         geo->all_luns = geo->nr_luns * geo->nr_chnls;
 811
 812         /* 1.2 spec device geometry values */
 813         geo->plane_mode = 1 << geo->ws_seq;
 814         geo->nr_planes = geo->ws_opt / geo->ws_min;
 815         geo->sec_per_pg = geo->ws_min;
 816         geo->sec_per_pl = geo->sec_per_pg * geo->nr_planes;
 817
 818         dev->total_secs = geo->all_luns * geo->sec_per_lun;
 819         dev->lun_map = kcalloc(BITS_TO_LONGS(geo->all_luns),
 820                                         sizeof(unsigned long), GFP_KERNEL);
 821         if (!dev->lun_map)
 822                 return -ENOMEM;
 823
 824         INIT_LIST_HEAD(&dev->area_list);
 825         INIT_LIST_HEAD(&dev->targets);
 826         mutex_init(&dev->mlock);
 827         spin_lock_init(&dev->lock);
 828
 829         ret = nvm_register_map(dev);
 830         if (ret)
 831                 goto err_fmtype;
 832
 833         blk_queue_logical_block_size(dev->q, geo->sec_size);
 834         return 0;
 835 err_fmtype:
 836         kfree(dev->lun_map);
 837         return ret;
 838 }
 839
 840 static void nvm_free(struct nvm_dev *dev)
 841 {
 842         if (!dev)
 843                 return;
 844
 845         if (dev->dma_pool)
 846                 dev->ops->destroy_dma_pool(dev->dma_pool);
 847
 848         nvm_unregister_map(dev);
 849         kfree(dev->lun_map);
 850         kfree(dev);
 851 }
 852
 853 static int nvm_init(struct nvm_dev *dev)
 854 {
 855         struct nvm_geo *geo = &dev->geo;
 856         int ret = -EINVAL;
 857
 858         if (dev->ops->identity(dev, &dev->identity)) {
 859                 pr_err("nvm: device could not be identified\n");
 860                 goto err;
 861         }
 862
 863         pr_debug("nvm: ver:%x nvm_vendor:%x\n",
 864                         dev->identity.ver_id, dev->identity.vmnt);
 865
 866         if (dev->identity.ver_id != 1) {
 867                 pr_err("nvm: device not supported by kernel.");
 868                 goto err;
 869         }
 870
 871         ret = nvm_core_init(dev);
 872         if (ret) {
 873                 pr_err("nvm: could not initialize core structures.\n");
 874                 goto err;
 875         }
 876
 877         pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
 878                         dev->name, geo->sec_per_pg, geo->nr_planes,
 879                         geo->ws_per_chk, geo->nr_chks,
 880                         geo->all_luns, geo->nr_chnls);
 881         return 0;
 882 err:
 883         pr_err("nvm: failed to initialize nvm\n");
 884         return ret;
 885 }
 886
 887 struct nvm_dev *nvm_alloc_dev(int node)
 888 {
 889         return kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node);
 890 }
 891 EXPORT_SYMBOL(nvm_alloc_dev);
 892
 893 int nvm_register(struct nvm_dev *dev)
 894 {
 895         int ret;
 896
 897         if (!dev->q || !dev->ops)
 898                 return -EINVAL;
 899
 900         if (dev->ops->max_phys_sect > 256) {
 901                 pr_info("nvm: max sectors supported is 256.\n");
 902                 return -EINVAL;
 903         }
 904
 905         if (dev->ops->max_phys_sect > 1) {
 906                 dev->dma_pool = dev->ops->create_dma_pool(dev, "ppalist");
 907                 if (!dev->dma_pool) {
 908                         pr_err("nvm: could not create dma pool\n");
 909                         return -ENOMEM;
 910                 }
 911         }
 912
 913         ret = nvm_init(dev);
 914         if (ret)
 915                 goto err_init;
 916
 917         /* register device with a supported media manager */
 918         down_write(&nvm_lock);
 919         list_add(&dev->devices, &nvm_devices);
 920         up_write(&nvm_lock);
 921
 922         return 0;
 923 err_init:
 924         dev->ops->destroy_dma_pool(dev->dma_pool);
 925         return ret;
 926 }
 927 EXPORT_SYMBOL(nvm_register);
 928
 929 void nvm_unregister(struct nvm_dev *dev)
 930 {
 931         struct nvm_target *t, *tmp;
 932
 933         mutex_lock(&dev->mlock);
 934         list_for_each_entry_safe(t, tmp, &dev->targets, list) {
 935                 if (t->dev->parent != dev)
 936                         continue;
 937                 __nvm_remove_target(t);
 938         }
 939         mutex_unlock(&dev->mlock);
 940
 941         down_write(&nvm_lock);
 942         list_del(&dev->devices);
 943         up_write(&nvm_lock);
 944
 945         nvm_free(dev);
 946 }
 947 EXPORT_SYMBOL(nvm_unregister);
 948
 949 static int __nvm_configure_create(struct nvm_ioctl_create *create)
 950 {
 951         struct nvm_dev *dev;
 952         struct nvm_ioctl_create_simple *s;
 953
 954         down_write(&nvm_lock);
 955         dev = nvm_find_nvm_dev(create->dev);
 956         up_write(&nvm_lock);
 957
 958         if (!dev) {
 959                 pr_err("nvm: device not found\n");
 960                 return -EINVAL;
 961         }
 962
 963         if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
 964                 pr_err("nvm: config type not valid\n");
 965                 return -EINVAL;
 966         }
 967         s = &create->conf.s;
 968
 969         if (s->lun_begin == -1 && s->lun_end == -1) {
 970                 s->lun_begin = 0;
 971                 s->lun_end = dev->geo.all_luns - 1;
 972         }
 973
 974         if (s->lun_begin > s->lun_end || s->lun_end >= dev->geo.all_luns) {
 975                 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
 976                         s->lun_begin, s->lun_end, dev->geo.all_luns - 1);
 977                 return -EINVAL;
 978         }
 979
 980         return nvm_create_tgt(dev, create);
 981 }
 982
 983 static long nvm_ioctl_info(struct file *file, void __user *arg)
 984 {
 985         struct nvm_ioctl_info *info;
 986         struct nvm_tgt_type *tt;
 987         int tgt_iter = 0;
 988
 989         if (!capable(CAP_SYS_ADMIN))
 990                 return -EPERM;
 991
 992         info = memdup_user(arg, sizeof(struct nvm_ioctl_info));
 993         if (IS_ERR(info))
 994                 return -EFAULT;
 995
 996         info->version[0] = NVM_VERSION_MAJOR;
 997         info->version[1] = NVM_VERSION_MINOR;
 998         info->version[2] = NVM_VERSION_PATCH;
 999
1000         down_write(&nvm_tgtt_lock);
1001         list_for_each_entry(tt, &nvm_tgt_types, list) {
1002                 struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter];
1003
1004                 tgt->version[0] = tt->version[0];
1005                 tgt->version[1] = tt->version[1];
1006                 tgt->version[2] = tt->version[2];
1007                 strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX);
1008
1009                 tgt_iter++;
1010         }
1011
1012         info->tgtsize = tgt_iter;
1013         up_write(&nvm_tgtt_lock);
1014
1015         if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) {
1016                 kfree(info);
1017                 return -EFAULT;
1018         }
1019
1020         kfree(info);
1021         return 0;
1022 }
1023
1024 static long nvm_ioctl_get_devices(struct file *file, void __user *arg)
1025 {
1026         struct nvm_ioctl_get_devices *devices;
1027         struct nvm_dev *dev;
1028         int i = 0;
1029
1030         if (!capable(CAP_SYS_ADMIN))
1031                 return -EPERM;
1032
1033         devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL);
1034         if (!devices)
1035                 return -ENOMEM;
1036
1037         down_write(&nvm_lock);
1038         list_for_each_entry(dev, &nvm_devices, devices) {
1039                 struct nvm_ioctl_device_info *info = &devices->info[i];
1040
1041                 strlcpy(info->devname, dev->name, sizeof(info->devname));
1042
1043                 /* kept for compatibility */
1044                 info->bmversion[0] = 1;
1045                 info->bmversion[1] = 0;
1046                 info->bmversion[2] = 0;
1047                 strlcpy(info->bmname, "gennvm", sizeof(info->bmname));
1048                 i++;
1049
1050                 if (i > 31) {
1051                         pr_err("nvm: max 31 devices can be reported.\n");
1052                         break;
1053                 }
1054         }
1055         up_write(&nvm_lock);
1056
1057         devices->nr_devices = i;
1058
1059         if (copy_to_user(arg, devices,
1060                          sizeof(struct nvm_ioctl_get_devices))) {
1061                 kfree(devices);
1062                 return -EFAULT;
1063         }
1064
1065         kfree(devices);
1066         return 0;
1067 }
1068
1069 static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
1070 {
1071         struct nvm_ioctl_create create;
1072
1073         if (!capable(CAP_SYS_ADMIN))
1074                 return -EPERM;
1075
1076         if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
1077                 return -EFAULT;
1078
1079         create.dev[DISK_NAME_LEN - 1] = '\0';
1080         create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
1081         create.tgtname[DISK_NAME_LEN - 1] = '\0';
1082
1083         if (create.flags != 0) {
1084                 __u32 flags = create.flags;
1085
1086                 /* Check for valid flags */
1087                 if (flags & NVM_TARGET_FACTORY)
1088                         flags &= ~NVM_TARGET_FACTORY;
1089
1090                 if (flags) {
1091                         pr_err("nvm: flag not supported\n");
1092                         return -EINVAL;
1093                 }
1094         }
1095
1096         return __nvm_configure_create(&create);
1097 }
1098
1099 static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
1100 {
1101         struct nvm_ioctl_remove remove;
1102         struct nvm_dev *dev;
1103         int ret = 0;
1104
1105         if (!capable(CAP_SYS_ADMIN))
1106                 return -EPERM;
1107
1108         if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove)))
1109                 return -EFAULT;
1110
1111         remove.tgtname[DISK_NAME_LEN - 1] = '\0';
1112
1113         if (remove.flags != 0) {
1114                 pr_err("nvm: no flags supported\n");
1115                 return -EINVAL;
1116         }
1117
1118         list_for_each_entry(dev, &nvm_devices, devices) {
1119                 ret = nvm_remove_tgt(dev, &remove);
1120                 if (!ret)
1121                         break;
1122         }
1123
1124         return ret;
1125 }
1126
1127 /* kept for compatibility reasons */
1128 static long nvm_ioctl_dev_init(struct file *file, void __user *arg)
1129 {
1130         struct nvm_ioctl_dev_init init;
1131
1132         if (!capable(CAP_SYS_ADMIN))
1133                 return -EPERM;
1134
1135         if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init)))
1136                 return -EFAULT;
1137
1138         if (init.flags != 0) {
1139                 pr_err("nvm: no flags supported\n");
1140                 return -EINVAL;
1141         }
1142
1143         return 0;
1144 }
1145
1146 /* Kept for compatibility reasons */
1147 static long nvm_ioctl_dev_factory(struct file *file, void __user *arg)
1148 {
1149         struct nvm_ioctl_dev_factory fact;
1150
1151         if (!capable(CAP_SYS_ADMIN))
1152                 return -EPERM;
1153
1154         if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory)))
1155                 return -EFAULT;
1156
1157         fact.dev[DISK_NAME_LEN - 1] = '\0';
1158
1159         if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1))
1160                 return -EINVAL;
1161
1162         return 0;
1163 }
1164
1165 static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
1166 {
1167         void __user *argp = (void __user *)arg;
1168
1169         switch (cmd) {
1170         case NVM_INFO:
1171                 return nvm_ioctl_info(file, argp);
1172         case NVM_GET_DEVICES:
1173                 return nvm_ioctl_get_devices(file, argp);
1174         case NVM_DEV_CREATE:
1175                 return nvm_ioctl_dev_create(file, argp);
1176         case NVM_DEV_REMOVE:
1177                 return nvm_ioctl_dev_remove(file, argp);
1178         case NVM_DEV_INIT:
1179                 return nvm_ioctl_dev_init(file, argp);
1180         case NVM_DEV_FACTORY:
1181                 return nvm_ioctl_dev_factory(file, argp);
1182         }
1183         return 0;
1184 }
1185
1186 static const struct file_operations _ctl_fops = {
1187         .open = nonseekable_open,
1188         .unlocked_ioctl = nvm_ctl_ioctl,
1189         .owner = THIS_MODULE,
1190         .llseek  = noop_llseek,
1191 };
1192
1193 static struct miscdevice _nvm_misc = {
1194         .minor          = MISC_DYNAMIC_MINOR,
1195         .name           = "lightnvm",
1196         .nodename       = "lightnvm/control",
1197         .fops           = &_ctl_fops,
1198 };
1199 builtin_misc_device(_nvm_misc);