drivers/lightnvm/core.c

   1 /*
   2  * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
   3  * Initial release: Matias Bjorling <m@bjorling.me>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version
   7  * 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; see the file COPYING.  If not, write to
  16  * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
  17  * USA.
  18  *
  19  */
  20
  21 #include <linux/list.h>
  22 #include <linux/types.h>
  23 #include <linux/sem.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/module.h>
  26 #include <linux/miscdevice.h>
  27 #include <linux/lightnvm.h>
  28 #include <linux/sched/sysctl.h>
  29
  30 static LIST_HEAD(nvm_tgt_types);
  31 static DECLARE_RWSEM(nvm_tgtt_lock);
  32 static LIST_HEAD(nvm_mgrs);
  33 static LIST_HEAD(nvm_devices);
  34 static DECLARE_RWSEM(nvm_lock);
  35
  36 struct nvm_tgt_type *nvm_find_target_type(const char *name, int lock)
  37 {
  38         struct nvm_tgt_type *tmp, *tt = NULL;
  39
  40         if (lock)
  41                 down_write(&nvm_tgtt_lock);
  42
  43         list_for_each_entry(tmp, &nvm_tgt_types, list)
  44                 if (!strcmp(name, tmp->name)) {
  45                         tt = tmp;
  46                         break;
  47                 }
  48
  49         if (lock)
  50                 up_write(&nvm_tgtt_lock);
  51         return tt;
  52 }
  53 EXPORT_SYMBOL(nvm_find_target_type);
  54
  55 int nvm_register_tgt_type(struct nvm_tgt_type *tt)
  56 {
  57         int ret = 0;
  58
  59         down_write(&nvm_tgtt_lock);
  60         if (nvm_find_target_type(tt->name, 0))
  61                 ret = -EEXIST;
  62         else
  63                 list_add(&tt->list, &nvm_tgt_types);
  64         up_write(&nvm_tgtt_lock);
  65
  66         return ret;
  67 }
  68 EXPORT_SYMBOL(nvm_register_tgt_type);
  69
  70 void nvm_unregister_tgt_type(struct nvm_tgt_type *tt)
  71 {
  72         if (!tt)
  73                 return;
  74
  75         down_write(&nvm_lock);
  76         list_del(&tt->list);
  77         up_write(&nvm_lock);
  78 }
  79 EXPORT_SYMBOL(nvm_unregister_tgt_type);
  80
  81 void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
  82                                                         dma_addr_t *dma_handler)
  83 {
  84         return dev->ops->dev_dma_alloc(dev, dev->dma_pool, mem_flags,
  85                                                                 dma_handler);
  86 }
  87 EXPORT_SYMBOL(nvm_dev_dma_alloc);
  88
  89 void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler)
  90 {
  91         dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler);
  92 }
  93 EXPORT_SYMBOL(nvm_dev_dma_free);
  94
  95 static struct nvmm_type *nvm_find_mgr_type(const char *name)
  96 {
  97         struct nvmm_type *mt;
  98
  99         list_for_each_entry(mt, &nvm_mgrs, list)
 100                 if (!strcmp(name, mt->name))
 101                         return mt;
 102
 103         return NULL;
 104 }
 105
 106 static struct nvmm_type *nvm_init_mgr(struct nvm_dev *dev)
 107 {
 108         struct nvmm_type *mt;
 109         int ret;
 110
 111         lockdep_assert_held(&nvm_lock);
 112
 113         list_for_each_entry(mt, &nvm_mgrs, list) {
 114                 if (strncmp(dev->sb.mmtype, mt->name, NVM_MMTYPE_LEN))
 115                         continue;
 116
 117                 ret = mt->register_mgr(dev);
 118                 if (ret < 0) {
 119                         pr_err("nvm: media mgr failed to init (%d) on dev %s\n",
 120                                                                 ret, dev->name);
 121                         return NULL; /* initialization failed */
 122                 } else if (ret > 0)
 123                         return mt;
 124         }
 125
 126         return NULL;
 127 }
 128
 129 int nvm_register_mgr(struct nvmm_type *mt)
 130 {
 131         struct nvm_dev *dev;
 132         int ret = 0;
 133
 134         down_write(&nvm_lock);
 135         if (nvm_find_mgr_type(mt->name)) {
 136                 ret = -EEXIST;
 137                 goto finish;
 138         } else {
 139                 list_add(&mt->list, &nvm_mgrs);
 140         }
 141
 142         /* try to register media mgr if any device have none configured */
 143         list_for_each_entry(dev, &nvm_devices, devices) {
 144                 if (dev->mt)
 145                         continue;
 146
 147                 dev->mt = nvm_init_mgr(dev);
 148         }
 149 finish:
 150         up_write(&nvm_lock);
 151
 152         return ret;
 153 }
 154 EXPORT_SYMBOL(nvm_register_mgr);
 155
 156 void nvm_unregister_mgr(struct nvmm_type *mt)
 157 {
 158         if (!mt)
 159                 return;
 160
 161         down_write(&nvm_lock);
 162         list_del(&mt->list);
 163         up_write(&nvm_lock);
 164 }
 165 EXPORT_SYMBOL(nvm_unregister_mgr);
 166
 167 static struct nvm_dev *nvm_find_nvm_dev(const char *name)
 168 {
 169         struct nvm_dev *dev;
 170
 171         list_for_each_entry(dev, &nvm_devices, devices)
 172                 if (!strcmp(name, dev->name))
 173                         return dev;
 174
 175         return NULL;
 176 }
 177
 178 int nvm_set_bb_tbl(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas,
 179                                                                 int type)
 180 {
 181         struct nvm_rq rqd;
 182         int ret;
 183
 184         if (nr_ppas > dev->ops->max_phys_sect) {
 185                 pr_err("nvm: unable to update all sysblocks atomically\n");
 186                 return -EINVAL;
 187         }
 188
 189         memset(&rqd, 0, sizeof(struct nvm_rq));
 190
 191         nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1);
 192         nvm_generic_to_addr_mode(dev, &rqd);
 193
 194         ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
 195         nvm_free_rqd_ppalist(dev, &rqd);
 196         if (ret) {
 197                 pr_err("nvm: sysblk failed bb mark\n");
 198                 return -EINVAL;
 199         }
 200
 201         return 0;
 202 }
 203 EXPORT_SYMBOL(nvm_set_bb_tbl);
 204
 205 int nvm_max_phys_sects(struct nvm_tgt_dev *tgt_dev)
 206 {
 207         struct nvm_dev *dev = tgt_dev->parent;
 208
 209         return dev->ops->max_phys_sect;
 210 }
 211 EXPORT_SYMBOL(nvm_max_phys_sects);
 212
 213 int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 214 {
 215         struct nvm_dev *dev = tgt_dev->parent;
 216
 217         return dev->mt->submit_io(tgt_dev, rqd);
 218 }
 219 EXPORT_SYMBOL(nvm_submit_io);
 220
 221 int nvm_erase_blk(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p, int flags)
 222 {
 223         struct nvm_dev *dev = tgt_dev->parent;
 224
 225         return dev->mt->erase_blk(tgt_dev, p, flags);
 226 }
 227 EXPORT_SYMBOL(nvm_erase_blk);
 228
 229 int nvm_get_l2p_tbl(struct nvm_tgt_dev *tgt_dev, u64 slba, u32 nlb,
 230                     nvm_l2p_update_fn *update_l2p, void *priv)
 231 {
 232         struct nvm_dev *dev = tgt_dev->parent;
 233
 234         if (!dev->ops->get_l2p_tbl)
 235                 return 0;
 236
 237         return dev->ops->get_l2p_tbl(dev, slba, nlb, update_l2p, priv);
 238 }
 239 EXPORT_SYMBOL(nvm_get_l2p_tbl);
 240
 241 int nvm_get_area(struct nvm_tgt_dev *tgt_dev, sector_t *lba, sector_t len)
 242 {
 243         struct nvm_dev *dev = tgt_dev->parent;
 244
 245         return dev->mt->get_area(dev, lba, len);
 246 }
 247 EXPORT_SYMBOL(nvm_get_area);
 248
 249 void nvm_put_area(struct nvm_tgt_dev *tgt_dev, sector_t lba)
 250 {
 251         struct nvm_dev *dev = tgt_dev->parent;
 252
 253         dev->mt->put_area(dev, lba);
 254 }
 255 EXPORT_SYMBOL(nvm_put_area);
 256
 257 void nvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
 258 {
 259         int i;
 260
 261         if (rqd->nr_ppas > 1) {
 262                 for (i = 0; i < rqd->nr_ppas; i++)
 263                         rqd->ppa_list[i] = dev_to_generic_addr(dev,
 264                                                         rqd->ppa_list[i]);
 265         } else {
 266                 rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
 267         }
 268 }
 269 EXPORT_SYMBOL(nvm_addr_to_generic_mode);
 270
 271 void nvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
 272 {
 273         int i;
 274
 275         if (rqd->nr_ppas > 1) {
 276                 for (i = 0; i < rqd->nr_ppas; i++)
 277                         rqd->ppa_list[i] = generic_to_dev_addr(dev,
 278                                                         rqd->ppa_list[i]);
 279         } else {
 280                 rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
 281         }
 282 }
 283 EXPORT_SYMBOL(nvm_generic_to_addr_mode);
 284
 285 int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
 286                         const struct ppa_addr *ppas, int nr_ppas, int vblk)
 287 {
 288         struct nvm_geo *geo = &dev->geo;
 289         int i, plane_cnt, pl_idx;
 290         struct ppa_addr ppa;
 291
 292         if ((!vblk || geo->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) {
 293                 rqd->nr_ppas = nr_ppas;
 294                 rqd->ppa_addr = ppas[0];
 295
 296                 return 0;
 297         }
 298
 299         rqd->nr_ppas = nr_ppas;
 300         rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
 301         if (!rqd->ppa_list) {
 302                 pr_err("nvm: failed to allocate dma memory\n");
 303                 return -ENOMEM;
 304         }
 305
 306         if (!vblk) {
 307                 for (i = 0; i < nr_ppas; i++)
 308                         rqd->ppa_list[i] = ppas[i];
 309         } else {
 310                 plane_cnt = geo->plane_mode;
 311                 rqd->nr_ppas *= plane_cnt;
 312
 313                 for (i = 0; i < nr_ppas; i++) {
 314                         for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
 315                                 ppa = ppas[i];
 316                                 ppa.g.pl = pl_idx;
 317                                 rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppa;
 318                         }
 319                 }
 320         }
 321
 322         return 0;
 323 }
 324 EXPORT_SYMBOL(nvm_set_rqd_ppalist);
 325
 326 void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd)
 327 {
 328         if (!rqd->ppa_list)
 329                 return;
 330
 331         nvm_dev_dma_free(dev, rqd->ppa_list, rqd->dma_ppa_list);
 332 }
 333 EXPORT_SYMBOL(nvm_free_rqd_ppalist);
 334
 335 int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas,
 336                                                                 int flags)
 337 {
 338         struct nvm_rq rqd;
 339         int ret;
 340
 341         if (!dev->ops->erase_block)
 342                 return 0;
 343
 344         memset(&rqd, 0, sizeof(struct nvm_rq));
 345
 346         ret = nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1);
 347         if (ret)
 348                 return ret;
 349
 350         nvm_generic_to_addr_mode(dev, &rqd);
 351
 352         rqd.flags = flags;
 353
 354         ret = dev->ops->erase_block(dev, &rqd);
 355
 356         nvm_free_rqd_ppalist(dev, &rqd);
 357
 358         return ret;
 359 }
 360 EXPORT_SYMBOL(nvm_erase_ppa);
 361
 362 void nvm_end_io(struct nvm_rq *rqd, int error)
 363 {
 364         rqd->error = error;
 365         rqd->end_io(rqd);
 366 }
 367 EXPORT_SYMBOL(nvm_end_io);
 368
 369 static void nvm_end_io_sync(struct nvm_rq *rqd)
 370 {
 371         struct completion *waiting = rqd->wait;
 372
 373         rqd->wait = NULL;
 374
 375         complete(waiting);
 376 }
 377
 378 static int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode,
 379                                                 int flags, void *buf, int len)
 380 {
 381         DECLARE_COMPLETION_ONSTACK(wait);
 382         struct bio *bio;
 383         int ret;
 384         unsigned long hang_check;
 385
 386         bio = bio_map_kern(dev->q, buf, len, GFP_KERNEL);
 387         if (IS_ERR_OR_NULL(bio))
 388                 return -ENOMEM;
 389
 390         nvm_generic_to_addr_mode(dev, rqd);
 391
 392         rqd->dev = NULL;
 393         rqd->opcode = opcode;
 394         rqd->flags = flags;
 395         rqd->bio = bio;
 396         rqd->wait = &wait;
 397         rqd->end_io = nvm_end_io_sync;
 398
 399         ret = dev->ops->submit_io(dev, rqd);
 400         if (ret) {
 401                 bio_put(bio);
 402                 return ret;
 403         }
 404
 405         /* Prevent hang_check timer from firing at us during very long I/O */
 406         hang_check = sysctl_hung_task_timeout_secs;
 407         if (hang_check)
 408                 while (!wait_for_completion_io_timeout(&wait,
 409                                                         hang_check * (HZ/2)))
 410                         ;
 411         else
 412                 wait_for_completion_io(&wait);
 413
 414         return rqd->error;
 415 }
 416
 417 /**
 418  * nvm_submit_ppa_list - submit user-defined ppa list to device. The user must
 419  *                       take to free ppa list if necessary.
 420  * @dev:        device
 421  * @ppa_list:   user created ppa_list
 422  * @nr_ppas:    length of ppa_list
 423  * @opcode:     device opcode
 424  * @flags:      device flags
 425  * @buf:        data buffer
 426  * @len:        data buffer length
 427  */
 428 int nvm_submit_ppa_list(struct nvm_dev *dev, struct ppa_addr *ppa_list,
 429                         int nr_ppas, int opcode, int flags, void *buf, int len)
 430 {
 431         struct nvm_rq rqd;
 432
 433         if (dev->ops->max_phys_sect < nr_ppas)
 434                 return -EINVAL;
 435
 436         memset(&rqd, 0, sizeof(struct nvm_rq));
 437
 438         rqd.nr_ppas = nr_ppas;
 439         if (nr_ppas > 1)
 440                 rqd.ppa_list = ppa_list;
 441         else
 442                 rqd.ppa_addr = ppa_list[0];
 443
 444         return __nvm_submit_ppa(dev, &rqd, opcode, flags, buf, len);
 445 }
 446 EXPORT_SYMBOL(nvm_submit_ppa_list);
 447
 448 /**
 449  * nvm_submit_ppa - submit PPAs to device. PPAs will automatically be unfolded
 450  *                  as single, dual, quad plane PPAs depending on device type.
 451  * @dev:        device
 452  * @ppa:        user created ppa_list
 453  * @nr_ppas:    length of ppa_list
 454  * @opcode:     device opcode
 455  * @flags:      device flags
 456  * @buf:        data buffer
 457  * @len:        data buffer length
 458  */
 459 int nvm_submit_ppa(struct nvm_dev *dev, struct ppa_addr *ppa, int nr_ppas,
 460                                 int opcode, int flags, void *buf, int len)
 461 {
 462         struct nvm_rq rqd;
 463         int ret;
 464
 465         memset(&rqd, 0, sizeof(struct nvm_rq));
 466         ret = nvm_set_rqd_ppalist(dev, &rqd, ppa, nr_ppas, 1);
 467         if (ret)
 468                 return ret;
 469
 470         ret = __nvm_submit_ppa(dev, &rqd, opcode, flags, buf, len);
 471
 472         nvm_free_rqd_ppalist(dev, &rqd);
 473
 474         return ret;
 475 }
 476 EXPORT_SYMBOL(nvm_submit_ppa);
 477
 478 /*
 479  * folds a bad block list from its plane representation to its virtual
 480  * block representation. The fold is done in place and reduced size is
 481  * returned.
 482  *
 483  * If any of the planes status are bad or grown bad block, the virtual block
 484  * is marked bad. If not bad, the first plane state acts as the block state.
 485  */
 486 int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
 487 {
 488         struct nvm_geo *geo = &dev->geo;
 489         int blk, offset, pl, blktype;
 490
 491         if (nr_blks != geo->blks_per_lun * geo->plane_mode)
 492                 return -EINVAL;
 493
 494         for (blk = 0; blk < geo->blks_per_lun; blk++) {
 495                 offset = blk * geo->plane_mode;
 496                 blktype = blks[offset];
 497
 498                 /* Bad blocks on any planes take precedence over other types */
 499                 for (pl = 0; pl < geo->plane_mode; pl++) {
 500                         if (blks[offset + pl] &
 501                                         (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) {
 502                                 blktype = blks[offset + pl];
 503                                 break;
 504                         }
 505                 }
 506
 507                 blks[blk] = blktype;
 508         }
 509
 510         return geo->blks_per_lun;
 511 }
 512 EXPORT_SYMBOL(nvm_bb_tbl_fold);
 513
 514 int nvm_get_bb_tbl(struct nvm_dev *dev, struct ppa_addr ppa, u8 *blks)
 515 {
 516         ppa = generic_to_dev_addr(dev, ppa);
 517
 518         return dev->ops->get_bb_tbl(dev, ppa, blks);
 519 }
 520 EXPORT_SYMBOL(nvm_get_bb_tbl);
 521
 522 static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
 523 {
 524         struct nvm_geo *geo = &dev->geo;
 525         int i;
 526
 527         dev->lps_per_blk = geo->pgs_per_blk;
 528         dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
 529         if (!dev->lptbl)
 530                 return -ENOMEM;
 531
 532         /* Just a linear array */
 533         for (i = 0; i < dev->lps_per_blk; i++)
 534                 dev->lptbl[i] = i;
 535
 536         return 0;
 537 }
 538
 539 static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
 540 {
 541         int i, p;
 542         struct nvm_id_lp_mlc *mlc = &grp->lptbl.mlc;
 543
 544         if (!mlc->num_pairs)
 545                 return 0;
 546
 547         dev->lps_per_blk = mlc->num_pairs;
 548         dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
 549         if (!dev->lptbl)
 550                 return -ENOMEM;
 551
 552         /* The lower page table encoding consists of a list of bytes, where each
 553          * has a lower and an upper half. The first half byte maintains the
 554          * increment value and every value after is an offset added to the
 555          * previous incrementation value
 556          */
 557         dev->lptbl[0] = mlc->pairs[0] & 0xF;
 558         for (i = 1; i < dev->lps_per_blk; i++) {
 559                 p = mlc->pairs[i >> 1];
 560                 if (i & 0x1) /* upper */
 561                         dev->lptbl[i] = dev->lptbl[i - 1] + ((p & 0xF0) >> 4);
 562                 else /* lower */
 563                         dev->lptbl[i] = dev->lptbl[i - 1] + (p & 0xF);
 564         }
 565
 566         return 0;
 567 }
 568
 569 static int nvm_core_init(struct nvm_dev *dev)
 570 {
 571         struct nvm_id *id = &dev->identity;
 572         struct nvm_id_group *grp = &id->groups[0];
 573         struct nvm_geo *geo = &dev->geo;
 574         int ret;
 575
 576         /* Whole device values */
 577         geo->nr_chnls = grp->num_ch;
 578         geo->luns_per_chnl = grp->num_lun;
 579
 580         /* Generic device values */
 581         geo->pgs_per_blk = grp->num_pg;
 582         geo->blks_per_lun = grp->num_blk;
 583         geo->nr_planes = grp->num_pln;
 584         geo->fpg_size = grp->fpg_sz;
 585         geo->pfpg_size = grp->fpg_sz * grp->num_pln;
 586         geo->sec_size = grp->csecs;
 587         geo->oob_size = grp->sos;
 588         geo->sec_per_pg = grp->fpg_sz / grp->csecs;
 589         geo->mccap = grp->mccap;
 590         memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
 591
 592         geo->plane_mode = NVM_PLANE_SINGLE;
 593         geo->max_rq_size = dev->ops->max_phys_sect * geo->sec_size;
 594
 595         if (grp->mpos & 0x020202)
 596                 geo->plane_mode = NVM_PLANE_DOUBLE;
 597         if (grp->mpos & 0x040404)
 598                 geo->plane_mode = NVM_PLANE_QUAD;
 599
 600         if (grp->mtype != 0) {
 601                 pr_err("nvm: memory type not supported\n");
 602                 return -EINVAL;
 603         }
 604
 605         /* calculated values */
 606         geo->sec_per_pl = geo->sec_per_pg * geo->nr_planes;
 607         geo->sec_per_blk = geo->sec_per_pl * geo->pgs_per_blk;
 608         geo->sec_per_lun = geo->sec_per_blk * geo->blks_per_lun;
 609         geo->nr_luns = geo->luns_per_chnl * geo->nr_chnls;
 610
 611         dev->total_secs = geo->nr_luns * geo->sec_per_lun;
 612         dev->lun_map = kcalloc(BITS_TO_LONGS(geo->nr_luns),
 613                                         sizeof(unsigned long), GFP_KERNEL);
 614         if (!dev->lun_map)
 615                 return -ENOMEM;
 616
 617         switch (grp->fmtype) {
 618         case NVM_ID_FMTYPE_SLC:
 619                 if (nvm_init_slc_tbl(dev, grp)) {
 620                         ret = -ENOMEM;
 621                         goto err_fmtype;
 622                 }
 623                 break;
 624         case NVM_ID_FMTYPE_MLC:
 625                 if (nvm_init_mlc_tbl(dev, grp)) {
 626                         ret = -ENOMEM;
 627                         goto err_fmtype;
 628                 }
 629                 break;
 630         default:
 631                 pr_err("nvm: flash type not supported\n");
 632                 ret = -EINVAL;
 633                 goto err_fmtype;
 634         }
 635
 636         mutex_init(&dev->mlock);
 637         spin_lock_init(&dev->lock);
 638
 639         blk_queue_logical_block_size(dev->q, geo->sec_size);
 640
 641         return 0;
 642 err_fmtype:
 643         kfree(dev->lun_map);
 644         return ret;
 645 }
 646
 647 static void nvm_free_mgr(struct nvm_dev *dev)
 648 {
 649         if (!dev->mt)
 650                 return;
 651
 652         dev->mt->unregister_mgr(dev);
 653         dev->mt = NULL;
 654 }
 655
 656 void nvm_free(struct nvm_dev *dev)
 657 {
 658         if (!dev)
 659                 return;
 660
 661         nvm_free_mgr(dev);
 662
 663         if (dev->dma_pool)
 664                 dev->ops->destroy_dma_pool(dev->dma_pool);
 665
 666         kfree(dev->lptbl);
 667         kfree(dev->lun_map);
 668         kfree(dev);
 669 }
 670
 671 static int nvm_init(struct nvm_dev *dev)
 672 {
 673         struct nvm_geo *geo = &dev->geo;
 674         int ret = -EINVAL;
 675
 676         if (!dev->q || !dev->ops)
 677                 return ret;
 678
 679         if (dev->ops->identity(dev, &dev->identity)) {
 680                 pr_err("nvm: device could not be identified\n");
 681                 goto err;
 682         }
 683
 684         pr_debug("nvm: ver:%x nvm_vendor:%x groups:%u\n",
 685                         dev->identity.ver_id, dev->identity.vmnt,
 686                                                         dev->identity.cgrps);
 687
 688         if (dev->identity.ver_id != 1) {
 689                 pr_err("nvm: device not supported by kernel.");
 690                 goto err;
 691         }
 692
 693         if (dev->identity.cgrps != 1) {
 694                 pr_err("nvm: only one group configuration supported.");
 695                 goto err;
 696         }
 697
 698         ret = nvm_core_init(dev);
 699         if (ret) {
 700                 pr_err("nvm: could not initialize core structures.\n");
 701                 goto err;
 702         }
 703
 704         pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
 705                         dev->name, geo->sec_per_pg, geo->nr_planes,
 706                         geo->pgs_per_blk, geo->blks_per_lun,
 707                         geo->nr_luns, geo->nr_chnls);
 708         return 0;
 709 err:
 710         pr_err("nvm: failed to initialize nvm\n");
 711         return ret;
 712 }
 713
 714 struct nvm_dev *nvm_alloc_dev(int node)
 715 {
 716         return kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node);
 717 }
 718 EXPORT_SYMBOL(nvm_alloc_dev);
 719
 720 int nvm_register(struct nvm_dev *dev)
 721 {
 722         int ret;
 723
 724         ret = nvm_init(dev);
 725         if (ret)
 726                 goto err_init;
 727
 728         if (dev->ops->max_phys_sect > 256) {
 729                 pr_info("nvm: max sectors supported is 256.\n");
 730                 ret = -EINVAL;
 731                 goto err_init;
 732         }
 733
 734         if (dev->ops->max_phys_sect > 1) {
 735                 dev->dma_pool = dev->ops->create_dma_pool(dev, "ppalist");
 736                 if (!dev->dma_pool) {
 737                         pr_err("nvm: could not create dma pool\n");
 738                         ret = -ENOMEM;
 739                         goto err_init;
 740                 }
 741         }
 742
 743         if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
 744                 ret = nvm_get_sysblock(dev, &dev->sb);
 745                 if (!ret)
 746                         pr_err("nvm: device not initialized.\n");
 747                 else if (ret < 0)
 748                         pr_err("nvm: err (%d) on device initialization\n", ret);
 749         }
 750
 751         /* register device with a supported media manager */
 752         down_write(&nvm_lock);
 753         if (ret > 0)
 754                 dev->mt = nvm_init_mgr(dev);
 755         list_add(&dev->devices, &nvm_devices);
 756         up_write(&nvm_lock);
 757
 758         return 0;
 759 err_init:
 760         kfree(dev->lun_map);
 761         return ret;
 762 }
 763 EXPORT_SYMBOL(nvm_register);
 764
 765 void nvm_unregister(struct nvm_dev *dev)
 766 {
 767         down_write(&nvm_lock);
 768         list_del(&dev->devices);
 769         up_write(&nvm_lock);
 770
 771         nvm_free(dev);
 772 }
 773 EXPORT_SYMBOL(nvm_unregister);
 774
 775 static int __nvm_configure_create(struct nvm_ioctl_create *create)
 776 {
 777         struct nvm_dev *dev;
 778         struct nvm_ioctl_create_simple *s;
 779
 780         down_write(&nvm_lock);
 781         dev = nvm_find_nvm_dev(create->dev);
 782         up_write(&nvm_lock);
 783
 784         if (!dev) {
 785                 pr_err("nvm: device not found\n");
 786                 return -EINVAL;
 787         }
 788
 789         if (!dev->mt) {
 790                 pr_info("nvm: device has no media manager registered.\n");
 791                 return -ENODEV;
 792         }
 793
 794         if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
 795                 pr_err("nvm: config type not valid\n");
 796                 return -EINVAL;
 797         }
 798         s = &create->conf.s;
 799
 800         if (s->lun_begin > s->lun_end || s->lun_end > dev->geo.nr_luns) {
 801                 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
 802                         s->lun_begin, s->lun_end, dev->geo.nr_luns);
 803                 return -EINVAL;
 804         }
 805
 806         return dev->mt->create_tgt(dev, create);
 807 }
 808
 809 static long nvm_ioctl_info(struct file *file, void __user *arg)
 810 {
 811         struct nvm_ioctl_info *info;
 812         struct nvm_tgt_type *tt;
 813         int tgt_iter = 0;
 814
 815         if (!capable(CAP_SYS_ADMIN))
 816                 return -EPERM;
 817
 818         info = memdup_user(arg, sizeof(struct nvm_ioctl_info));
 819         if (IS_ERR(info))
 820                 return -EFAULT;
 821
 822         info->version[0] = NVM_VERSION_MAJOR;
 823         info->version[1] = NVM_VERSION_MINOR;
 824         info->version[2] = NVM_VERSION_PATCH;
 825
 826         down_write(&nvm_lock);
 827         list_for_each_entry(tt, &nvm_tgt_types, list) {
 828                 struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter];
 829
 830                 tgt->version[0] = tt->version[0];
 831                 tgt->version[1] = tt->version[1];
 832                 tgt->version[2] = tt->version[2];
 833                 strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX);
 834
 835                 tgt_iter++;
 836         }
 837
 838         info->tgtsize = tgt_iter;
 839         up_write(&nvm_lock);
 840
 841         if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) {
 842                 kfree(info);
 843                 return -EFAULT;
 844         }
 845
 846         kfree(info);
 847         return 0;
 848 }
 849
 850 static long nvm_ioctl_get_devices(struct file *file, void __user *arg)
 851 {
 852         struct nvm_ioctl_get_devices *devices;
 853         struct nvm_dev *dev;
 854         int i = 0;
 855
 856         if (!capable(CAP_SYS_ADMIN))
 857                 return -EPERM;
 858
 859         devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL);
 860         if (!devices)
 861                 return -ENOMEM;
 862
 863         down_write(&nvm_lock);
 864         list_for_each_entry(dev, &nvm_devices, devices) {
 865                 struct nvm_ioctl_device_info *info = &devices->info[i];
 866
 867                 sprintf(info->devname, "%s", dev->name);
 868                 if (dev->mt) {
 869                         info->bmversion[0] = dev->mt->version[0];
 870                         info->bmversion[1] = dev->mt->version[1];
 871                         info->bmversion[2] = dev->mt->version[2];
 872                         sprintf(info->bmname, "%s", dev->mt->name);
 873                 } else {
 874                         sprintf(info->bmname, "none");
 875                 }
 876
 877                 i++;
 878                 if (i > 31) {
 879                         pr_err("nvm: max 31 devices can be reported.\n");
 880                         break;
 881                 }
 882         }
 883         up_write(&nvm_lock);
 884
 885         devices->nr_devices = i;
 886
 887         if (copy_to_user(arg, devices,
 888                          sizeof(struct nvm_ioctl_get_devices))) {
 889                 kfree(devices);
 890                 return -EFAULT;
 891         }
 892
 893         kfree(devices);
 894         return 0;
 895 }
 896
 897 static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
 898 {
 899         struct nvm_ioctl_create create;
 900
 901         if (!capable(CAP_SYS_ADMIN))
 902                 return -EPERM;
 903
 904         if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
 905                 return -EFAULT;
 906
 907         create.dev[DISK_NAME_LEN - 1] = '\0';
 908         create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
 909         create.tgtname[DISK_NAME_LEN - 1] = '\0';
 910
 911         if (create.flags != 0) {
 912                 pr_err("nvm: no flags supported\n");
 913                 return -EINVAL;
 914         }
 915
 916         return __nvm_configure_create(&create);
 917 }
 918
 919 static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
 920 {
 921         struct nvm_ioctl_remove remove;
 922         struct nvm_dev *dev;
 923         int ret = 0;
 924
 925         if (!capable(CAP_SYS_ADMIN))
 926                 return -EPERM;
 927
 928         if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove)))
 929                 return -EFAULT;
 930
 931         remove.tgtname[DISK_NAME_LEN - 1] = '\0';
 932
 933         if (remove.flags != 0) {
 934                 pr_err("nvm: no flags supported\n");
 935                 return -EINVAL;
 936         }
 937
 938         list_for_each_entry(dev, &nvm_devices, devices) {
 939                 ret = dev->mt->remove_tgt(dev, &remove);
 940                 if (!ret)
 941                         break;
 942         }
 943
 944         return ret;
 945 }
 946
 947 static void nvm_setup_nvm_sb_info(struct nvm_sb_info *info)
 948 {
 949         info->seqnr = 1;
 950         info->erase_cnt = 0;
 951         info->version = 1;
 952 }
 953
 954 static long __nvm_ioctl_dev_init(struct nvm_ioctl_dev_init *init)
 955 {
 956         struct nvm_dev *dev;
 957         struct nvm_sb_info info;
 958         int ret;
 959
 960         down_write(&nvm_lock);
 961         dev = nvm_find_nvm_dev(init->dev);
 962         up_write(&nvm_lock);
 963         if (!dev) {
 964                 pr_err("nvm: device not found\n");
 965                 return -EINVAL;
 966         }
 967
 968         nvm_setup_nvm_sb_info(&info);
 969
 970         strncpy(info.mmtype, init->mmtype, NVM_MMTYPE_LEN);
 971         info.fs_ppa.ppa = -1;
 972
 973         if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
 974                 ret = nvm_init_sysblock(dev, &info);
 975                 if (ret)
 976                         return ret;
 977         }
 978
 979         memcpy(&dev->sb, &info, sizeof(struct nvm_sb_info));
 980
 981         down_write(&nvm_lock);
 982         dev->mt = nvm_init_mgr(dev);
 983         up_write(&nvm_lock);
 984
 985         return 0;
 986 }
 987
 988 static long nvm_ioctl_dev_init(struct file *file, void __user *arg)
 989 {
 990         struct nvm_ioctl_dev_init init;
 991
 992         if (!capable(CAP_SYS_ADMIN))
 993                 return -EPERM;
 994
 995         if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init)))
 996                 return -EFAULT;
 997
 998         if (init.flags != 0) {
 999                 pr_err("nvm: no flags supported\n");
1000                 return -EINVAL;
1001         }
1002
1003         init.dev[DISK_NAME_LEN - 1] = '\0';
1004
1005         return __nvm_ioctl_dev_init(&init);
1006 }
1007
1008 static long nvm_ioctl_dev_factory(struct file *file, void __user *arg)
1009 {
1010         struct nvm_ioctl_dev_factory fact;
1011         struct nvm_dev *dev;
1012
1013         if (!capable(CAP_SYS_ADMIN))
1014                 return -EPERM;
1015
1016         if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory)))
1017                 return -EFAULT;
1018
1019         fact.dev[DISK_NAME_LEN - 1] = '\0';
1020
1021         if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1))
1022                 return -EINVAL;
1023
1024         down_write(&nvm_lock);
1025         dev = nvm_find_nvm_dev(fact.dev);
1026         up_write(&nvm_lock);
1027         if (!dev) {
1028                 pr_err("nvm: device not found\n");
1029                 return -EINVAL;
1030         }
1031
1032         nvm_free_mgr(dev);
1033
1034         if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT)
1035                 return nvm_dev_factory(dev, fact.flags);
1036
1037         return 0;
1038 }
1039
1040 static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
1041 {
1042         void __user *argp = (void __user *)arg;
1043
1044         switch (cmd) {
1045         case NVM_INFO:
1046                 return nvm_ioctl_info(file, argp);
1047         case NVM_GET_DEVICES:
1048                 return nvm_ioctl_get_devices(file, argp);
1049         case NVM_DEV_CREATE:
1050                 return nvm_ioctl_dev_create(file, argp);
1051         case NVM_DEV_REMOVE:
1052                 return nvm_ioctl_dev_remove(file, argp);
1053         case NVM_DEV_INIT:
1054                 return nvm_ioctl_dev_init(file, argp);
1055         case NVM_DEV_FACTORY:
1056                 return nvm_ioctl_dev_factory(file, argp);
1057         }
1058         return 0;
1059 }
1060
1061 static const struct file_operations _ctl_fops = {
1062         .open = nonseekable_open,
1063         .unlocked_ioctl = nvm_ctl_ioctl,
1064         .owner = THIS_MODULE,
1065         .llseek  = noop_llseek,
1066 };
1067
1068 static struct miscdevice _nvm_misc = {
1069         .minor          = MISC_DYNAMIC_MINOR,
1070         .name           = "lightnvm",
1071         .nodename       = "lightnvm/control",
1072         .fops           = &_ctl_fops,
1073 };
1074 module_misc_device(_nvm_misc);
1075
1076 MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR);
1077
1078 MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>");
1079 MODULE_LICENSE("GPL v2");
1080 MODULE_VERSION("0.1");