drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/nand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_ECC_BLOCK_512       BIT(5)
 149 #define NFC_RANDOM_EN           BIT(9)
 150 #define NFC_RANDOM_DIRECTION    BIT(10)
 151 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 152 #define NFC_ECC_MODE(x)         ((x) << 12)
 153 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 154 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 155
 156 /* define bit use in NFC_ECC_ST */
 157 #define NFC_ECC_ERR(x)          BIT(x)
 158 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 159 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 160 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 161
 162 #define NFC_DEFAULT_TIMEOUT_MS  1000
 163
 164 #define NFC_SRAM_SIZE           1024
 165
 166 #define NFC_MAX_CS              7
 167
 168 /*
 169  * Ready/Busy detection type: describes the Ready/Busy detection modes
 170  *
 171  * @RB_NONE:    no external detection available, rely on STATUS command
 172  *              and software timeouts
 173  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 174  *              pin of the NAND flash chip must be connected to one of the
 175  *              native NAND R/B pins (those which can be muxed to the NAND
 176  *              Controller)
 177  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 178  *              pin of the NAND flash chip must be connected to a GPIO capable
 179  *              pin.
 180  */
 181 enum sunxi_nand_rb_type {
 182         RB_NONE,
 183         RB_NATIVE,
 184         RB_GPIO,
 185 };
 186
 187 /*
 188  * Ready/Busy structure: stores information related to Ready/Busy detection
 189  *
 190  * @type:       the Ready/Busy detection mode
 191  * @info:       information related to the R/B detection mode. Either a gpio
 192  *              id or a native R/B id (those supported by the NAND controller).
 193  */
 194 struct sunxi_nand_rb {
 195         enum sunxi_nand_rb_type type;
 196         union {
 197                 int gpio;
 198                 int nativeid;
 199         } info;
 200 };
 201
 202 /*
 203  * Chip Select structure: stores information related to NAND Chip Select
 204  *
 205  * @cs:         the NAND CS id used to communicate with a NAND Chip
 206  * @rb:         the Ready/Busy description
 207  */
 208 struct sunxi_nand_chip_sel {
 209         u8 cs;
 210         struct sunxi_nand_rb rb;
 211 };
 212
 213 /*
 214  * sunxi HW ECC infos: stores information related to HW ECC support
 215  *
 216  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 217  */
 218 struct sunxi_nand_hw_ecc {
 219         int mode;
 220 };
 221
 222 /*
 223  * NAND chip structure: stores NAND chip device related information
 224  *
 225  * @node:               used to store NAND chips into a list
 226  * @nand:               base NAND chip structure
 227  * @mtd:                base MTD structure
 228  * @clk_rate:           clk_rate required for this NAND chip
 229  * @timing_cfg          TIMING_CFG register value for this NAND chip
 230  * @selected:           current active CS
 231  * @nsels:              number of CS lines required by the NAND chip
 232  * @sels:               array of CS lines descriptions
 233  */
 234 struct sunxi_nand_chip {
 235         struct list_head node;
 236         struct nand_chip nand;
 237         unsigned long clk_rate;
 238         u32 timing_cfg;
 239         u32 timing_ctl;
 240         int selected;
 241         int addr_cycles;
 242         u32 addr[2];
 243         int cmd_cycles;
 244         u8 cmd[2];
 245         int nsels;
 246         struct sunxi_nand_chip_sel sels[0];
 247 };
 248
 249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 250 {
 251         return container_of(nand, struct sunxi_nand_chip, nand);
 252 }
 253
 254 /*
 255  * NAND Controller structure: stores sunxi NAND controller information
 256  *
 257  * @controller:         base controller structure
 258  * @dev:                parent device (used to print error messages)
 259  * @regs:               NAND controller registers
 260  * @ahb_clk:            NAND Controller AHB clock
 261  * @mod_clk:            NAND Controller mod clock
 262  * @assigned_cs:        bitmask describing already assigned CS lines
 263  * @clk_rate:           NAND controller current clock rate
 264  * @chips:              a list containing all the NAND chips attached to
 265  *                      this NAND controller
 266  * @complete:           a completion object used to wait for NAND
 267  *                      controller events
 268  */
 269 struct sunxi_nfc {
 270         struct nand_hw_control controller;
 271         struct device *dev;
 272         void __iomem *regs;
 273         struct clk *ahb_clk;
 274         struct clk *mod_clk;
 275         struct reset_control *reset;
 276         unsigned long assigned_cs;
 277         unsigned long clk_rate;
 278         struct list_head chips;
 279         struct completion complete;
 280         struct dma_chan *dmac;
 281 };
 282
 283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 284 {
 285         return container_of(ctrl, struct sunxi_nfc, controller);
 286 }
 287
 288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 289 {
 290         struct sunxi_nfc *nfc = dev_id;
 291         u32 st = readl(nfc->regs + NFC_REG_ST);
 292         u32 ien = readl(nfc->regs + NFC_REG_INT);
 293
 294         if (!(ien & st))
 295                 return IRQ_NONE;
 296
 297         if ((ien & st) == ien)
 298                 complete(&nfc->complete);
 299
 300         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 301         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 307                                  bool use_polling, unsigned int timeout_ms)
 308 {
 309         int ret;
 310
 311         if (events & ~NFC_INT_MASK)
 312                 return -EINVAL;
 313
 314         if (!timeout_ms)
 315                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 316
 317         if (!use_polling) {
 318                 init_completion(&nfc->complete);
 319
 320                 writel(events, nfc->regs + NFC_REG_INT);
 321
 322                 ret = wait_for_completion_timeout(&nfc->complete,
 323                                                 msecs_to_jiffies(timeout_ms));
 324                 if (!ret)
 325                         ret = -ETIMEDOUT;
 326                 else
 327                         ret = 0;
 328
 329                 writel(0, nfc->regs + NFC_REG_INT);
 330         } else {
 331                 u32 status;
 332
 333                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 334                                          (status & events) == events, 1,
 335                                          timeout_ms * 1000);
 336         }
 337
 338         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 339
 340         if (ret)
 341                 dev_err(nfc->dev, "wait interrupt timedout\n");
 342
 343         return ret;
 344 }
 345
 346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 347 {
 348         u32 status;
 349         int ret;
 350
 351         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 352                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 353                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 354         if (ret)
 355                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 356
 357         return ret;
 358 }
 359
 360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 361 {
 362         u32 ctl;
 363         int ret;
 364
 365         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 366         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 367
 368         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 369                                  !(ctl & NFC_RESET), 1,
 370                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 371         if (ret)
 372                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 373
 374         return ret;
 375 }
 376
 377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 378                                     int chunksize, int nchunks,
 379                                     enum dma_data_direction ddir,
 380                                     struct scatterlist *sg)
 381 {
 382         struct nand_chip *nand = mtd_to_nand(mtd);
 383         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 384         struct dma_async_tx_descriptor *dmad;
 385         enum dma_transfer_direction tdir;
 386         dma_cookie_t dmat;
 387         int ret;
 388
 389         if (ddir == DMA_FROM_DEVICE)
 390                 tdir = DMA_DEV_TO_MEM;
 391         else
 392                 tdir = DMA_MEM_TO_DEV;
 393
 394         sg_init_one(sg, buf, nchunks * chunksize);
 395         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 396         if (!ret)
 397                 return -ENOMEM;
 398
 399         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 400         if (!dmad) {
 401                 ret = -EINVAL;
 402                 goto err_unmap_buf;
 403         }
 404
 405         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 406                nfc->regs + NFC_REG_CTL);
 407         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 408         writel(chunksize, nfc->regs + NFC_REG_CNT);
 409         dmat = dmaengine_submit(dmad);
 410
 411         ret = dma_submit_error(dmat);
 412         if (ret)
 413                 goto err_clr_dma_flag;
 414
 415         return 0;
 416
 417 err_clr_dma_flag:
 418         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 419                nfc->regs + NFC_REG_CTL);
 420
 421 err_unmap_buf:
 422         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 423         return ret;
 424 }
 425
 426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 427                                      enum dma_data_direction ddir,
 428                                      struct scatterlist *sg)
 429 {
 430         struct nand_chip *nand = mtd_to_nand(mtd);
 431         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 432
 433         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 434         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 435                nfc->regs + NFC_REG_CTL);
 436 }
 437
 438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 439 {
 440         struct nand_chip *nand = mtd_to_nand(mtd);
 441         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 442         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 443         struct sunxi_nand_rb *rb;
 444         int ret;
 445
 446         if (sunxi_nand->selected < 0)
 447                 return 0;
 448
 449         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 450
 451         switch (rb->type) {
 452         case RB_NATIVE:
 453                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 454                          NFC_RB_STATE(rb->info.nativeid));
 455                 break;
 456         case RB_GPIO:
 457                 ret = gpio_get_value(rb->info.gpio);
 458                 break;
 459         case RB_NONE:
 460         default:
 461                 ret = 0;
 462                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 463                 break;
 464         }
 465
 466         return ret;
 467 }
 468
 469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 470 {
 471         struct nand_chip *nand = mtd_to_nand(mtd);
 472         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 473         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 474         struct sunxi_nand_chip_sel *sel;
 475         u32 ctl;
 476
 477         if (chip > 0 && chip >= sunxi_nand->nsels)
 478                 return;
 479
 480         if (chip == sunxi_nand->selected)
 481                 return;
 482
 483         ctl = readl(nfc->regs + NFC_REG_CTL) &
 484               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 485
 486         if (chip >= 0) {
 487                 sel = &sunxi_nand->sels[chip];
 488
 489                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 490                        NFC_PAGE_SHIFT(nand->page_shift);
 491                 if (sel->rb.type == RB_NONE) {
 492                         nand->dev_ready = NULL;
 493                 } else {
 494                         nand->dev_ready = sunxi_nfc_dev_ready;
 495                         if (sel->rb.type == RB_NATIVE)
 496                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 497                 }
 498
 499                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 500
 501                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 502                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 503                         nfc->clk_rate = sunxi_nand->clk_rate;
 504                 }
 505         }
 506
 507         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 508         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 509         writel(ctl, nfc->regs + NFC_REG_CTL);
 510
 511         sunxi_nand->selected = chip;
 512 }
 513
 514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 515 {
 516         struct nand_chip *nand = mtd_to_nand(mtd);
 517         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 518         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 519         int ret;
 520         int cnt;
 521         int offs = 0;
 522         u32 tmp;
 523
 524         while (len > offs) {
 525                 cnt = min(len - offs, NFC_SRAM_SIZE);
 526
 527                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 528                 if (ret)
 529                         break;
 530
 531                 writel(cnt, nfc->regs + NFC_REG_CNT);
 532                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 533                 writel(tmp, nfc->regs + NFC_REG_CMD);
 534
 535                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 536                 if (ret)
 537                         break;
 538
 539                 if (buf)
 540                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 541                                       cnt);
 542                 offs += cnt;
 543         }
 544 }
 545
 546 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 547                                 int len)
 548 {
 549         struct nand_chip *nand = mtd_to_nand(mtd);
 550         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 551         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 552         int ret;
 553         int cnt;
 554         int offs = 0;
 555         u32 tmp;
 556
 557         while (len > offs) {
 558                 cnt = min(len - offs, NFC_SRAM_SIZE);
 559
 560                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 561                 if (ret)
 562                         break;
 563
 564                 writel(cnt, nfc->regs + NFC_REG_CNT);
 565                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 566                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 567                       NFC_ACCESS_DIR;
 568                 writel(tmp, nfc->regs + NFC_REG_CMD);
 569
 570                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 571                 if (ret)
 572                         break;
 573
 574                 offs += cnt;
 575         }
 576 }
 577
 578 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 579 {
 580         uint8_t ret;
 581
 582         sunxi_nfc_read_buf(mtd, &ret, 1);
 583
 584         return ret;
 585 }
 586
 587 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 588                                unsigned int ctrl)
 589 {
 590         struct nand_chip *nand = mtd_to_nand(mtd);
 591         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 592         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 593         int ret;
 594
 595         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 596         if (ret)
 597                 return;
 598
 599         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 600             !(ctrl & (NAND_CLE | NAND_ALE))) {
 601                 u32 cmd = 0;
 602
 603                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 604                         return;
 605
 606                 if (sunxi_nand->cmd_cycles--)
 607                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 608
 609                 if (sunxi_nand->cmd_cycles--) {
 610                         cmd |= NFC_SEND_CMD2;
 611                         writel(sunxi_nand->cmd[1],
 612                                nfc->regs + NFC_REG_RCMD_SET);
 613                 }
 614
 615                 sunxi_nand->cmd_cycles = 0;
 616
 617                 if (sunxi_nand->addr_cycles) {
 618                         cmd |= NFC_SEND_ADR |
 619                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 620                         writel(sunxi_nand->addr[0],
 621                                nfc->regs + NFC_REG_ADDR_LOW);
 622                 }
 623
 624                 if (sunxi_nand->addr_cycles > 4)
 625                         writel(sunxi_nand->addr[1],
 626                                nfc->regs + NFC_REG_ADDR_HIGH);
 627
 628                 writel(cmd, nfc->regs + NFC_REG_CMD);
 629                 sunxi_nand->addr[0] = 0;
 630                 sunxi_nand->addr[1] = 0;
 631                 sunxi_nand->addr_cycles = 0;
 632                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 633         }
 634
 635         if (ctrl & NAND_CLE) {
 636                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 637         } else if (ctrl & NAND_ALE) {
 638                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 639                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 640                 sunxi_nand->addr_cycles++;
 641         }
 642 }
 643
 644 /* These seed values have been extracted from Allwinner's BSP */
 645 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 646         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 647         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 648         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 649         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 650         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 651         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 652         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 653         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 654         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 655         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 656         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 657         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 658         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 659         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 660         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 661         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 662 };
 663
 664 /*
 665  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 666  * have been generated using
 667  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 668  * the randomizer engine does internally before de/scrambling OOB data.
 669  *
 670  * Those tables are statically defined to avoid calculating randomizer state
 671  * at runtime.
 672  */
 673 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 674         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 675         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 676         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 677         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 678         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 679         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 680         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 681         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 682         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 683         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 684         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 685         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 686         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 687         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 688         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 689         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 690 };
 691
 692 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 693         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 694         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 695         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 696         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 697         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 698         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 699         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 700         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 701         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 702         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 703         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 704         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 705         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 706         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 707         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 708         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 709 };
 710
 711 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 712 {
 713         state &= 0x7fff;
 714
 715         /*
 716          * This loop is just a simple implementation of a Fibonacci LFSR using
 717          * the x16 + x15 + 1 polynomial.
 718          */
 719         while (count--)
 720                 state = ((state >> 1) |
 721                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 722
 723         return state;
 724 }
 725
 726 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 727 {
 728         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 729         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 730
 731         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 732                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 733
 734         if (ecc) {
 735                 if (mtd->ecc_step_size == 512)
 736                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 737                 else
 738                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 739         }
 740
 741         return seeds[page % mod];
 742 }
 743
 744 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 745                                         int page, bool ecc)
 746 {
 747         struct nand_chip *nand = mtd_to_nand(mtd);
 748         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 749         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 750         u16 state;
 751
 752         if (!(nand->options & NAND_NEED_SCRAMBLING))
 753                 return;
 754
 755         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 756         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 757         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 758         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 759 }
 760
 761 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 762 {
 763         struct nand_chip *nand = mtd_to_nand(mtd);
 764         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 765
 766         if (!(nand->options & NAND_NEED_SCRAMBLING))
 767                 return;
 768
 769         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 770                nfc->regs + NFC_REG_ECC_CTL);
 771 }
 772
 773 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 774 {
 775         struct nand_chip *nand = mtd_to_nand(mtd);
 776         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 777
 778         if (!(nand->options & NAND_NEED_SCRAMBLING))
 779                 return;
 780
 781         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 782                nfc->regs + NFC_REG_ECC_CTL);
 783 }
 784
 785 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 786 {
 787         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 788
 789         bbm[0] ^= state;
 790         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 791 }
 792
 793 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 794                                            const uint8_t *buf, int len,
 795                                            bool ecc, int page)
 796 {
 797         sunxi_nfc_randomizer_config(mtd, page, ecc);
 798         sunxi_nfc_randomizer_enable(mtd);
 799         sunxi_nfc_write_buf(mtd, buf, len);
 800         sunxi_nfc_randomizer_disable(mtd);
 801 }
 802
 803 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 804                                           int len, bool ecc, int page)
 805 {
 806         sunxi_nfc_randomizer_config(mtd, page, ecc);
 807         sunxi_nfc_randomizer_enable(mtd);
 808         sunxi_nfc_read_buf(mtd, buf, len);
 809         sunxi_nfc_randomizer_disable(mtd);
 810 }
 811
 812 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 813 {
 814         struct nand_chip *nand = mtd_to_nand(mtd);
 815         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 816         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 817         u32 ecc_ctl;
 818
 819         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 820         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 821                      NFC_ECC_BLOCK_SIZE_MSK);
 822         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 823                    NFC_ECC_PIPELINE;
 824
 825         if (nand->ecc.size == 512)
 826                 ecc_ctl |= NFC_ECC_BLOCK_512;
 827
 828         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 829 }
 830
 831 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 832 {
 833         struct nand_chip *nand = mtd_to_nand(mtd);
 834         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 835
 836         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 837                nfc->regs + NFC_REG_ECC_CTL);
 838 }
 839
 840 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 841 {
 842         buf[0] = user_data;
 843         buf[1] = user_data >> 8;
 844         buf[2] = user_data >> 16;
 845         buf[3] = user_data >> 24;
 846 }
 847
 848 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 849 {
 850         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 851 }
 852
 853 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 854                                                 int step, bool bbm, int page)
 855 {
 856         struct nand_chip *nand = mtd_to_nand(mtd);
 857         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 858
 859         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 860                                    oob);
 861
 862         /* De-randomize the Bad Block Marker. */
 863         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 864                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 865 }
 866
 867 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 868                                                 const u8 *oob, int step,
 869                                                 bool bbm, int page)
 870 {
 871         struct nand_chip *nand = mtd_to_nand(mtd);
 872         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 873         u8 user_data[4];
 874
 875         /* Randomize the Bad Block Marker. */
 876         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 877                 memcpy(user_data, oob, sizeof(user_data));
 878                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 879                 oob = user_data;
 880         }
 881
 882         writel(sunxi_nfc_buf_to_user_data(oob),
 883                nfc->regs + NFC_REG_USER_DATA(step));
 884 }
 885
 886 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 887                                           unsigned int *max_bitflips, int ret)
 888 {
 889         if (ret < 0) {
 890                 mtd->ecc_stats.failed++;
 891         } else {
 892                 mtd->ecc_stats.corrected += ret;
 893                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 894         }
 895 }
 896
 897 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 898                                     int step, u32 status, bool *erased)
 899 {
 900         struct nand_chip *nand = mtd_to_nand(mtd);
 901         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 902         struct nand_ecc_ctrl *ecc = &nand->ecc;
 903         u32 tmp;
 904
 905         *erased = false;
 906
 907         if (status & NFC_ECC_ERR(step))
 908                 return -EBADMSG;
 909
 910         if (status & NFC_ECC_PAT_FOUND(step)) {
 911                 u8 pattern;
 912
 913                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 914                         pattern = 0x0;
 915                 } else {
 916                         pattern = 0xff;
 917                         *erased = true;
 918                 }
 919
 920                 if (data)
 921                         memset(data, pattern, ecc->size);
 922
 923                 if (oob)
 924                         memset(oob, pattern, ecc->bytes + 4);
 925
 926                 return 0;
 927         }
 928
 929         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 930
 931         return NFC_ECC_ERR_CNT(step, tmp);
 932 }
 933
 934 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 935                                        u8 *data, int data_off,
 936                                        u8 *oob, int oob_off,
 937                                        int *cur_off,
 938                                        unsigned int *max_bitflips,
 939                                        bool bbm, bool oob_required, int page)
 940 {
 941         struct nand_chip *nand = mtd_to_nand(mtd);
 942         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 943         struct nand_ecc_ctrl *ecc = &nand->ecc;
 944         int raw_mode = 0;
 945         bool erased;
 946         int ret;
 947
 948         if (*cur_off != data_off)
 949                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 950
 951         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 952
 953         if (data_off + ecc->size != oob_off)
 954                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 955
 956         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 957         if (ret)
 958                 return ret;
 959
 960         sunxi_nfc_randomizer_enable(mtd);
 961         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 962                nfc->regs + NFC_REG_CMD);
 963
 964         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 965         sunxi_nfc_randomizer_disable(mtd);
 966         if (ret)
 967                 return ret;
 968
 969         *cur_off = oob_off + ecc->bytes + 4;
 970
 971         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 972                                        readl(nfc->regs + NFC_REG_ECC_ST),
 973                                        &erased);
 974         if (erased)
 975                 return 1;
 976
 977         if (ret < 0) {
 978                 /*
 979                  * Re-read the data with the randomizer disabled to identify
 980                  * bitflips in erased pages.
 981                  */
 982                 if (nand->options & NAND_NEED_SCRAMBLING) {
 983                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 984                         nand->read_buf(mtd, data, ecc->size);
 985                 } else {
 986                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 987                                       ecc->size);
 988                 }
 989
 990                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 991                 nand->read_buf(mtd, oob, ecc->bytes + 4);
 992
 993                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 994                                                   oob, ecc->bytes + 4,
 995                                                   NULL, 0, ecc->strength);
 996                 if (ret >= 0)
 997                         raw_mode = 1;
 998         } else {
 999                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
1000
1001                 if (oob_required) {
1002                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1003                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1004                                                       true, page);
1005
1006                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1007                                                             bbm, page);
1008                 }
1009         }
1010
1011         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1012
1013         return raw_mode;
1014 }
1015
1016 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1017                                             u8 *oob, int *cur_off,
1018                                             bool randomize, int page)
1019 {
1020         struct nand_chip *nand = mtd_to_nand(mtd);
1021         struct nand_ecc_ctrl *ecc = &nand->ecc;
1022         int offset = ((ecc->bytes + 4) * ecc->steps);
1023         int len = mtd->oobsize - offset;
1024
1025         if (len <= 0)
1026                 return;
1027
1028         if (!cur_off || *cur_off != offset)
1029                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1030                               offset + mtd->writesize, -1);
1031
1032         if (!randomize)
1033                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1034         else
1035                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1036                                               false, page);
1037
1038         if (cur_off)
1039                 *cur_off = mtd->oobsize + mtd->writesize;
1040 }
1041
1042 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1043                                             int oob_required, int page,
1044                                             int nchunks)
1045 {
1046         struct nand_chip *nand = mtd_to_nand(mtd);
1047         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1048         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1049         struct nand_ecc_ctrl *ecc = &nand->ecc;
1050         unsigned int max_bitflips = 0;
1051         int ret, i, raw_mode = 0;
1052         struct scatterlist sg;
1053         u32 status;
1054
1055         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1056         if (ret)
1057                 return ret;
1058
1059         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1060                                        DMA_FROM_DEVICE, &sg);
1061         if (ret)
1062                 return ret;
1063
1064         sunxi_nfc_hw_ecc_enable(mtd);
1065         sunxi_nfc_randomizer_config(mtd, page, false);
1066         sunxi_nfc_randomizer_enable(mtd);
1067
1068         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1069                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1070
1071         dma_async_issue_pending(nfc->dmac);
1072
1073         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1074                nfc->regs + NFC_REG_CMD);
1075
1076         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1077         if (ret)
1078                 dmaengine_terminate_all(nfc->dmac);
1079
1080         sunxi_nfc_randomizer_disable(mtd);
1081         sunxi_nfc_hw_ecc_disable(mtd);
1082
1083         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1084
1085         if (ret)
1086                 return ret;
1087
1088         status = readl(nfc->regs + NFC_REG_ECC_ST);
1089
1090         for (i = 0; i < nchunks; i++) {
1091                 int data_off = i * ecc->size;
1092                 int oob_off = i * (ecc->bytes + 4);
1093                 u8 *data = buf + data_off;
1094                 u8 *oob = nand->oob_poi + oob_off;
1095                 bool erased;
1096
1097                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1098                                                oob_required ? oob : NULL,
1099                                                i, status, &erased);
1100
1101                 /* ECC errors are handled in the second loop. */
1102                 if (ret < 0)
1103                         continue;
1104
1105                 if (oob_required && !erased) {
1106                         /* TODO: use DMA to retrieve OOB */
1107                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1108                                       mtd->writesize + oob_off, -1);
1109                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1110
1111                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1112                                                             !i, page);
1113                 }
1114
1115                 if (erased)
1116                         raw_mode = 1;
1117
1118                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1119         }
1120
1121         if (status & NFC_ECC_ERR_MSK) {
1122                 for (i = 0; i < nchunks; i++) {
1123                         int data_off = i * ecc->size;
1124                         int oob_off = i * (ecc->bytes + 4);
1125                         u8 *data = buf + data_off;
1126                         u8 *oob = nand->oob_poi + oob_off;
1127
1128                         if (!(status & NFC_ECC_ERR(i)))
1129                                 continue;
1130
1131                         /*
1132                          * Re-read the data with the randomizer disabled to
1133                          * identify bitflips in erased pages.
1134                          */
1135                         if (randomized) {
1136                                 /* TODO: use DMA to read page in raw mode */
1137                                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1138                                               data_off, -1);
1139                                 nand->read_buf(mtd, data, ecc->size);
1140                         }
1141
1142                         /* TODO: use DMA to retrieve OOB */
1143                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1144                                       mtd->writesize + oob_off, -1);
1145                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1146
1147                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1148                                                           oob, ecc->bytes + 4,
1149                                                           NULL, 0,
1150                                                           ecc->strength);
1151                         if (ret >= 0)
1152                                 raw_mode = 1;
1153
1154                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1155                 }
1156         }
1157
1158         if (oob_required)
1159                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1160                                                 NULL, !raw_mode,
1161                                                 page);
1162
1163         return max_bitflips;
1164 }
1165
1166 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1167                                         const u8 *data, int data_off,
1168                                         const u8 *oob, int oob_off,
1169                                         int *cur_off, bool bbm,
1170                                         int page)
1171 {
1172         struct nand_chip *nand = mtd_to_nand(mtd);
1173         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1174         struct nand_ecc_ctrl *ecc = &nand->ecc;
1175         int ret;
1176
1177         if (data_off != *cur_off)
1178                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1179
1180         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1181
1182         if (data_off + ecc->size != oob_off)
1183                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1184
1185         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1186         if (ret)
1187                 return ret;
1188
1189         sunxi_nfc_randomizer_enable(mtd);
1190         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1191
1192         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1193                NFC_ACCESS_DIR | NFC_ECC_OP,
1194                nfc->regs + NFC_REG_CMD);
1195
1196         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1197         sunxi_nfc_randomizer_disable(mtd);
1198         if (ret)
1199                 return ret;
1200
1201         *cur_off = oob_off + ecc->bytes + 4;
1202
1203         return 0;
1204 }
1205
1206 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1207                                              u8 *oob, int *cur_off,
1208                                              int page)
1209 {
1210         struct nand_chip *nand = mtd_to_nand(mtd);
1211         struct nand_ecc_ctrl *ecc = &nand->ecc;
1212         int offset = ((ecc->bytes + 4) * ecc->steps);
1213         int len = mtd->oobsize - offset;
1214
1215         if (len <= 0)
1216                 return;
1217
1218         if (!cur_off || *cur_off != offset)
1219                 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1220                               offset + mtd->writesize, -1);
1221
1222         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1223
1224         if (cur_off)
1225                 *cur_off = mtd->oobsize + mtd->writesize;
1226 }
1227
1228 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1229                                       struct nand_chip *chip, uint8_t *buf,
1230                                       int oob_required, int page)
1231 {
1232         struct nand_ecc_ctrl *ecc = &chip->ecc;
1233         unsigned int max_bitflips = 0;
1234         int ret, i, cur_off = 0;
1235         bool raw_mode = false;
1236
1237         sunxi_nfc_hw_ecc_enable(mtd);
1238
1239         for (i = 0; i < ecc->steps; i++) {
1240                 int data_off = i * ecc->size;
1241                 int oob_off = i * (ecc->bytes + 4);
1242                 u8 *data = buf + data_off;
1243                 u8 *oob = chip->oob_poi + oob_off;
1244
1245                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1246                                                   oob_off + mtd->writesize,
1247                                                   &cur_off, &max_bitflips,
1248                                                   !i, oob_required, page);
1249                 if (ret < 0)
1250                         return ret;
1251                 else if (ret)
1252                         raw_mode = true;
1253         }
1254
1255         if (oob_required)
1256                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1257                                                 !raw_mode, page);
1258
1259         sunxi_nfc_hw_ecc_disable(mtd);
1260
1261         return max_bitflips;
1262 }
1263
1264 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1265                                           struct nand_chip *chip, u8 *buf,
1266                                           int oob_required, int page)
1267 {
1268         int ret;
1269
1270         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1271                                                chip->ecc.steps);
1272         if (ret >= 0)
1273                 return ret;
1274
1275         /* Fallback to PIO mode */
1276         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1277
1278         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1279 }
1280
1281 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1282                                          struct nand_chip *chip,
1283                                          u32 data_offs, u32 readlen,
1284                                          u8 *bufpoi, int page)
1285 {
1286         struct nand_ecc_ctrl *ecc = &chip->ecc;
1287         int ret, i, cur_off = 0;
1288         unsigned int max_bitflips = 0;
1289
1290         sunxi_nfc_hw_ecc_enable(mtd);
1291
1292         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1293         for (i = data_offs / ecc->size;
1294              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1295                 int data_off = i * ecc->size;
1296                 int oob_off = i * (ecc->bytes + 4);
1297                 u8 *data = bufpoi + data_off;
1298                 u8 *oob = chip->oob_poi + oob_off;
1299
1300                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1301                                                   oob,
1302                                                   oob_off + mtd->writesize,
1303                                                   &cur_off, &max_bitflips, !i,
1304                                                   false, page);
1305                 if (ret < 0)
1306                         return ret;
1307         }
1308
1309         sunxi_nfc_hw_ecc_disable(mtd);
1310
1311         return max_bitflips;
1312 }
1313
1314 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1315                                              struct nand_chip *chip,
1316                                              u32 data_offs, u32 readlen,
1317                                              u8 *buf, int page)
1318 {
1319         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1320         int ret;
1321
1322         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1323         if (ret >= 0)
1324                 return ret;
1325
1326         /* Fallback to PIO mode */
1327         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1328
1329         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1330                                              buf, page);
1331 }
1332
1333 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1334                                        struct nand_chip *chip,
1335                                        const uint8_t *buf, int oob_required,
1336                                        int page)
1337 {
1338         struct nand_ecc_ctrl *ecc = &chip->ecc;
1339         int ret, i, cur_off = 0;
1340
1341         sunxi_nfc_hw_ecc_enable(mtd);
1342
1343         for (i = 0; i < ecc->steps; i++) {
1344                 int data_off = i * ecc->size;
1345                 int oob_off = i * (ecc->bytes + 4);
1346                 const u8 *data = buf + data_off;
1347                 const u8 *oob = chip->oob_poi + oob_off;
1348
1349                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1350                                                    oob_off + mtd->writesize,
1351                                                    &cur_off, !i, page);
1352                 if (ret)
1353                         return ret;
1354         }
1355
1356         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1357                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1358                                                  &cur_off, page);
1359
1360         sunxi_nfc_hw_ecc_disable(mtd);
1361
1362         return 0;
1363 }
1364
1365 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1366                                           struct nand_chip *chip,
1367                                           u32 data_offs, u32 data_len,
1368                                           const u8 *buf, int oob_required,
1369                                           int page)
1370 {
1371         struct nand_ecc_ctrl *ecc = &chip->ecc;
1372         int ret, i, cur_off = 0;
1373
1374         sunxi_nfc_hw_ecc_enable(mtd);
1375
1376         for (i = data_offs / ecc->size;
1377              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1378                 int data_off = i * ecc->size;
1379                 int oob_off = i * (ecc->bytes + 4);
1380                 const u8 *data = buf + data_off;
1381                 const u8 *oob = chip->oob_poi + oob_off;
1382
1383                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1384                                                    oob_off + mtd->writesize,
1385                                                    &cur_off, !i, page);
1386                 if (ret)
1387                         return ret;
1388         }
1389
1390         sunxi_nfc_hw_ecc_disable(mtd);
1391
1392         return 0;
1393 }
1394
1395 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1396                                            struct nand_chip *chip,
1397                                            const u8 *buf,
1398                                            int oob_required,
1399                                            int page)
1400 {
1401         struct nand_chip *nand = mtd_to_nand(mtd);
1402         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1403         struct nand_ecc_ctrl *ecc = &nand->ecc;
1404         struct scatterlist sg;
1405         int ret, i;
1406
1407         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1408         if (ret)
1409                 return ret;
1410
1411         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1412                                        DMA_TO_DEVICE, &sg);
1413         if (ret)
1414                 goto pio_fallback;
1415
1416         for (i = 0; i < ecc->steps; i++) {
1417                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1418
1419                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1420         }
1421
1422         sunxi_nfc_hw_ecc_enable(mtd);
1423         sunxi_nfc_randomizer_config(mtd, page, false);
1424         sunxi_nfc_randomizer_enable(mtd);
1425
1426         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1427                nfc->regs + NFC_REG_RCMD_SET);
1428
1429         dma_async_issue_pending(nfc->dmac);
1430
1431         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1432                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1433                nfc->regs + NFC_REG_CMD);
1434
1435         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1436         if (ret)
1437                 dmaengine_terminate_all(nfc->dmac);
1438
1439         sunxi_nfc_randomizer_disable(mtd);
1440         sunxi_nfc_hw_ecc_disable(mtd);
1441
1442         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1443
1444         if (ret)
1445                 return ret;
1446
1447         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1448                 /* TODO: use DMA to transfer extra OOB bytes ? */
1449                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1450                                                  NULL, page);
1451
1452         return 0;
1453
1454 pio_fallback:
1455         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1456 }
1457
1458 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1459                                                struct nand_chip *chip,
1460                                                uint8_t *buf, int oob_required,
1461                                                int page)
1462 {
1463         struct nand_ecc_ctrl *ecc = &chip->ecc;
1464         unsigned int max_bitflips = 0;
1465         int ret, i, cur_off = 0;
1466         bool raw_mode = false;
1467
1468         sunxi_nfc_hw_ecc_enable(mtd);
1469
1470         for (i = 0; i < ecc->steps; i++) {
1471                 int data_off = i * (ecc->size + ecc->bytes + 4);
1472                 int oob_off = data_off + ecc->size;
1473                 u8 *data = buf + (i * ecc->size);
1474                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1475
1476                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1477                                                   oob_off, &cur_off,
1478                                                   &max_bitflips, !i,
1479                                                   oob_required,
1480                                                   page);
1481                 if (ret < 0)
1482                         return ret;
1483                 else if (ret)
1484                         raw_mode = true;
1485         }
1486
1487         if (oob_required)
1488                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1489                                                 !raw_mode, page);
1490
1491         sunxi_nfc_hw_ecc_disable(mtd);
1492
1493         return max_bitflips;
1494 }
1495
1496 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1497                                                 struct nand_chip *chip,
1498                                                 const uint8_t *buf,
1499                                                 int oob_required, int page)
1500 {
1501         struct nand_ecc_ctrl *ecc = &chip->ecc;
1502         int ret, i, cur_off = 0;
1503
1504         sunxi_nfc_hw_ecc_enable(mtd);
1505
1506         for (i = 0; i < ecc->steps; i++) {
1507                 int data_off = i * (ecc->size + ecc->bytes + 4);
1508                 int oob_off = data_off + ecc->size;
1509                 const u8 *data = buf + (i * ecc->size);
1510                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1511
1512                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1513                                                    oob, oob_off, &cur_off,
1514                                                    false, page);
1515                 if (ret)
1516                         return ret;
1517         }
1518
1519         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1520                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1521                                                  &cur_off, page);
1522
1523         sunxi_nfc_hw_ecc_disable(mtd);
1524
1525         return 0;
1526 }
1527
1528 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1529                                             struct nand_chip *chip,
1530                                             int page)
1531 {
1532         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1533
1534         chip->pagebuf = -1;
1535
1536         return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1537 }
1538
1539 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1540                                              struct nand_chip *chip,
1541                                              int page)
1542 {
1543         int ret, status;
1544
1545         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1546
1547         chip->pagebuf = -1;
1548
1549         memset(chip->buffers->databuf, 0xff, mtd->writesize);
1550         ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1551         if (ret)
1552                 return ret;
1553
1554         /* Send command to program the OOB data */
1555         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1556
1557         status = chip->waitfunc(mtd, chip);
1558
1559         return status & NAND_STATUS_FAIL ? -EIO : 0;
1560 }
1561
1562 static const s32 tWB_lut[] = {6, 12, 16, 20};
1563 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1564
1565 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1566                 u32 clk_period)
1567 {
1568         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1569         int i;
1570
1571         for (i = 0; i < lut_size; i++) {
1572                 if (clk_cycles <= lut[i])
1573                         return i;
1574         }
1575
1576         /* Doesn't fit */
1577         return -EINVAL;
1578 }
1579
1580 #define sunxi_nand_lookup_timing(l, p, c) \
1581                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1582
1583 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1584                                         const struct nand_data_interface *conf,
1585                                         bool check_only)
1586 {
1587         struct nand_chip *nand = mtd_to_nand(mtd);
1588         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1589         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1590         const struct nand_sdr_timings *timings;
1591         u32 min_clk_period = 0;
1592         s32 tWB, tADL, tWHR, tRHW, tCAD;
1593         long real_clk_rate;
1594
1595         timings = nand_get_sdr_timings(conf);
1596         if (IS_ERR(timings))
1597                 return -ENOTSUPP;
1598
1599         /* T1 <=> tCLS */
1600         if (timings->tCLS_min > min_clk_period)
1601                 min_clk_period = timings->tCLS_min;
1602
1603         /* T2 <=> tCLH */
1604         if (timings->tCLH_min > min_clk_period)
1605                 min_clk_period = timings->tCLH_min;
1606
1607         /* T3 <=> tCS */
1608         if (timings->tCS_min > min_clk_period)
1609                 min_clk_period = timings->tCS_min;
1610
1611         /* T4 <=> tCH */
1612         if (timings->tCH_min > min_clk_period)
1613                 min_clk_period = timings->tCH_min;
1614
1615         /* T5 <=> tWP */
1616         if (timings->tWP_min > min_clk_period)
1617                 min_clk_period = timings->tWP_min;
1618
1619         /* T6 <=> tWH */
1620         if (timings->tWH_min > min_clk_period)
1621                 min_clk_period = timings->tWH_min;
1622
1623         /* T7 <=> tALS */
1624         if (timings->tALS_min > min_clk_period)
1625                 min_clk_period = timings->tALS_min;
1626
1627         /* T8 <=> tDS */
1628         if (timings->tDS_min > min_clk_period)
1629                 min_clk_period = timings->tDS_min;
1630
1631         /* T9 <=> tDH */
1632         if (timings->tDH_min > min_clk_period)
1633                 min_clk_period = timings->tDH_min;
1634
1635         /* T10 <=> tRR */
1636         if (timings->tRR_min > (min_clk_period * 3))
1637                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1638
1639         /* T11 <=> tALH */
1640         if (timings->tALH_min > min_clk_period)
1641                 min_clk_period = timings->tALH_min;
1642
1643         /* T12 <=> tRP */
1644         if (timings->tRP_min > min_clk_period)
1645                 min_clk_period = timings->tRP_min;
1646
1647         /* T13 <=> tREH */
1648         if (timings->tREH_min > min_clk_period)
1649                 min_clk_period = timings->tREH_min;
1650
1651         /* T14 <=> tRC */
1652         if (timings->tRC_min > (min_clk_period * 2))
1653                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1654
1655         /* T15 <=> tWC */
1656         if (timings->tWC_min > (min_clk_period * 2))
1657                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1658
1659         /* T16 - T19 + tCAD */
1660         if (timings->tWB_max > (min_clk_period * 20))
1661                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1662
1663         if (timings->tADL_min > (min_clk_period * 32))
1664                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1665
1666         if (timings->tWHR_min > (min_clk_period * 32))
1667                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1668
1669         if (timings->tRHW_min > (min_clk_period * 20))
1670                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1671
1672         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1673                                         min_clk_period);
1674         if (tWB < 0) {
1675                 dev_err(nfc->dev, "unsupported tWB\n");
1676                 return tWB;
1677         }
1678
1679         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1680         if (tADL > 3) {
1681                 dev_err(nfc->dev, "unsupported tADL\n");
1682                 return -EINVAL;
1683         }
1684
1685         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1686         if (tWHR > 3) {
1687                 dev_err(nfc->dev, "unsupported tWHR\n");
1688                 return -EINVAL;
1689         }
1690
1691         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1692                                         min_clk_period);
1693         if (tRHW < 0) {
1694                 dev_err(nfc->dev, "unsupported tRHW\n");
1695                 return tRHW;
1696         }
1697
1698         if (check_only)
1699                 return 0;
1700
1701         /*
1702          * TODO: according to ONFI specs this value only applies for DDR NAND,
1703          * but Allwinner seems to set this to 0x7. Mimic them for now.
1704          */
1705         tCAD = 0x7;
1706
1707         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1708         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1709
1710         /* Convert min_clk_period from picoseconds to nanoseconds */
1711         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1712
1713         /*
1714          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1715          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1716          * This new formula was verified with a scope and validated by
1717          * Allwinner engineers.
1718          */
1719         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1720         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1721
1722         /*
1723          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1724          * output cycle timings shall be used if the host drives tRC less than
1725          * 30 ns.
1726          */
1727         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1728         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1729                            NFC_TIMING_CTL_EDO : 0;
1730
1731         return 0;
1732 }
1733
1734 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1735                                     struct mtd_oob_region *oobregion)
1736 {
1737         struct nand_chip *nand = mtd_to_nand(mtd);
1738         struct nand_ecc_ctrl *ecc = &nand->ecc;
1739
1740         if (section >= ecc->steps)
1741                 return -ERANGE;
1742
1743         oobregion->offset = section * (ecc->bytes + 4) + 4;
1744         oobregion->length = ecc->bytes;
1745
1746         return 0;
1747 }
1748
1749 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1750                                      struct mtd_oob_region *oobregion)
1751 {
1752         struct nand_chip *nand = mtd_to_nand(mtd);
1753         struct nand_ecc_ctrl *ecc = &nand->ecc;
1754
1755         if (section > ecc->steps)
1756                 return -ERANGE;
1757
1758         /*
1759          * The first 2 bytes are used for BB markers, hence we
1760          * only have 2 bytes available in the first user data
1761          * section.
1762          */
1763         if (!section && ecc->mode == NAND_ECC_HW) {
1764                 oobregion->offset = 2;
1765                 oobregion->length = 2;
1766
1767                 return 0;
1768         }
1769
1770         oobregion->offset = section * (ecc->bytes + 4);
1771
1772         if (section < ecc->steps)
1773                 oobregion->length = 4;
1774         else
1775                 oobregion->offset = mtd->oobsize - oobregion->offset;
1776
1777         return 0;
1778 }
1779
1780 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1781         .ecc = sunxi_nand_ooblayout_ecc,
1782         .free = sunxi_nand_ooblayout_free,
1783 };
1784
1785 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1786                                               struct nand_ecc_ctrl *ecc,
1787                                               struct device_node *np)
1788 {
1789         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1790         struct nand_chip *nand = mtd_to_nand(mtd);
1791         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1792         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1793         struct sunxi_nand_hw_ecc *data;
1794         int nsectors;
1795         int ret;
1796         int i;
1797
1798         if (ecc->options & NAND_ECC_MAXIMIZE) {
1799                 int bytes;
1800
1801                 ecc->size = 1024;
1802                 nsectors = mtd->writesize / ecc->size;
1803
1804                 /* Reserve 2 bytes for the BBM */
1805                 bytes = (mtd->oobsize - 2) / nsectors;
1806
1807                 /* 4 non-ECC bytes are added before each ECC bytes section */
1808                 bytes -= 4;
1809
1810                 /* and bytes has to be even. */
1811                 if (bytes % 2)
1812                         bytes--;
1813
1814                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1815
1816                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1817                         if (strengths[i] > ecc->strength)
1818                                 break;
1819                 }
1820
1821                 if (!i)
1822                         ecc->strength = 0;
1823                 else
1824                         ecc->strength = strengths[i - 1];
1825         }
1826
1827         if (ecc->size != 512 && ecc->size != 1024)
1828                 return -EINVAL;
1829
1830         data = kzalloc(sizeof(*data), GFP_KERNEL);
1831         if (!data)
1832                 return -ENOMEM;
1833
1834         /* Prefer 1k ECC chunk over 512 ones */
1835         if (ecc->size == 512 && mtd->writesize > 512) {
1836                 ecc->size = 1024;
1837                 ecc->strength *= 2;
1838         }
1839
1840         /* Add ECC info retrieval from DT */
1841         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1842                 if (ecc->strength <= strengths[i])
1843                         break;
1844         }
1845
1846         if (i >= ARRAY_SIZE(strengths)) {
1847                 dev_err(nfc->dev, "unsupported strength\n");
1848                 ret = -ENOTSUPP;
1849                 goto err;
1850         }
1851
1852         data->mode = i;
1853
1854         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1855         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1856
1857         /* HW ECC always work with even numbers of ECC bytes */
1858         ecc->bytes = ALIGN(ecc->bytes, 2);
1859
1860         nsectors = mtd->writesize / ecc->size;
1861
1862         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1863                 ret = -EINVAL;
1864                 goto err;
1865         }
1866
1867         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1868         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1869         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1870         ecc->priv = data;
1871
1872         return 0;
1873
1874 err:
1875         kfree(data);
1876
1877         return ret;
1878 }
1879
1880 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1881 {
1882         kfree(ecc->priv);
1883 }
1884
1885 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1886                                        struct nand_ecc_ctrl *ecc,
1887                                        struct device_node *np)
1888 {
1889         struct nand_chip *nand = mtd_to_nand(mtd);
1890         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1891         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1892         int ret;
1893
1894         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1895         if (ret)
1896                 return ret;
1897
1898         if (nfc->dmac) {
1899                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1900                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1901                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1902                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1903         } else {
1904                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1905                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1906                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1907         }
1908
1909         /* TODO: support DMA for raw accesses and subpage write */
1910         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1911         ecc->read_oob_raw = nand_read_oob_std;
1912         ecc->write_oob_raw = nand_write_oob_std;
1913         ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1914
1915         return 0;
1916 }
1917
1918 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1919                                                 struct nand_ecc_ctrl *ecc,
1920                                                 struct device_node *np)
1921 {
1922         int ret;
1923
1924         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1925         if (ret)
1926                 return ret;
1927
1928         ecc->prepad = 4;
1929         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1930         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1931         ecc->read_oob_raw = nand_read_oob_syndrome;
1932         ecc->write_oob_raw = nand_write_oob_syndrome;
1933
1934         return 0;
1935 }
1936
1937 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1938 {
1939         switch (ecc->mode) {
1940         case NAND_ECC_HW:
1941         case NAND_ECC_HW_SYNDROME:
1942                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1943                 break;
1944         case NAND_ECC_NONE:
1945         default:
1946                 break;
1947         }
1948 }
1949
1950 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1951                                struct device_node *np)
1952 {
1953         struct nand_chip *nand = mtd_to_nand(mtd);
1954         int ret;
1955
1956         if (!ecc->size) {
1957                 ecc->size = nand->ecc_step_ds;
1958                 ecc->strength = nand->ecc_strength_ds;
1959         }
1960
1961         if (!ecc->size || !ecc->strength)
1962                 return -EINVAL;
1963
1964         switch (ecc->mode) {
1965         case NAND_ECC_HW:
1966                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1967                 if (ret)
1968                         return ret;
1969                 break;
1970         case NAND_ECC_HW_SYNDROME:
1971                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1972                 if (ret)
1973                         return ret;
1974                 break;
1975         case NAND_ECC_NONE:
1976         case NAND_ECC_SOFT:
1977                 break;
1978         default:
1979                 return -EINVAL;
1980         }
1981
1982         return 0;
1983 }
1984
1985 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1986                                 struct device_node *np)
1987 {
1988         struct sunxi_nand_chip *chip;
1989         struct mtd_info *mtd;
1990         struct nand_chip *nand;
1991         int nsels;
1992         int ret;
1993         int i;
1994         u32 tmp;
1995
1996         if (!of_get_property(np, "reg", &nsels))
1997                 return -EINVAL;
1998
1999         nsels /= sizeof(u32);
2000         if (!nsels) {
2001                 dev_err(dev, "invalid reg property size\n");
2002                 return -EINVAL;
2003         }
2004
2005         chip = devm_kzalloc(dev,
2006                             sizeof(*chip) +
2007                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2008                             GFP_KERNEL);
2009         if (!chip) {
2010                 dev_err(dev, "could not allocate chip\n");
2011                 return -ENOMEM;
2012         }
2013
2014         chip->nsels = nsels;
2015         chip->selected = -1;
2016
2017         for (i = 0; i < nsels; i++) {
2018                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2019                 if (ret) {
2020                         dev_err(dev, "could not retrieve reg property: %d\n",
2021                                 ret);
2022                         return ret;
2023                 }
2024
2025                 if (tmp > NFC_MAX_CS) {
2026                         dev_err(dev,
2027                                 "invalid reg value: %u (max CS = 7)\n",
2028                                 tmp);
2029                         return -EINVAL;
2030                 }
2031
2032                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2033                         dev_err(dev, "CS %d already assigned\n", tmp);
2034                         return -EINVAL;
2035                 }
2036
2037                 chip->sels[i].cs = tmp;
2038
2039                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2040                     tmp < 2) {
2041                         chip->sels[i].rb.type = RB_NATIVE;
2042                         chip->sels[i].rb.info.nativeid = tmp;
2043                 } else {
2044                         ret = of_get_named_gpio(np, "rb-gpios", i);
2045                         if (ret >= 0) {
2046                                 tmp = ret;
2047                                 chip->sels[i].rb.type = RB_GPIO;
2048                                 chip->sels[i].rb.info.gpio = tmp;
2049                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2050                                 if (ret)
2051                                         return ret;
2052
2053                                 ret = gpio_direction_input(tmp);
2054                                 if (ret)
2055                                         return ret;
2056                         } else {
2057                                 chip->sels[i].rb.type = RB_NONE;
2058                         }
2059                 }
2060         }
2061
2062         nand = &chip->nand;
2063         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2064         nand->chip_delay = 200;
2065         nand->controller = &nfc->controller;
2066         /*
2067          * Set the ECC mode to the default value in case nothing is specified
2068          * in the DT.
2069          */
2070         nand->ecc.mode = NAND_ECC_HW;
2071         nand_set_flash_node(nand, np);
2072         nand->select_chip = sunxi_nfc_select_chip;
2073         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2074         nand->read_buf = sunxi_nfc_read_buf;
2075         nand->write_buf = sunxi_nfc_write_buf;
2076         nand->read_byte = sunxi_nfc_read_byte;
2077         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2078
2079         mtd = nand_to_mtd(nand);
2080         mtd->dev.parent = dev;
2081
2082         ret = nand_scan_ident(mtd, nsels, NULL);
2083         if (ret)
2084                 return ret;
2085
2086         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2087                 nand->bbt_options |= NAND_BBT_NO_OOB;
2088
2089         if (nand->options & NAND_NEED_SCRAMBLING)
2090                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2091
2092         nand->options |= NAND_SUBPAGE_READ;
2093
2094         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2095         if (ret) {
2096                 dev_err(dev, "ECC init failed: %d\n", ret);
2097                 return ret;
2098         }
2099
2100         ret = nand_scan_tail(mtd);
2101         if (ret) {
2102                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2103                 return ret;
2104         }
2105
2106         ret = mtd_device_register(mtd, NULL, 0);
2107         if (ret) {
2108                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2109                 nand_release(mtd);
2110                 return ret;
2111         }
2112
2113         list_add_tail(&chip->node, &nfc->chips);
2114
2115         return 0;
2116 }
2117
2118 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2119 {
2120         struct device_node *np = dev->of_node;
2121         struct device_node *nand_np;
2122         int nchips = of_get_child_count(np);
2123         int ret;
2124
2125         if (nchips > 8) {
2126                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2127                 return -EINVAL;
2128         }
2129
2130         for_each_child_of_node(np, nand_np) {
2131                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2132                 if (ret) {
2133                         of_node_put(nand_np);
2134                         return ret;
2135                 }
2136         }
2137
2138         return 0;
2139 }
2140
2141 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2142 {
2143         struct sunxi_nand_chip *chip;
2144
2145         while (!list_empty(&nfc->chips)) {
2146                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2147                                         node);
2148                 nand_release(nand_to_mtd(&chip->nand));
2149                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2150                 list_del(&chip->node);
2151         }
2152 }
2153
2154 static int sunxi_nfc_probe(struct platform_device *pdev)
2155 {
2156         struct device *dev = &pdev->dev;
2157         struct resource *r;
2158         struct sunxi_nfc *nfc;
2159         int irq;
2160         int ret;
2161
2162         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2163         if (!nfc)
2164                 return -ENOMEM;
2165
2166         nfc->dev = dev;
2167         nand_hw_control_init(&nfc->controller);
2168         INIT_LIST_HEAD(&nfc->chips);
2169
2170         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2171         nfc->regs = devm_ioremap_resource(dev, r);
2172         if (IS_ERR(nfc->regs))
2173                 return PTR_ERR(nfc->regs);
2174
2175         irq = platform_get_irq(pdev, 0);
2176         if (irq < 0) {
2177                 dev_err(dev, "failed to retrieve irq\n");
2178                 return irq;
2179         }
2180
2181         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2182         if (IS_ERR(nfc->ahb_clk)) {
2183                 dev_err(dev, "failed to retrieve ahb clk\n");
2184                 return PTR_ERR(nfc->ahb_clk);
2185         }
2186
2187         ret = clk_prepare_enable(nfc->ahb_clk);
2188         if (ret)
2189                 return ret;
2190
2191         nfc->mod_clk = devm_clk_get(dev, "mod");
2192         if (IS_ERR(nfc->mod_clk)) {
2193                 dev_err(dev, "failed to retrieve mod clk\n");
2194                 ret = PTR_ERR(nfc->mod_clk);
2195                 goto out_ahb_clk_unprepare;
2196         }
2197
2198         ret = clk_prepare_enable(nfc->mod_clk);
2199         if (ret)
2200                 goto out_ahb_clk_unprepare;
2201
2202         nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2203         if (!IS_ERR(nfc->reset)) {
2204                 ret = reset_control_deassert(nfc->reset);
2205                 if (ret) {
2206                         dev_err(dev, "reset err %d\n", ret);
2207                         goto out_mod_clk_unprepare;
2208                 }
2209         } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2210                 ret = PTR_ERR(nfc->reset);
2211                 goto out_mod_clk_unprepare;
2212         }
2213
2214         ret = sunxi_nfc_rst(nfc);
2215         if (ret)
2216                 goto out_ahb_reset_reassert;
2217
2218         writel(0, nfc->regs + NFC_REG_INT);
2219         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2220                                0, "sunxi-nand", nfc);
2221         if (ret)
2222                 goto out_ahb_reset_reassert;
2223
2224         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2225         if (nfc->dmac) {
2226                 struct dma_slave_config dmac_cfg = { };
2227
2228                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2229                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2230                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2231                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2232                 dmac_cfg.src_maxburst = 4;
2233                 dmac_cfg.dst_maxburst = 4;
2234                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2235         } else {
2236                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2237         }
2238
2239         platform_set_drvdata(pdev, nfc);
2240
2241         ret = sunxi_nand_chips_init(dev, nfc);
2242         if (ret) {
2243                 dev_err(dev, "failed to init nand chips\n");
2244                 goto out_release_dmac;
2245         }
2246
2247         return 0;
2248
2249 out_release_dmac:
2250         if (nfc->dmac)
2251                 dma_release_channel(nfc->dmac);
2252 out_ahb_reset_reassert:
2253         if (!IS_ERR(nfc->reset))
2254                 reset_control_assert(nfc->reset);
2255 out_mod_clk_unprepare:
2256         clk_disable_unprepare(nfc->mod_clk);
2257 out_ahb_clk_unprepare:
2258         clk_disable_unprepare(nfc->ahb_clk);
2259
2260         return ret;
2261 }
2262
2263 static int sunxi_nfc_remove(struct platform_device *pdev)
2264 {
2265         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2266
2267         sunxi_nand_chips_cleanup(nfc);
2268
2269         if (!IS_ERR(nfc->reset))
2270                 reset_control_assert(nfc->reset);
2271
2272         if (nfc->dmac)
2273                 dma_release_channel(nfc->dmac);
2274         clk_disable_unprepare(nfc->mod_clk);
2275         clk_disable_unprepare(nfc->ahb_clk);
2276
2277         return 0;
2278 }
2279
2280 static const struct of_device_id sunxi_nfc_ids[] = {
2281         { .compatible = "allwinner,sun4i-a10-nand" },
2282         { /* sentinel */ }
2283 };
2284 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2285
2286 static struct platform_driver sunxi_nfc_driver = {
2287         .driver = {
2288                 .name = "sunxi_nand",
2289                 .of_match_table = sunxi_nfc_ids,
2290         },
2291         .probe = sunxi_nfc_probe,
2292         .remove = sunxi_nfc_remove,
2293 };
2294 module_platform_driver(sunxi_nfc_driver);
2295
2296 MODULE_LICENSE("GPL v2");
2297 MODULE_AUTHOR("Boris BREZILLON");
2298 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2299 MODULE_ALIAS("platform:sunxi_nand");