drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/nand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_ECC_BLOCK_512       BIT(5)
 149 #define NFC_RANDOM_EN           BIT(9)
 150 #define NFC_RANDOM_DIRECTION    BIT(10)
 151 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 152 #define NFC_ECC_MODE(x)         ((x) << 12)
 153 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 154 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 155
 156 /* define bit use in NFC_ECC_ST */
 157 #define NFC_ECC_ERR(x)          BIT(x)
 158 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 159 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 160 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 161
 162 #define NFC_DEFAULT_TIMEOUT_MS  1000
 163
 164 #define NFC_SRAM_SIZE           1024
 165
 166 #define NFC_MAX_CS              7
 167
 168 /*
 169  * Ready/Busy detection type: describes the Ready/Busy detection modes
 170  *
 171  * @RB_NONE:    no external detection available, rely on STATUS command
 172  *              and software timeouts
 173  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 174  *              pin of the NAND flash chip must be connected to one of the
 175  *              native NAND R/B pins (those which can be muxed to the NAND
 176  *              Controller)
 177  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 178  *              pin of the NAND flash chip must be connected to a GPIO capable
 179  *              pin.
 180  */
 181 enum sunxi_nand_rb_type {
 182         RB_NONE,
 183         RB_NATIVE,
 184         RB_GPIO,
 185 };
 186
 187 /*
 188  * Ready/Busy structure: stores information related to Ready/Busy detection
 189  *
 190  * @type:       the Ready/Busy detection mode
 191  * @info:       information related to the R/B detection mode. Either a gpio
 192  *              id or a native R/B id (those supported by the NAND controller).
 193  */
 194 struct sunxi_nand_rb {
 195         enum sunxi_nand_rb_type type;
 196         union {
 197                 int gpio;
 198                 int nativeid;
 199         } info;
 200 };
 201
 202 /*
 203  * Chip Select structure: stores information related to NAND Chip Select
 204  *
 205  * @cs:         the NAND CS id used to communicate with a NAND Chip
 206  * @rb:         the Ready/Busy description
 207  */
 208 struct sunxi_nand_chip_sel {
 209         u8 cs;
 210         struct sunxi_nand_rb rb;
 211 };
 212
 213 /*
 214  * sunxi HW ECC infos: stores information related to HW ECC support
 215  *
 216  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 217  */
 218 struct sunxi_nand_hw_ecc {
 219         int mode;
 220 };
 221
 222 /*
 223  * NAND chip structure: stores NAND chip device related information
 224  *
 225  * @node:               used to store NAND chips into a list
 226  * @nand:               base NAND chip structure
 227  * @mtd:                base MTD structure
 228  * @clk_rate:           clk_rate required for this NAND chip
 229  * @timing_cfg          TIMING_CFG register value for this NAND chip
 230  * @selected:           current active CS
 231  * @nsels:              number of CS lines required by the NAND chip
 232  * @sels:               array of CS lines descriptions
 233  */
 234 struct sunxi_nand_chip {
 235         struct list_head node;
 236         struct nand_chip nand;
 237         unsigned long clk_rate;
 238         u32 timing_cfg;
 239         u32 timing_ctl;
 240         int selected;
 241         int addr_cycles;
 242         u32 addr[2];
 243         int cmd_cycles;
 244         u8 cmd[2];
 245         int nsels;
 246         struct sunxi_nand_chip_sel sels[0];
 247 };
 248
 249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 250 {
 251         return container_of(nand, struct sunxi_nand_chip, nand);
 252 }
 253
 254 /*
 255  * NAND Controller structure: stores sunxi NAND controller information
 256  *
 257  * @controller:         base controller structure
 258  * @dev:                parent device (used to print error messages)
 259  * @regs:               NAND controller registers
 260  * @ahb_clk:            NAND Controller AHB clock
 261  * @mod_clk:            NAND Controller mod clock
 262  * @assigned_cs:        bitmask describing already assigned CS lines
 263  * @clk_rate:           NAND controller current clock rate
 264  * @chips:              a list containing all the NAND chips attached to
 265  *                      this NAND controller
 266  * @complete:           a completion object used to wait for NAND
 267  *                      controller events
 268  */
 269 struct sunxi_nfc {
 270         struct nand_hw_control controller;
 271         struct device *dev;
 272         void __iomem *regs;
 273         struct clk *ahb_clk;
 274         struct clk *mod_clk;
 275         struct reset_control *reset;
 276         unsigned long assigned_cs;
 277         unsigned long clk_rate;
 278         struct list_head chips;
 279         struct completion complete;
 280         struct dma_chan *dmac;
 281 };
 282
 283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 284 {
 285         return container_of(ctrl, struct sunxi_nfc, controller);
 286 }
 287
 288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 289 {
 290         struct sunxi_nfc *nfc = dev_id;
 291         u32 st = readl(nfc->regs + NFC_REG_ST);
 292         u32 ien = readl(nfc->regs + NFC_REG_INT);
 293
 294         if (!(ien & st))
 295                 return IRQ_NONE;
 296
 297         if ((ien & st) == ien)
 298                 complete(&nfc->complete);
 299
 300         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 301         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 307                                  bool use_polling, unsigned int timeout_ms)
 308 {
 309         int ret;
 310
 311         if (events & ~NFC_INT_MASK)
 312                 return -EINVAL;
 313
 314         if (!timeout_ms)
 315                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 316
 317         if (!use_polling) {
 318                 init_completion(&nfc->complete);
 319
 320                 writel(events, nfc->regs + NFC_REG_INT);
 321
 322                 ret = wait_for_completion_timeout(&nfc->complete,
 323                                                 msecs_to_jiffies(timeout_ms));
 324
 325                 writel(0, nfc->regs + NFC_REG_INT);
 326         } else {
 327                 u32 status;
 328
 329                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 330                                          (status & events) == events, 1,
 331                                          timeout_ms * 1000);
 332         }
 333
 334         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 335
 336         if (ret)
 337                 dev_err(nfc->dev, "wait interrupt timedout\n");
 338
 339         return ret;
 340 }
 341
 342 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 343 {
 344         u32 status;
 345         int ret;
 346
 347         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 348                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 349                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 350         if (ret)
 351                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 352
 353         return ret;
 354 }
 355
 356 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 357 {
 358         u32 ctl;
 359         int ret;
 360
 361         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 362         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 363
 364         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 365                                  !(ctl & NFC_RESET), 1,
 366                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 367         if (ret)
 368                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 369
 370         return ret;
 371 }
 372
 373 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 374                                     int chunksize, int nchunks,
 375                                     enum dma_data_direction ddir,
 376                                     struct scatterlist *sg)
 377 {
 378         struct nand_chip *nand = mtd_to_nand(mtd);
 379         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 380         struct dma_async_tx_descriptor *dmad;
 381         enum dma_transfer_direction tdir;
 382         dma_cookie_t dmat;
 383         int ret;
 384
 385         if (ddir == DMA_FROM_DEVICE)
 386                 tdir = DMA_DEV_TO_MEM;
 387         else
 388                 tdir = DMA_MEM_TO_DEV;
 389
 390         sg_init_one(sg, buf, nchunks * chunksize);
 391         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 392         if (!ret)
 393                 return -ENOMEM;
 394
 395         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 396         if (!dmad) {
 397                 ret = -EINVAL;
 398                 goto err_unmap_buf;
 399         }
 400
 401         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 402                nfc->regs + NFC_REG_CTL);
 403         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 404         writel(chunksize, nfc->regs + NFC_REG_CNT);
 405         dmat = dmaengine_submit(dmad);
 406
 407         ret = dma_submit_error(dmat);
 408         if (ret)
 409                 goto err_clr_dma_flag;
 410
 411         return 0;
 412
 413 err_clr_dma_flag:
 414         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 415                nfc->regs + NFC_REG_CTL);
 416
 417 err_unmap_buf:
 418         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 419         return ret;
 420 }
 421
 422 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 423                                      enum dma_data_direction ddir,
 424                                      struct scatterlist *sg)
 425 {
 426         struct nand_chip *nand = mtd_to_nand(mtd);
 427         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 428
 429         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 430         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 431                nfc->regs + NFC_REG_CTL);
 432 }
 433
 434 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 435 {
 436         struct nand_chip *nand = mtd_to_nand(mtd);
 437         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 438         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 439         struct sunxi_nand_rb *rb;
 440         int ret;
 441
 442         if (sunxi_nand->selected < 0)
 443                 return 0;
 444
 445         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 446
 447         switch (rb->type) {
 448         case RB_NATIVE:
 449                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 450                          NFC_RB_STATE(rb->info.nativeid));
 451                 break;
 452         case RB_GPIO:
 453                 ret = gpio_get_value(rb->info.gpio);
 454                 break;
 455         case RB_NONE:
 456         default:
 457                 ret = 0;
 458                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 459                 break;
 460         }
 461
 462         return ret;
 463 }
 464
 465 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 466 {
 467         struct nand_chip *nand = mtd_to_nand(mtd);
 468         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 469         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 470         struct sunxi_nand_chip_sel *sel;
 471         u32 ctl;
 472
 473         if (chip > 0 && chip >= sunxi_nand->nsels)
 474                 return;
 475
 476         if (chip == sunxi_nand->selected)
 477                 return;
 478
 479         ctl = readl(nfc->regs + NFC_REG_CTL) &
 480               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 481
 482         if (chip >= 0) {
 483                 sel = &sunxi_nand->sels[chip];
 484
 485                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 486                        NFC_PAGE_SHIFT(nand->page_shift);
 487                 if (sel->rb.type == RB_NONE) {
 488                         nand->dev_ready = NULL;
 489                 } else {
 490                         nand->dev_ready = sunxi_nfc_dev_ready;
 491                         if (sel->rb.type == RB_NATIVE)
 492                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 493                 }
 494
 495                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 496
 497                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 498                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 499                         nfc->clk_rate = sunxi_nand->clk_rate;
 500                 }
 501         }
 502
 503         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 504         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 505         writel(ctl, nfc->regs + NFC_REG_CTL);
 506
 507         sunxi_nand->selected = chip;
 508 }
 509
 510 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 511 {
 512         struct nand_chip *nand = mtd_to_nand(mtd);
 513         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 514         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 515         int ret;
 516         int cnt;
 517         int offs = 0;
 518         u32 tmp;
 519
 520         while (len > offs) {
 521                 cnt = min(len - offs, NFC_SRAM_SIZE);
 522
 523                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 524                 if (ret)
 525                         break;
 526
 527                 writel(cnt, nfc->regs + NFC_REG_CNT);
 528                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 529                 writel(tmp, nfc->regs + NFC_REG_CMD);
 530
 531                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 532                 if (ret)
 533                         break;
 534
 535                 if (buf)
 536                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 537                                       cnt);
 538                 offs += cnt;
 539         }
 540 }
 541
 542 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 543                                 int len)
 544 {
 545         struct nand_chip *nand = mtd_to_nand(mtd);
 546         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 547         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 548         int ret;
 549         int cnt;
 550         int offs = 0;
 551         u32 tmp;
 552
 553         while (len > offs) {
 554                 cnt = min(len - offs, NFC_SRAM_SIZE);
 555
 556                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 557                 if (ret)
 558                         break;
 559
 560                 writel(cnt, nfc->regs + NFC_REG_CNT);
 561                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 562                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 563                       NFC_ACCESS_DIR;
 564                 writel(tmp, nfc->regs + NFC_REG_CMD);
 565
 566                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 567                 if (ret)
 568                         break;
 569
 570                 offs += cnt;
 571         }
 572 }
 573
 574 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 575 {
 576         uint8_t ret;
 577
 578         sunxi_nfc_read_buf(mtd, &ret, 1);
 579
 580         return ret;
 581 }
 582
 583 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 584                                unsigned int ctrl)
 585 {
 586         struct nand_chip *nand = mtd_to_nand(mtd);
 587         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 588         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 589         int ret;
 590
 591         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 592         if (ret)
 593                 return;
 594
 595         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 596             !(ctrl & (NAND_CLE | NAND_ALE))) {
 597                 u32 cmd = 0;
 598
 599                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 600                         return;
 601
 602                 if (sunxi_nand->cmd_cycles--)
 603                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 604
 605                 if (sunxi_nand->cmd_cycles--) {
 606                         cmd |= NFC_SEND_CMD2;
 607                         writel(sunxi_nand->cmd[1],
 608                                nfc->regs + NFC_REG_RCMD_SET);
 609                 }
 610
 611                 sunxi_nand->cmd_cycles = 0;
 612
 613                 if (sunxi_nand->addr_cycles) {
 614                         cmd |= NFC_SEND_ADR |
 615                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 616                         writel(sunxi_nand->addr[0],
 617                                nfc->regs + NFC_REG_ADDR_LOW);
 618                 }
 619
 620                 if (sunxi_nand->addr_cycles > 4)
 621                         writel(sunxi_nand->addr[1],
 622                                nfc->regs + NFC_REG_ADDR_HIGH);
 623
 624                 writel(cmd, nfc->regs + NFC_REG_CMD);
 625                 sunxi_nand->addr[0] = 0;
 626                 sunxi_nand->addr[1] = 0;
 627                 sunxi_nand->addr_cycles = 0;
 628                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 629         }
 630
 631         if (ctrl & NAND_CLE) {
 632                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 633         } else if (ctrl & NAND_ALE) {
 634                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 635                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 636                 sunxi_nand->addr_cycles++;
 637         }
 638 }
 639
 640 /* These seed values have been extracted from Allwinner's BSP */
 641 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 642         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 643         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 644         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 645         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 646         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 647         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 648         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 649         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 650         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 651         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 652         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 653         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 654         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 655         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 656         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 657         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 658 };
 659
 660 /*
 661  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 662  * have been generated using
 663  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 664  * the randomizer engine does internally before de/scrambling OOB data.
 665  *
 666  * Those tables are statically defined to avoid calculating randomizer state
 667  * at runtime.
 668  */
 669 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 670         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 671         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 672         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 673         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 674         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 675         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 676         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 677         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 678         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 679         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 680         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 681         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 682         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 683         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 684         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 685         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 686 };
 687
 688 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 689         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 690         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 691         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 692         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 693         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 694         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 695         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 696         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 697         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 698         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 699         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 700         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 701         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 702         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 703         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 704         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 705 };
 706
 707 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 708 {
 709         state &= 0x7fff;
 710
 711         /*
 712          * This loop is just a simple implementation of a Fibonacci LFSR using
 713          * the x16 + x15 + 1 polynomial.
 714          */
 715         while (count--)
 716                 state = ((state >> 1) |
 717                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 718
 719         return state;
 720 }
 721
 722 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 723 {
 724         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 725         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 726
 727         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 728                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 729
 730         if (ecc) {
 731                 if (mtd->ecc_step_size == 512)
 732                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 733                 else
 734                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 735         }
 736
 737         return seeds[page % mod];
 738 }
 739
 740 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 741                                         int page, bool ecc)
 742 {
 743         struct nand_chip *nand = mtd_to_nand(mtd);
 744         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 745         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 746         u16 state;
 747
 748         if (!(nand->options & NAND_NEED_SCRAMBLING))
 749                 return;
 750
 751         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 752         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 753         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 754         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 755 }
 756
 757 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 758 {
 759         struct nand_chip *nand = mtd_to_nand(mtd);
 760         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 761
 762         if (!(nand->options & NAND_NEED_SCRAMBLING))
 763                 return;
 764
 765         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 766                nfc->regs + NFC_REG_ECC_CTL);
 767 }
 768
 769 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 770 {
 771         struct nand_chip *nand = mtd_to_nand(mtd);
 772         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 773
 774         if (!(nand->options & NAND_NEED_SCRAMBLING))
 775                 return;
 776
 777         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 778                nfc->regs + NFC_REG_ECC_CTL);
 779 }
 780
 781 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 782 {
 783         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 784
 785         bbm[0] ^= state;
 786         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 787 }
 788
 789 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 790                                            const uint8_t *buf, int len,
 791                                            bool ecc, int page)
 792 {
 793         sunxi_nfc_randomizer_config(mtd, page, ecc);
 794         sunxi_nfc_randomizer_enable(mtd);
 795         sunxi_nfc_write_buf(mtd, buf, len);
 796         sunxi_nfc_randomizer_disable(mtd);
 797 }
 798
 799 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 800                                           int len, bool ecc, int page)
 801 {
 802         sunxi_nfc_randomizer_config(mtd, page, ecc);
 803         sunxi_nfc_randomizer_enable(mtd);
 804         sunxi_nfc_read_buf(mtd, buf, len);
 805         sunxi_nfc_randomizer_disable(mtd);
 806 }
 807
 808 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 809 {
 810         struct nand_chip *nand = mtd_to_nand(mtd);
 811         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 812         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 813         u32 ecc_ctl;
 814
 815         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 816         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 817                      NFC_ECC_BLOCK_SIZE_MSK);
 818         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 819                    NFC_ECC_PIPELINE;
 820
 821         if (nand->ecc.size == 512)
 822                 ecc_ctl |= NFC_ECC_BLOCK_512;
 823
 824         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 825 }
 826
 827 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 828 {
 829         struct nand_chip *nand = mtd_to_nand(mtd);
 830         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 831
 832         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 833                nfc->regs + NFC_REG_ECC_CTL);
 834 }
 835
 836 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 837 {
 838         buf[0] = user_data;
 839         buf[1] = user_data >> 8;
 840         buf[2] = user_data >> 16;
 841         buf[3] = user_data >> 24;
 842 }
 843
 844 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 845 {
 846         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 847 }
 848
 849 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 850                                                 int step, bool bbm, int page)
 851 {
 852         struct nand_chip *nand = mtd_to_nand(mtd);
 853         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 854
 855         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 856                                    oob);
 857
 858         /* De-randomize the Bad Block Marker. */
 859         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 860                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 861 }
 862
 863 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 864                                                 const u8 *oob, int step,
 865                                                 bool bbm, int page)
 866 {
 867         struct nand_chip *nand = mtd_to_nand(mtd);
 868         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 869         u8 user_data[4];
 870
 871         /* Randomize the Bad Block Marker. */
 872         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 873                 memcpy(user_data, oob, sizeof(user_data));
 874                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 875                 oob = user_data;
 876         }
 877
 878         writel(sunxi_nfc_buf_to_user_data(oob),
 879                nfc->regs + NFC_REG_USER_DATA(step));
 880 }
 881
 882 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 883                                           unsigned int *max_bitflips, int ret)
 884 {
 885         if (ret < 0) {
 886                 mtd->ecc_stats.failed++;
 887         } else {
 888                 mtd->ecc_stats.corrected += ret;
 889                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 890         }
 891 }
 892
 893 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 894                                     int step, u32 status, bool *erased)
 895 {
 896         struct nand_chip *nand = mtd_to_nand(mtd);
 897         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 898         struct nand_ecc_ctrl *ecc = &nand->ecc;
 899         u32 tmp;
 900
 901         *erased = false;
 902
 903         if (status & NFC_ECC_ERR(step))
 904                 return -EBADMSG;
 905
 906         if (status & NFC_ECC_PAT_FOUND(step)) {
 907                 u8 pattern;
 908
 909                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 910                         pattern = 0x0;
 911                 } else {
 912                         pattern = 0xff;
 913                         *erased = true;
 914                 }
 915
 916                 if (data)
 917                         memset(data, pattern, ecc->size);
 918
 919                 if (oob)
 920                         memset(oob, pattern, ecc->bytes + 4);
 921
 922                 return 0;
 923         }
 924
 925         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 926
 927         return NFC_ECC_ERR_CNT(step, tmp);
 928 }
 929
 930 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 931                                        u8 *data, int data_off,
 932                                        u8 *oob, int oob_off,
 933                                        int *cur_off,
 934                                        unsigned int *max_bitflips,
 935                                        bool bbm, bool oob_required, int page)
 936 {
 937         struct nand_chip *nand = mtd_to_nand(mtd);
 938         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 939         struct nand_ecc_ctrl *ecc = &nand->ecc;
 940         int raw_mode = 0;
 941         bool erased;
 942         int ret;
 943
 944         if (*cur_off != data_off)
 945                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 946
 947         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 948
 949         if (data_off + ecc->size != oob_off)
 950                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 951
 952         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 953         if (ret)
 954                 return ret;
 955
 956         sunxi_nfc_randomizer_enable(mtd);
 957         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 958                nfc->regs + NFC_REG_CMD);
 959
 960         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 961         sunxi_nfc_randomizer_disable(mtd);
 962         if (ret)
 963                 return ret;
 964
 965         *cur_off = oob_off + ecc->bytes + 4;
 966
 967         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 968                                        readl(nfc->regs + NFC_REG_ECC_ST),
 969                                        &erased);
 970         if (erased)
 971                 return 1;
 972
 973         if (ret < 0) {
 974                 /*
 975                  * Re-read the data with the randomizer disabled to identify
 976                  * bitflips in erased pages.
 977                  */
 978                 if (nand->options & NAND_NEED_SCRAMBLING) {
 979                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 980                         nand->read_buf(mtd, data, ecc->size);
 981                 } else {
 982                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 983                                       ecc->size);
 984                 }
 985
 986                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 987                 nand->read_buf(mtd, oob, ecc->bytes + 4);
 988
 989                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 990                                                   oob, ecc->bytes + 4,
 991                                                   NULL, 0, ecc->strength);
 992                 if (ret >= 0)
 993                         raw_mode = 1;
 994         } else {
 995                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 996
 997                 if (oob_required) {
 998                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 999                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1000                                                       true, page);
1001
1002                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1003                                                             bbm, page);
1004                 }
1005         }
1006
1007         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1008
1009         return raw_mode;
1010 }
1011
1012 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1013                                             u8 *oob, int *cur_off,
1014                                             bool randomize, int page)
1015 {
1016         struct nand_chip *nand = mtd_to_nand(mtd);
1017         struct nand_ecc_ctrl *ecc = &nand->ecc;
1018         int offset = ((ecc->bytes + 4) * ecc->steps);
1019         int len = mtd->oobsize - offset;
1020
1021         if (len <= 0)
1022                 return;
1023
1024         if (!cur_off || *cur_off != offset)
1025                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1026                               offset + mtd->writesize, -1);
1027
1028         if (!randomize)
1029                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1030         else
1031                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1032                                               false, page);
1033
1034         if (cur_off)
1035                 *cur_off = mtd->oobsize + mtd->writesize;
1036 }
1037
1038 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1039                                             int oob_required, int page,
1040                                             int nchunks)
1041 {
1042         struct nand_chip *nand = mtd_to_nand(mtd);
1043         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1044         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1045         struct nand_ecc_ctrl *ecc = &nand->ecc;
1046         unsigned int max_bitflips = 0;
1047         int ret, i, raw_mode = 0;
1048         struct scatterlist sg;
1049         u32 status;
1050
1051         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1052         if (ret)
1053                 return ret;
1054
1055         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1056                                        DMA_FROM_DEVICE, &sg);
1057         if (ret)
1058                 return ret;
1059
1060         sunxi_nfc_hw_ecc_enable(mtd);
1061         sunxi_nfc_randomizer_config(mtd, page, false);
1062         sunxi_nfc_randomizer_enable(mtd);
1063
1064         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1065                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1066
1067         dma_async_issue_pending(nfc->dmac);
1068
1069         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1070                nfc->regs + NFC_REG_CMD);
1071
1072         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1073         if (ret)
1074                 dmaengine_terminate_all(nfc->dmac);
1075
1076         sunxi_nfc_randomizer_disable(mtd);
1077         sunxi_nfc_hw_ecc_disable(mtd);
1078
1079         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1080
1081         if (ret)
1082                 return ret;
1083
1084         status = readl(nfc->regs + NFC_REG_ECC_ST);
1085
1086         for (i = 0; i < nchunks; i++) {
1087                 int data_off = i * ecc->size;
1088                 int oob_off = i * (ecc->bytes + 4);
1089                 u8 *data = buf + data_off;
1090                 u8 *oob = nand->oob_poi + oob_off;
1091                 bool erased;
1092
1093                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1094                                                oob_required ? oob : NULL,
1095                                                i, status, &erased);
1096
1097                 /* ECC errors are handled in the second loop. */
1098                 if (ret < 0)
1099                         continue;
1100
1101                 if (oob_required && !erased) {
1102                         /* TODO: use DMA to retrieve OOB */
1103                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1104                                       mtd->writesize + oob_off, -1);
1105                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1106
1107                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1108                                                             !i, page);
1109                 }
1110
1111                 if (erased)
1112                         raw_mode = 1;
1113
1114                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1115         }
1116
1117         if (status & NFC_ECC_ERR_MSK) {
1118                 for (i = 0; i < nchunks; i++) {
1119                         int data_off = i * ecc->size;
1120                         int oob_off = i * (ecc->bytes + 4);
1121                         u8 *data = buf + data_off;
1122                         u8 *oob = nand->oob_poi + oob_off;
1123
1124                         if (!(status & NFC_ECC_ERR(i)))
1125                                 continue;
1126
1127                         /*
1128                          * Re-read the data with the randomizer disabled to
1129                          * identify bitflips in erased pages.
1130                          */
1131                         if (randomized) {
1132                                 /* TODO: use DMA to read page in raw mode */
1133                                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1134                                               data_off, -1);
1135                                 nand->read_buf(mtd, data, ecc->size);
1136                         }
1137
1138                         /* TODO: use DMA to retrieve OOB */
1139                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1140                                       mtd->writesize + oob_off, -1);
1141                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1142
1143                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1144                                                           oob, ecc->bytes + 4,
1145                                                           NULL, 0,
1146                                                           ecc->strength);
1147                         if (ret >= 0)
1148                                 raw_mode = 1;
1149
1150                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1151                 }
1152         }
1153
1154         if (oob_required)
1155                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1156                                                 NULL, !raw_mode,
1157                                                 page);
1158
1159         return max_bitflips;
1160 }
1161
1162 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1163                                         const u8 *data, int data_off,
1164                                         const u8 *oob, int oob_off,
1165                                         int *cur_off, bool bbm,
1166                                         int page)
1167 {
1168         struct nand_chip *nand = mtd_to_nand(mtd);
1169         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1170         struct nand_ecc_ctrl *ecc = &nand->ecc;
1171         int ret;
1172
1173         if (data_off != *cur_off)
1174                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1175
1176         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1177
1178         if (data_off + ecc->size != oob_off)
1179                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1180
1181         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1182         if (ret)
1183                 return ret;
1184
1185         sunxi_nfc_randomizer_enable(mtd);
1186         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1187
1188         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1189                NFC_ACCESS_DIR | NFC_ECC_OP,
1190                nfc->regs + NFC_REG_CMD);
1191
1192         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1193         sunxi_nfc_randomizer_disable(mtd);
1194         if (ret)
1195                 return ret;
1196
1197         *cur_off = oob_off + ecc->bytes + 4;
1198
1199         return 0;
1200 }
1201
1202 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1203                                              u8 *oob, int *cur_off,
1204                                              int page)
1205 {
1206         struct nand_chip *nand = mtd_to_nand(mtd);
1207         struct nand_ecc_ctrl *ecc = &nand->ecc;
1208         int offset = ((ecc->bytes + 4) * ecc->steps);
1209         int len = mtd->oobsize - offset;
1210
1211         if (len <= 0)
1212                 return;
1213
1214         if (!cur_off || *cur_off != offset)
1215                 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1216                               offset + mtd->writesize, -1);
1217
1218         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1219
1220         if (cur_off)
1221                 *cur_off = mtd->oobsize + mtd->writesize;
1222 }
1223
1224 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1225                                       struct nand_chip *chip, uint8_t *buf,
1226                                       int oob_required, int page)
1227 {
1228         struct nand_ecc_ctrl *ecc = &chip->ecc;
1229         unsigned int max_bitflips = 0;
1230         int ret, i, cur_off = 0;
1231         bool raw_mode = false;
1232
1233         sunxi_nfc_hw_ecc_enable(mtd);
1234
1235         for (i = 0; i < ecc->steps; i++) {
1236                 int data_off = i * ecc->size;
1237                 int oob_off = i * (ecc->bytes + 4);
1238                 u8 *data = buf + data_off;
1239                 u8 *oob = chip->oob_poi + oob_off;
1240
1241                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1242                                                   oob_off + mtd->writesize,
1243                                                   &cur_off, &max_bitflips,
1244                                                   !i, oob_required, page);
1245                 if (ret < 0)
1246                         return ret;
1247                 else if (ret)
1248                         raw_mode = true;
1249         }
1250
1251         if (oob_required)
1252                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1253                                                 !raw_mode, page);
1254
1255         sunxi_nfc_hw_ecc_disable(mtd);
1256
1257         return max_bitflips;
1258 }
1259
1260 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1261                                           struct nand_chip *chip, u8 *buf,
1262                                           int oob_required, int page)
1263 {
1264         int ret;
1265
1266         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1267                                                chip->ecc.steps);
1268         if (ret >= 0)
1269                 return ret;
1270
1271         /* Fallback to PIO mode */
1272         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1273
1274         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1275 }
1276
1277 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1278                                          struct nand_chip *chip,
1279                                          u32 data_offs, u32 readlen,
1280                                          u8 *bufpoi, int page)
1281 {
1282         struct nand_ecc_ctrl *ecc = &chip->ecc;
1283         int ret, i, cur_off = 0;
1284         unsigned int max_bitflips = 0;
1285
1286         sunxi_nfc_hw_ecc_enable(mtd);
1287
1288         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1289         for (i = data_offs / ecc->size;
1290              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1291                 int data_off = i * ecc->size;
1292                 int oob_off = i * (ecc->bytes + 4);
1293                 u8 *data = bufpoi + data_off;
1294                 u8 *oob = chip->oob_poi + oob_off;
1295
1296                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1297                                                   oob,
1298                                                   oob_off + mtd->writesize,
1299                                                   &cur_off, &max_bitflips, !i,
1300                                                   false, page);
1301                 if (ret < 0)
1302                         return ret;
1303         }
1304
1305         sunxi_nfc_hw_ecc_disable(mtd);
1306
1307         return max_bitflips;
1308 }
1309
1310 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1311                                              struct nand_chip *chip,
1312                                              u32 data_offs, u32 readlen,
1313                                              u8 *buf, int page)
1314 {
1315         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1316         int ret;
1317
1318         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1319         if (ret >= 0)
1320                 return ret;
1321
1322         /* Fallback to PIO mode */
1323         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1324
1325         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1326                                              buf, page);
1327 }
1328
1329 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1330                                        struct nand_chip *chip,
1331                                        const uint8_t *buf, int oob_required,
1332                                        int page)
1333 {
1334         struct nand_ecc_ctrl *ecc = &chip->ecc;
1335         int ret, i, cur_off = 0;
1336
1337         sunxi_nfc_hw_ecc_enable(mtd);
1338
1339         for (i = 0; i < ecc->steps; i++) {
1340                 int data_off = i * ecc->size;
1341                 int oob_off = i * (ecc->bytes + 4);
1342                 const u8 *data = buf + data_off;
1343                 const u8 *oob = chip->oob_poi + oob_off;
1344
1345                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1346                                                    oob_off + mtd->writesize,
1347                                                    &cur_off, !i, page);
1348                 if (ret)
1349                         return ret;
1350         }
1351
1352         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1353                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1354                                                  &cur_off, page);
1355
1356         sunxi_nfc_hw_ecc_disable(mtd);
1357
1358         return 0;
1359 }
1360
1361 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1362                                           struct nand_chip *chip,
1363                                           u32 data_offs, u32 data_len,
1364                                           const u8 *buf, int oob_required,
1365                                           int page)
1366 {
1367         struct nand_ecc_ctrl *ecc = &chip->ecc;
1368         int ret, i, cur_off = 0;
1369
1370         sunxi_nfc_hw_ecc_enable(mtd);
1371
1372         for (i = data_offs / ecc->size;
1373              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1374                 int data_off = i * ecc->size;
1375                 int oob_off = i * (ecc->bytes + 4);
1376                 const u8 *data = buf + data_off;
1377                 const u8 *oob = chip->oob_poi + oob_off;
1378
1379                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1380                                                    oob_off + mtd->writesize,
1381                                                    &cur_off, !i, page);
1382                 if (ret)
1383                         return ret;
1384         }
1385
1386         sunxi_nfc_hw_ecc_disable(mtd);
1387
1388         return 0;
1389 }
1390
1391 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1392                                            struct nand_chip *chip,
1393                                            const u8 *buf,
1394                                            int oob_required,
1395                                            int page)
1396 {
1397         struct nand_chip *nand = mtd_to_nand(mtd);
1398         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1399         struct nand_ecc_ctrl *ecc = &nand->ecc;
1400         struct scatterlist sg;
1401         int ret, i;
1402
1403         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1404         if (ret)
1405                 return ret;
1406
1407         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1408                                        DMA_TO_DEVICE, &sg);
1409         if (ret)
1410                 goto pio_fallback;
1411
1412         for (i = 0; i < ecc->steps; i++) {
1413                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1414
1415                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1416         }
1417
1418         sunxi_nfc_hw_ecc_enable(mtd);
1419         sunxi_nfc_randomizer_config(mtd, page, false);
1420         sunxi_nfc_randomizer_enable(mtd);
1421
1422         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1423                nfc->regs + NFC_REG_RCMD_SET);
1424
1425         dma_async_issue_pending(nfc->dmac);
1426
1427         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1428                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1429                nfc->regs + NFC_REG_CMD);
1430
1431         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1432         if (ret)
1433                 dmaengine_terminate_all(nfc->dmac);
1434
1435         sunxi_nfc_randomizer_disable(mtd);
1436         sunxi_nfc_hw_ecc_disable(mtd);
1437
1438         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1439
1440         if (ret)
1441                 return ret;
1442
1443         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1444                 /* TODO: use DMA to transfer extra OOB bytes ? */
1445                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1446                                                  NULL, page);
1447
1448         return 0;
1449
1450 pio_fallback:
1451         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1452 }
1453
1454 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1455                                                struct nand_chip *chip,
1456                                                uint8_t *buf, int oob_required,
1457                                                int page)
1458 {
1459         struct nand_ecc_ctrl *ecc = &chip->ecc;
1460         unsigned int max_bitflips = 0;
1461         int ret, i, cur_off = 0;
1462         bool raw_mode = false;
1463
1464         sunxi_nfc_hw_ecc_enable(mtd);
1465
1466         for (i = 0; i < ecc->steps; i++) {
1467                 int data_off = i * (ecc->size + ecc->bytes + 4);
1468                 int oob_off = data_off + ecc->size;
1469                 u8 *data = buf + (i * ecc->size);
1470                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1471
1472                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1473                                                   oob_off, &cur_off,
1474                                                   &max_bitflips, !i,
1475                                                   oob_required,
1476                                                   page);
1477                 if (ret < 0)
1478                         return ret;
1479                 else if (ret)
1480                         raw_mode = true;
1481         }
1482
1483         if (oob_required)
1484                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1485                                                 !raw_mode, page);
1486
1487         sunxi_nfc_hw_ecc_disable(mtd);
1488
1489         return max_bitflips;
1490 }
1491
1492 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1493                                                 struct nand_chip *chip,
1494                                                 const uint8_t *buf,
1495                                                 int oob_required, int page)
1496 {
1497         struct nand_ecc_ctrl *ecc = &chip->ecc;
1498         int ret, i, cur_off = 0;
1499
1500         sunxi_nfc_hw_ecc_enable(mtd);
1501
1502         for (i = 0; i < ecc->steps; i++) {
1503                 int data_off = i * (ecc->size + ecc->bytes + 4);
1504                 int oob_off = data_off + ecc->size;
1505                 const u8 *data = buf + (i * ecc->size);
1506                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1507
1508                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1509                                                    oob, oob_off, &cur_off,
1510                                                    false, page);
1511                 if (ret)
1512                         return ret;
1513         }
1514
1515         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1516                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1517                                                  &cur_off, page);
1518
1519         sunxi_nfc_hw_ecc_disable(mtd);
1520
1521         return 0;
1522 }
1523
1524 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1525                                             struct nand_chip *chip,
1526                                             int page)
1527 {
1528         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1529
1530         chip->pagebuf = -1;
1531
1532         return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1533 }
1534
1535 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1536                                              struct nand_chip *chip,
1537                                              int page)
1538 {
1539         int ret, status;
1540
1541         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1542
1543         chip->pagebuf = -1;
1544
1545         memset(chip->buffers->databuf, 0xff, mtd->writesize);
1546         ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1547         if (ret)
1548                 return ret;
1549
1550         /* Send command to program the OOB data */
1551         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1552
1553         status = chip->waitfunc(mtd, chip);
1554
1555         return status & NAND_STATUS_FAIL ? -EIO : 0;
1556 }
1557
1558 static const s32 tWB_lut[] = {6, 12, 16, 20};
1559 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1560
1561 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1562                 u32 clk_period)
1563 {
1564         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1565         int i;
1566
1567         for (i = 0; i < lut_size; i++) {
1568                 if (clk_cycles <= lut[i])
1569                         return i;
1570         }
1571
1572         /* Doesn't fit */
1573         return -EINVAL;
1574 }
1575
1576 #define sunxi_nand_lookup_timing(l, p, c) \
1577                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1578
1579 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1580                                         const struct nand_data_interface *conf,
1581                                         bool check_only)
1582 {
1583         struct nand_chip *nand = mtd_to_nand(mtd);
1584         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1585         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1586         const struct nand_sdr_timings *timings;
1587         u32 min_clk_period = 0;
1588         s32 tWB, tADL, tWHR, tRHW, tCAD;
1589         long real_clk_rate;
1590
1591         timings = nand_get_sdr_timings(conf);
1592         if (IS_ERR(timings))
1593                 return -ENOTSUPP;
1594
1595         /* T1 <=> tCLS */
1596         if (timings->tCLS_min > min_clk_period)
1597                 min_clk_period = timings->tCLS_min;
1598
1599         /* T2 <=> tCLH */
1600         if (timings->tCLH_min > min_clk_period)
1601                 min_clk_period = timings->tCLH_min;
1602
1603         /* T3 <=> tCS */
1604         if (timings->tCS_min > min_clk_period)
1605                 min_clk_period = timings->tCS_min;
1606
1607         /* T4 <=> tCH */
1608         if (timings->tCH_min > min_clk_period)
1609                 min_clk_period = timings->tCH_min;
1610
1611         /* T5 <=> tWP */
1612         if (timings->tWP_min > min_clk_period)
1613                 min_clk_period = timings->tWP_min;
1614
1615         /* T6 <=> tWH */
1616         if (timings->tWH_min > min_clk_period)
1617                 min_clk_period = timings->tWH_min;
1618
1619         /* T7 <=> tALS */
1620         if (timings->tALS_min > min_clk_period)
1621                 min_clk_period = timings->tALS_min;
1622
1623         /* T8 <=> tDS */
1624         if (timings->tDS_min > min_clk_period)
1625                 min_clk_period = timings->tDS_min;
1626
1627         /* T9 <=> tDH */
1628         if (timings->tDH_min > min_clk_period)
1629                 min_clk_period = timings->tDH_min;
1630
1631         /* T10 <=> tRR */
1632         if (timings->tRR_min > (min_clk_period * 3))
1633                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1634
1635         /* T11 <=> tALH */
1636         if (timings->tALH_min > min_clk_period)
1637                 min_clk_period = timings->tALH_min;
1638
1639         /* T12 <=> tRP */
1640         if (timings->tRP_min > min_clk_period)
1641                 min_clk_period = timings->tRP_min;
1642
1643         /* T13 <=> tREH */
1644         if (timings->tREH_min > min_clk_period)
1645                 min_clk_period = timings->tREH_min;
1646
1647         /* T14 <=> tRC */
1648         if (timings->tRC_min > (min_clk_period * 2))
1649                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1650
1651         /* T15 <=> tWC */
1652         if (timings->tWC_min > (min_clk_period * 2))
1653                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1654
1655         /* T16 - T19 + tCAD */
1656         if (timings->tWB_max > (min_clk_period * 20))
1657                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1658
1659         if (timings->tADL_min > (min_clk_period * 32))
1660                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1661
1662         if (timings->tWHR_min > (min_clk_period * 32))
1663                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1664
1665         if (timings->tRHW_min > (min_clk_period * 20))
1666                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1667
1668         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1669                                         min_clk_period);
1670         if (tWB < 0) {
1671                 dev_err(nfc->dev, "unsupported tWB\n");
1672                 return tWB;
1673         }
1674
1675         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1676         if (tADL > 3) {
1677                 dev_err(nfc->dev, "unsupported tADL\n");
1678                 return -EINVAL;
1679         }
1680
1681         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1682         if (tWHR > 3) {
1683                 dev_err(nfc->dev, "unsupported tWHR\n");
1684                 return -EINVAL;
1685         }
1686
1687         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1688                                         min_clk_period);
1689         if (tRHW < 0) {
1690                 dev_err(nfc->dev, "unsupported tRHW\n");
1691                 return tRHW;
1692         }
1693
1694         if (check_only)
1695                 return 0;
1696
1697         /*
1698          * TODO: according to ONFI specs this value only applies for DDR NAND,
1699          * but Allwinner seems to set this to 0x7. Mimic them for now.
1700          */
1701         tCAD = 0x7;
1702
1703         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1704         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1705
1706         /* Convert min_clk_period from picoseconds to nanoseconds */
1707         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1708
1709         /*
1710          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1711          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1712          * This new formula was verified with a scope and validated by
1713          * Allwinner engineers.
1714          */
1715         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1716         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1717
1718         /*
1719          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1720          * output cycle timings shall be used if the host drives tRC less than
1721          * 30 ns.
1722          */
1723         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1724         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1725                            NFC_TIMING_CTL_EDO : 0;
1726
1727         return 0;
1728 }
1729
1730 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1731                                     struct mtd_oob_region *oobregion)
1732 {
1733         struct nand_chip *nand = mtd_to_nand(mtd);
1734         struct nand_ecc_ctrl *ecc = &nand->ecc;
1735
1736         if (section >= ecc->steps)
1737                 return -ERANGE;
1738
1739         oobregion->offset = section * (ecc->bytes + 4) + 4;
1740         oobregion->length = ecc->bytes;
1741
1742         return 0;
1743 }
1744
1745 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1746                                      struct mtd_oob_region *oobregion)
1747 {
1748         struct nand_chip *nand = mtd_to_nand(mtd);
1749         struct nand_ecc_ctrl *ecc = &nand->ecc;
1750
1751         if (section > ecc->steps)
1752                 return -ERANGE;
1753
1754         /*
1755          * The first 2 bytes are used for BB markers, hence we
1756          * only have 2 bytes available in the first user data
1757          * section.
1758          */
1759         if (!section && ecc->mode == NAND_ECC_HW) {
1760                 oobregion->offset = 2;
1761                 oobregion->length = 2;
1762
1763                 return 0;
1764         }
1765
1766         oobregion->offset = section * (ecc->bytes + 4);
1767
1768         if (section < ecc->steps)
1769                 oobregion->length = 4;
1770         else
1771                 oobregion->offset = mtd->oobsize - oobregion->offset;
1772
1773         return 0;
1774 }
1775
1776 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1777         .ecc = sunxi_nand_ooblayout_ecc,
1778         .free = sunxi_nand_ooblayout_free,
1779 };
1780
1781 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1782                                               struct nand_ecc_ctrl *ecc,
1783                                               struct device_node *np)
1784 {
1785         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1786         struct nand_chip *nand = mtd_to_nand(mtd);
1787         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1788         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1789         struct sunxi_nand_hw_ecc *data;
1790         int nsectors;
1791         int ret;
1792         int i;
1793
1794         if (ecc->options & NAND_ECC_MAXIMIZE) {
1795                 int bytes;
1796
1797                 ecc->size = 1024;
1798                 nsectors = mtd->writesize / ecc->size;
1799
1800                 /* Reserve 2 bytes for the BBM */
1801                 bytes = (mtd->oobsize - 2) / nsectors;
1802
1803                 /* 4 non-ECC bytes are added before each ECC bytes section */
1804                 bytes -= 4;
1805
1806                 /* and bytes has to be even. */
1807                 if (bytes % 2)
1808                         bytes--;
1809
1810                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1811
1812                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1813                         if (strengths[i] > ecc->strength)
1814                                 break;
1815                 }
1816
1817                 if (!i)
1818                         ecc->strength = 0;
1819                 else
1820                         ecc->strength = strengths[i - 1];
1821         }
1822
1823         if (ecc->size != 512 && ecc->size != 1024)
1824                 return -EINVAL;
1825
1826         data = kzalloc(sizeof(*data), GFP_KERNEL);
1827         if (!data)
1828                 return -ENOMEM;
1829
1830         /* Prefer 1k ECC chunk over 512 ones */
1831         if (ecc->size == 512 && mtd->writesize > 512) {
1832                 ecc->size = 1024;
1833                 ecc->strength *= 2;
1834         }
1835
1836         /* Add ECC info retrieval from DT */
1837         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1838                 if (ecc->strength <= strengths[i])
1839                         break;
1840         }
1841
1842         if (i >= ARRAY_SIZE(strengths)) {
1843                 dev_err(nfc->dev, "unsupported strength\n");
1844                 ret = -ENOTSUPP;
1845                 goto err;
1846         }
1847
1848         data->mode = i;
1849
1850         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1851         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1852
1853         /* HW ECC always work with even numbers of ECC bytes */
1854         ecc->bytes = ALIGN(ecc->bytes, 2);
1855
1856         nsectors = mtd->writesize / ecc->size;
1857
1858         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1859                 ret = -EINVAL;
1860                 goto err;
1861         }
1862
1863         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1864         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1865         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1866         ecc->priv = data;
1867
1868         return 0;
1869
1870 err:
1871         kfree(data);
1872
1873         return ret;
1874 }
1875
1876 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1877 {
1878         kfree(ecc->priv);
1879 }
1880
1881 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1882                                        struct nand_ecc_ctrl *ecc,
1883                                        struct device_node *np)
1884 {
1885         struct nand_chip *nand = mtd_to_nand(mtd);
1886         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1887         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1888         int ret;
1889
1890         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1891         if (ret)
1892                 return ret;
1893
1894         if (nfc->dmac) {
1895                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1896                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1897                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1898                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1899         } else {
1900                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1901                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1902                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1903         }
1904
1905         /* TODO: support DMA for raw accesses and subpage write */
1906         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1907         ecc->read_oob_raw = nand_read_oob_std;
1908         ecc->write_oob_raw = nand_write_oob_std;
1909         ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1910
1911         return 0;
1912 }
1913
1914 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1915                                                 struct nand_ecc_ctrl *ecc,
1916                                                 struct device_node *np)
1917 {
1918         int ret;
1919
1920         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1921         if (ret)
1922                 return ret;
1923
1924         ecc->prepad = 4;
1925         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1926         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1927         ecc->read_oob_raw = nand_read_oob_syndrome;
1928         ecc->write_oob_raw = nand_write_oob_syndrome;
1929
1930         return 0;
1931 }
1932
1933 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1934 {
1935         switch (ecc->mode) {
1936         case NAND_ECC_HW:
1937         case NAND_ECC_HW_SYNDROME:
1938                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1939                 break;
1940         case NAND_ECC_NONE:
1941         default:
1942                 break;
1943         }
1944 }
1945
1946 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1947                                struct device_node *np)
1948 {
1949         struct nand_chip *nand = mtd_to_nand(mtd);
1950         int ret;
1951
1952         if (!ecc->size) {
1953                 ecc->size = nand->ecc_step_ds;
1954                 ecc->strength = nand->ecc_strength_ds;
1955         }
1956
1957         if (!ecc->size || !ecc->strength)
1958                 return -EINVAL;
1959
1960         switch (ecc->mode) {
1961         case NAND_ECC_HW:
1962                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1963                 if (ret)
1964                         return ret;
1965                 break;
1966         case NAND_ECC_HW_SYNDROME:
1967                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1968                 if (ret)
1969                         return ret;
1970                 break;
1971         case NAND_ECC_NONE:
1972         case NAND_ECC_SOFT:
1973                 break;
1974         default:
1975                 return -EINVAL;
1976         }
1977
1978         return 0;
1979 }
1980
1981 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1982                                 struct device_node *np)
1983 {
1984         struct sunxi_nand_chip *chip;
1985         struct mtd_info *mtd;
1986         struct nand_chip *nand;
1987         int nsels;
1988         int ret;
1989         int i;
1990         u32 tmp;
1991
1992         if (!of_get_property(np, "reg", &nsels))
1993                 return -EINVAL;
1994
1995         nsels /= sizeof(u32);
1996         if (!nsels) {
1997                 dev_err(dev, "invalid reg property size\n");
1998                 return -EINVAL;
1999         }
2000
2001         chip = devm_kzalloc(dev,
2002                             sizeof(*chip) +
2003                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2004                             GFP_KERNEL);
2005         if (!chip) {
2006                 dev_err(dev, "could not allocate chip\n");
2007                 return -ENOMEM;
2008         }
2009
2010         chip->nsels = nsels;
2011         chip->selected = -1;
2012
2013         for (i = 0; i < nsels; i++) {
2014                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2015                 if (ret) {
2016                         dev_err(dev, "could not retrieve reg property: %d\n",
2017                                 ret);
2018                         return ret;
2019                 }
2020
2021                 if (tmp > NFC_MAX_CS) {
2022                         dev_err(dev,
2023                                 "invalid reg value: %u (max CS = 7)\n",
2024                                 tmp);
2025                         return -EINVAL;
2026                 }
2027
2028                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2029                         dev_err(dev, "CS %d already assigned\n", tmp);
2030                         return -EINVAL;
2031                 }
2032
2033                 chip->sels[i].cs = tmp;
2034
2035                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2036                     tmp < 2) {
2037                         chip->sels[i].rb.type = RB_NATIVE;
2038                         chip->sels[i].rb.info.nativeid = tmp;
2039                 } else {
2040                         ret = of_get_named_gpio(np, "rb-gpios", i);
2041                         if (ret >= 0) {
2042                                 tmp = ret;
2043                                 chip->sels[i].rb.type = RB_GPIO;
2044                                 chip->sels[i].rb.info.gpio = tmp;
2045                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2046                                 if (ret)
2047                                         return ret;
2048
2049                                 ret = gpio_direction_input(tmp);
2050                                 if (ret)
2051                                         return ret;
2052                         } else {
2053                                 chip->sels[i].rb.type = RB_NONE;
2054                         }
2055                 }
2056         }
2057
2058         nand = &chip->nand;
2059         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2060         nand->chip_delay = 200;
2061         nand->controller = &nfc->controller;
2062         /*
2063          * Set the ECC mode to the default value in case nothing is specified
2064          * in the DT.
2065          */
2066         nand->ecc.mode = NAND_ECC_HW;
2067         nand_set_flash_node(nand, np);
2068         nand->select_chip = sunxi_nfc_select_chip;
2069         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2070         nand->read_buf = sunxi_nfc_read_buf;
2071         nand->write_buf = sunxi_nfc_write_buf;
2072         nand->read_byte = sunxi_nfc_read_byte;
2073         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2074
2075         mtd = nand_to_mtd(nand);
2076         mtd->dev.parent = dev;
2077
2078         ret = nand_scan_ident(mtd, nsels, NULL);
2079         if (ret)
2080                 return ret;
2081
2082         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2083                 nand->bbt_options |= NAND_BBT_NO_OOB;
2084
2085         if (nand->options & NAND_NEED_SCRAMBLING)
2086                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2087
2088         nand->options |= NAND_SUBPAGE_READ;
2089
2090         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2091         if (ret) {
2092                 dev_err(dev, "ECC init failed: %d\n", ret);
2093                 return ret;
2094         }
2095
2096         ret = nand_scan_tail(mtd);
2097         if (ret) {
2098                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2099                 return ret;
2100         }
2101
2102         ret = mtd_device_register(mtd, NULL, 0);
2103         if (ret) {
2104                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2105                 nand_release(mtd);
2106                 return ret;
2107         }
2108
2109         list_add_tail(&chip->node, &nfc->chips);
2110
2111         return 0;
2112 }
2113
2114 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2115 {
2116         struct device_node *np = dev->of_node;
2117         struct device_node *nand_np;
2118         int nchips = of_get_child_count(np);
2119         int ret;
2120
2121         if (nchips > 8) {
2122                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2123                 return -EINVAL;
2124         }
2125
2126         for_each_child_of_node(np, nand_np) {
2127                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2128                 if (ret) {
2129                         of_node_put(nand_np);
2130                         return ret;
2131                 }
2132         }
2133
2134         return 0;
2135 }
2136
2137 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2138 {
2139         struct sunxi_nand_chip *chip;
2140
2141         while (!list_empty(&nfc->chips)) {
2142                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2143                                         node);
2144                 nand_release(nand_to_mtd(&chip->nand));
2145                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2146                 list_del(&chip->node);
2147         }
2148 }
2149
2150 static int sunxi_nfc_probe(struct platform_device *pdev)
2151 {
2152         struct device *dev = &pdev->dev;
2153         struct resource *r;
2154         struct sunxi_nfc *nfc;
2155         int irq;
2156         int ret;
2157
2158         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2159         if (!nfc)
2160                 return -ENOMEM;
2161
2162         nfc->dev = dev;
2163         nand_hw_control_init(&nfc->controller);
2164         INIT_LIST_HEAD(&nfc->chips);
2165
2166         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2167         nfc->regs = devm_ioremap_resource(dev, r);
2168         if (IS_ERR(nfc->regs))
2169                 return PTR_ERR(nfc->regs);
2170
2171         irq = platform_get_irq(pdev, 0);
2172         if (irq < 0) {
2173                 dev_err(dev, "failed to retrieve irq\n");
2174                 return irq;
2175         }
2176
2177         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2178         if (IS_ERR(nfc->ahb_clk)) {
2179                 dev_err(dev, "failed to retrieve ahb clk\n");
2180                 return PTR_ERR(nfc->ahb_clk);
2181         }
2182
2183         ret = clk_prepare_enable(nfc->ahb_clk);
2184         if (ret)
2185                 return ret;
2186
2187         nfc->mod_clk = devm_clk_get(dev, "mod");
2188         if (IS_ERR(nfc->mod_clk)) {
2189                 dev_err(dev, "failed to retrieve mod clk\n");
2190                 ret = PTR_ERR(nfc->mod_clk);
2191                 goto out_ahb_clk_unprepare;
2192         }
2193
2194         ret = clk_prepare_enable(nfc->mod_clk);
2195         if (ret)
2196                 goto out_ahb_clk_unprepare;
2197
2198         nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2199         if (!IS_ERR(nfc->reset)) {
2200                 ret = reset_control_deassert(nfc->reset);
2201                 if (ret) {
2202                         dev_err(dev, "reset err %d\n", ret);
2203                         goto out_mod_clk_unprepare;
2204                 }
2205         } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2206                 ret = PTR_ERR(nfc->reset);
2207                 goto out_mod_clk_unprepare;
2208         }
2209
2210         ret = sunxi_nfc_rst(nfc);
2211         if (ret)
2212                 goto out_ahb_reset_reassert;
2213
2214         writel(0, nfc->regs + NFC_REG_INT);
2215         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2216                                0, "sunxi-nand", nfc);
2217         if (ret)
2218                 goto out_ahb_reset_reassert;
2219
2220         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2221         if (nfc->dmac) {
2222                 struct dma_slave_config dmac_cfg = { };
2223
2224                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2225                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2226                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2227                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2228                 dmac_cfg.src_maxburst = 4;
2229                 dmac_cfg.dst_maxburst = 4;
2230                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2231         } else {
2232                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2233         }
2234
2235         platform_set_drvdata(pdev, nfc);
2236
2237         ret = sunxi_nand_chips_init(dev, nfc);
2238         if (ret) {
2239                 dev_err(dev, "failed to init nand chips\n");
2240                 goto out_release_dmac;
2241         }
2242
2243         return 0;
2244
2245 out_release_dmac:
2246         if (nfc->dmac)
2247                 dma_release_channel(nfc->dmac);
2248 out_ahb_reset_reassert:
2249         if (!IS_ERR(nfc->reset))
2250                 reset_control_assert(nfc->reset);
2251 out_mod_clk_unprepare:
2252         clk_disable_unprepare(nfc->mod_clk);
2253 out_ahb_clk_unprepare:
2254         clk_disable_unprepare(nfc->ahb_clk);
2255
2256         return ret;
2257 }
2258
2259 static int sunxi_nfc_remove(struct platform_device *pdev)
2260 {
2261         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2262
2263         sunxi_nand_chips_cleanup(nfc);
2264
2265         if (!IS_ERR(nfc->reset))
2266                 reset_control_assert(nfc->reset);
2267
2268         if (nfc->dmac)
2269                 dma_release_channel(nfc->dmac);
2270         clk_disable_unprepare(nfc->mod_clk);
2271         clk_disable_unprepare(nfc->ahb_clk);
2272
2273         return 0;
2274 }
2275
2276 static const struct of_device_id sunxi_nfc_ids[] = {
2277         { .compatible = "allwinner,sun4i-a10-nand" },
2278         { /* sentinel */ }
2279 };
2280 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2281
2282 static struct platform_driver sunxi_nfc_driver = {
2283         .driver = {
2284                 .name = "sunxi_nand",
2285                 .of_match_table = sunxi_nfc_ids,
2286         },
2287         .probe = sunxi_nfc_probe,
2288         .remove = sunxi_nfc_remove,
2289 };
2290 module_platform_driver(sunxi_nfc_driver);
2291
2292 MODULE_LICENSE("GPL v2");
2293 MODULE_AUTHOR("Boris BREZILLON");
2294 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2295 MODULE_ALIAS("platform:sunxi_nand");