drivers/mtd/nand/raw/sunxi_nand.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   4  *
   5  * Derived from:
   6  *      https://github.com/yuq/sunxi-nfc-mtd
   7  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   8  *
   9  *      https://github.com/hno/Allwinner-Info
  10  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  11  *
  12  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  13  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  14  */
  15
  16 #include <linux/dma-mapping.h>
  17 #include <linux/slab.h>
  18 #include <linux/module.h>
  19 #include <linux/moduleparam.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/of.h>
  22 #include <linux/of_device.h>
  23 #include <linux/mtd/mtd.h>
  24 #include <linux/mtd/rawnand.h>
  25 #include <linux/mtd/partitions.h>
  26 #include <linux/clk.h>
  27 #include <linux/delay.h>
  28 #include <linux/dmaengine.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/iopoll.h>
  31 #include <linux/reset.h>
  32
  33 #define NFC_REG_CTL             0x0000
  34 #define NFC_REG_ST              0x0004
  35 #define NFC_REG_INT             0x0008
  36 #define NFC_REG_TIMING_CTL      0x000C
  37 #define NFC_REG_TIMING_CFG      0x0010
  38 #define NFC_REG_ADDR_LOW        0x0014
  39 #define NFC_REG_ADDR_HIGH       0x0018
  40 #define NFC_REG_SECTOR_NUM      0x001C
  41 #define NFC_REG_CNT             0x0020
  42 #define NFC_REG_CMD             0x0024
  43 #define NFC_REG_RCMD_SET        0x0028
  44 #define NFC_REG_WCMD_SET        0x002C
  45 #define NFC_REG_A10_IO_DATA     0x0030
  46 #define NFC_REG_ECC_CTL         0x0034
  47 #define NFC_REG_ECC_ST          0x0038
  48 #define NFC_REG_DEBUG           0x003C
  49 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  50 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  51 #define NFC_REG_SPARE_AREA      0x00A0
  52 #define NFC_REG_PAT_ID          0x00A4
  53 #define NFC_RAM0_BASE           0x0400
  54 #define NFC_RAM1_BASE           0x0800
  55
  56 /* define bit use in NFC_CTL */
  57 #define NFC_EN                  BIT(0)
  58 #define NFC_RESET               BIT(1)
  59 #define NFC_BUS_WIDTH_MSK       BIT(2)
  60 #define NFC_BUS_WIDTH_8         (0 << 2)
  61 #define NFC_BUS_WIDTH_16        (1 << 2)
  62 #define NFC_RB_SEL_MSK          BIT(3)
  63 #define NFC_RB_SEL(x)           ((x) << 3)
  64 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  65 #define NFC_CE_SEL(x)           ((x) << 24)
  66 #define NFC_CE_CTL              BIT(6)
  67 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  68 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  69 #define NFC_SAM                 BIT(12)
  70 #define NFC_RAM_METHOD          BIT(14)
  71 #define NFC_DEBUG_CTL           BIT(31)
  72
  73 /* define bit use in NFC_ST */
  74 #define NFC_RB_B2R              BIT(0)
  75 #define NFC_CMD_INT_FLAG        BIT(1)
  76 #define NFC_DMA_INT_FLAG        BIT(2)
  77 #define NFC_CMD_FIFO_STATUS     BIT(3)
  78 #define NFC_STA                 BIT(4)
  79 #define NFC_NATCH_INT_FLAG      BIT(5)
  80 #define NFC_RB_STATE(x)         BIT(x + 8)
  81
  82 /* define bit use in NFC_INT */
  83 #define NFC_B2R_INT_ENABLE      BIT(0)
  84 #define NFC_CMD_INT_ENABLE      BIT(1)
  85 #define NFC_DMA_INT_ENABLE      BIT(2)
  86 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  87                                  NFC_CMD_INT_ENABLE | \
  88                                  NFC_DMA_INT_ENABLE)
  89
  90 /* define bit use in NFC_TIMING_CTL */
  91 #define NFC_TIMING_CTL_EDO      BIT(8)
  92
  93 /* define NFC_TIMING_CFG register layout */
  94 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
  95         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
  96         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
  97         (((tCAD) & 0x7) << 8))
  98
  99 /* define bit use in NFC_CMD */
 100 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 101 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 102 #define NFC_CMD(x)              (x)
 103 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 104 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 105 #define NFC_SEND_ADR            BIT(19)
 106 #define NFC_ACCESS_DIR          BIT(20)
 107 #define NFC_DATA_TRANS          BIT(21)
 108 #define NFC_SEND_CMD1           BIT(22)
 109 #define NFC_WAIT_FLAG           BIT(23)
 110 #define NFC_SEND_CMD2           BIT(24)
 111 #define NFC_SEQ                 BIT(25)
 112 #define NFC_DATA_SWAP_METHOD    BIT(26)
 113 #define NFC_ROW_AUTO_INC        BIT(27)
 114 #define NFC_SEND_CMD3           BIT(28)
 115 #define NFC_SEND_CMD4           BIT(29)
 116 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 117 #define NFC_NORMAL_OP           (0 << 30)
 118 #define NFC_ECC_OP              (1 << 30)
 119 #define NFC_PAGE_OP             (2U << 30)
 120
 121 /* define bit use in NFC_RCMD_SET */
 122 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 123 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 124 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 125
 126 /* define bit use in NFC_WCMD_SET */
 127 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 128 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 129 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 130 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 131
 132 /* define bit use in NFC_ECC_CTL */
 133 #define NFC_ECC_EN              BIT(0)
 134 #define NFC_ECC_PIPELINE        BIT(3)
 135 #define NFC_ECC_EXCEPTION       BIT(4)
 136 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 137 #define NFC_ECC_BLOCK_512       BIT(5)
 138 #define NFC_RANDOM_EN           BIT(9)
 139 #define NFC_RANDOM_DIRECTION    BIT(10)
 140 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 141 #define NFC_ECC_MODE(x)         ((x) << 12)
 142 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 143 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 144
 145 /* define bit use in NFC_ECC_ST */
 146 #define NFC_ECC_ERR(x)          BIT(x)
 147 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 148 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 149 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 150
 151 #define NFC_DEFAULT_TIMEOUT_MS  1000
 152
 153 #define NFC_SRAM_SIZE           1024
 154
 155 #define NFC_MAX_CS              7
 156
 157 /**
 158  * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
 159  *
 160  * @cs: the NAND CS id used to communicate with a NAND Chip
 161  * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
 162  */
 163 struct sunxi_nand_chip_sel {
 164         u8 cs;
 165         s8 rb;
 166 };
 167
 168 /**
 169  * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
 170  *
 171  * @mode: the sunxi ECC mode field deduced from ECC requirements
 172  */
 173 struct sunxi_nand_hw_ecc {
 174         int mode;
 175 };
 176
 177 /**
 178  * struct sunxi_nand_chip - stores NAND chip device related information
 179  *
 180  * @node: used to store NAND chips into a list
 181  * @nand: base NAND chip structure
 182  * @clk_rate: clk_rate required for this NAND chip
 183  * @timing_cfg: TIMING_CFG register value for this NAND chip
 184  * @timing_ctl: TIMING_CTL register value for this NAND chip
 185  * @nsels: number of CS lines required by the NAND chip
 186  * @sels: array of CS lines descriptions
 187  */
 188 struct sunxi_nand_chip {
 189         struct list_head node;
 190         struct nand_chip nand;
 191         unsigned long clk_rate;
 192         u32 timing_cfg;
 193         u32 timing_ctl;
 194         int nsels;
 195         struct sunxi_nand_chip_sel sels[0];
 196 };
 197
 198 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 199 {
 200         return container_of(nand, struct sunxi_nand_chip, nand);
 201 }
 202
 203 /*
 204  * NAND Controller capabilities structure: stores NAND controller capabilities
 205  * for distinction between compatible strings.
 206  *
 207  * @reg_io_data:        I/O data register
 208  * @dma_maxburst:       DMA maxburst
 209  */
 210 struct sunxi_nfc_caps {
 211         unsigned int reg_io_data;
 212         unsigned int dma_maxburst;
 213 };
 214
 215 /**
 216  * struct sunxi_nfc - stores sunxi NAND controller information
 217  *
 218  * @controller: base controller structure
 219  * @dev: parent device (used to print error messages)
 220  * @regs: NAND controller registers
 221  * @ahb_clk: NAND controller AHB clock
 222  * @mod_clk: NAND controller mod clock
 223  * @reset: NAND controller reset line
 224  * @assigned_cs: bitmask describing already assigned CS lines
 225  * @clk_rate: NAND controller current clock rate
 226  * @chips: a list containing all the NAND chips attached to this NAND
 227  *         controller
 228  * @complete: a completion object used to wait for NAND controller events
 229  * @dmac: the DMA channel attached to the NAND controller
 230  */
 231 struct sunxi_nfc {
 232         struct nand_controller controller;
 233         struct device *dev;
 234         void __iomem *regs;
 235         struct clk *ahb_clk;
 236         struct clk *mod_clk;
 237         struct reset_control *reset;
 238         unsigned long assigned_cs;
 239         unsigned long clk_rate;
 240         struct list_head chips;
 241         struct completion complete;
 242         struct dma_chan *dmac;
 243         const struct sunxi_nfc_caps *caps;
 244 };
 245
 246 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 247 {
 248         return container_of(ctrl, struct sunxi_nfc, controller);
 249 }
 250
 251 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 252 {
 253         struct sunxi_nfc *nfc = dev_id;
 254         u32 st = readl(nfc->regs + NFC_REG_ST);
 255         u32 ien = readl(nfc->regs + NFC_REG_INT);
 256
 257         if (!(ien & st))
 258                 return IRQ_NONE;
 259
 260         if ((ien & st) == ien)
 261                 complete(&nfc->complete);
 262
 263         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 264         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 265
 266         return IRQ_HANDLED;
 267 }
 268
 269 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 270                                  bool use_polling, unsigned int timeout_ms)
 271 {
 272         int ret;
 273
 274         if (events & ~NFC_INT_MASK)
 275                 return -EINVAL;
 276
 277         if (!timeout_ms)
 278                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 279
 280         if (!use_polling) {
 281                 init_completion(&nfc->complete);
 282
 283                 writel(events, nfc->regs + NFC_REG_INT);
 284
 285                 ret = wait_for_completion_timeout(&nfc->complete,
 286                                                 msecs_to_jiffies(timeout_ms));
 287                 if (!ret)
 288                         ret = -ETIMEDOUT;
 289                 else
 290                         ret = 0;
 291
 292                 writel(0, nfc->regs + NFC_REG_INT);
 293         } else {
 294                 u32 status;
 295
 296                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 297                                          (status & events) == events, 1,
 298                                          timeout_ms * 1000);
 299         }
 300
 301         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 302
 303         if (ret)
 304                 dev_err(nfc->dev, "wait interrupt timedout\n");
 305
 306         return ret;
 307 }
 308
 309 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 310 {
 311         u32 status;
 312         int ret;
 313
 314         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 315                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 316                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 317         if (ret)
 318                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 319
 320         return ret;
 321 }
 322
 323 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 324 {
 325         u32 ctl;
 326         int ret;
 327
 328         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 329         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 330
 331         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 332                                  !(ctl & NFC_RESET), 1,
 333                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 334         if (ret)
 335                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 336
 337         return ret;
 338 }
 339
 340 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
 341                                     int chunksize, int nchunks,
 342                                     enum dma_data_direction ddir,
 343                                     struct scatterlist *sg)
 344 {
 345         struct dma_async_tx_descriptor *dmad;
 346         enum dma_transfer_direction tdir;
 347         dma_cookie_t dmat;
 348         int ret;
 349
 350         if (ddir == DMA_FROM_DEVICE)
 351                 tdir = DMA_DEV_TO_MEM;
 352         else
 353                 tdir = DMA_MEM_TO_DEV;
 354
 355         sg_init_one(sg, buf, nchunks * chunksize);
 356         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 357         if (!ret)
 358                 return -ENOMEM;
 359
 360         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 361         if (!dmad) {
 362                 ret = -EINVAL;
 363                 goto err_unmap_buf;
 364         }
 365
 366         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 367                nfc->regs + NFC_REG_CTL);
 368         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 369         writel(chunksize, nfc->regs + NFC_REG_CNT);
 370         dmat = dmaengine_submit(dmad);
 371
 372         ret = dma_submit_error(dmat);
 373         if (ret)
 374                 goto err_clr_dma_flag;
 375
 376         return 0;
 377
 378 err_clr_dma_flag:
 379         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 380                nfc->regs + NFC_REG_CTL);
 381
 382 err_unmap_buf:
 383         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 384         return ret;
 385 }
 386
 387 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
 388                                      enum dma_data_direction ddir,
 389                                      struct scatterlist *sg)
 390 {
 391         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 392         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 393                nfc->regs + NFC_REG_CTL);
 394 }
 395
 396 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
 397 {
 398         struct mtd_info *mtd = nand_to_mtd(nand);
 399         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 400         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 401         struct sunxi_nand_chip_sel *sel;
 402         u32 ctl;
 403
 404         if (cs > 0 && cs >= sunxi_nand->nsels)
 405                 return;
 406
 407         ctl = readl(nfc->regs + NFC_REG_CTL) &
 408               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 409
 410         sel = &sunxi_nand->sels[cs];
 411         ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
 412         if (sel->rb >= 0)
 413                 ctl |= NFC_RB_SEL(sel->rb);
 414
 415         writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 416
 417         if (nfc->clk_rate != sunxi_nand->clk_rate) {
 418                 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 419                 nfc->clk_rate = sunxi_nand->clk_rate;
 420         }
 421
 422         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 423         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 424         writel(ctl, nfc->regs + NFC_REG_CTL);
 425 }
 426
 427 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
 428 {
 429         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 430         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 431         int ret;
 432         int cnt;
 433         int offs = 0;
 434         u32 tmp;
 435
 436         while (len > offs) {
 437                 bool poll = false;
 438
 439                 cnt = min(len - offs, NFC_SRAM_SIZE);
 440
 441                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 442                 if (ret)
 443                         break;
 444
 445                 writel(cnt, nfc->regs + NFC_REG_CNT);
 446                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 447                 writel(tmp, nfc->regs + NFC_REG_CMD);
 448
 449                 /* Arbitrary limit for polling mode */
 450                 if (cnt < 64)
 451                         poll = true;
 452
 453                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 454                 if (ret)
 455                         break;
 456
 457                 if (buf)
 458                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 459                                       cnt);
 460                 offs += cnt;
 461         }
 462 }
 463
 464 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
 465                                 int len)
 466 {
 467         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 468         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 469         int ret;
 470         int cnt;
 471         int offs = 0;
 472         u32 tmp;
 473
 474         while (len > offs) {
 475                 bool poll = false;
 476
 477                 cnt = min(len - offs, NFC_SRAM_SIZE);
 478
 479                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 480                 if (ret)
 481                         break;
 482
 483                 writel(cnt, nfc->regs + NFC_REG_CNT);
 484                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 485                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 486                       NFC_ACCESS_DIR;
 487                 writel(tmp, nfc->regs + NFC_REG_CMD);
 488
 489                 /* Arbitrary limit for polling mode */
 490                 if (cnt < 64)
 491                         poll = true;
 492
 493                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 494                 if (ret)
 495                         break;
 496
 497                 offs += cnt;
 498         }
 499 }
 500
 501 /* These seed values have been extracted from Allwinner's BSP */
 502 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 503         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 504         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 505         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 506         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 507         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 508         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 509         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 510         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 511         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 512         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 513         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 514         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 515         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 516         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 517         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 518         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 519 };
 520
 521 /*
 522  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 523  * have been generated using
 524  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 525  * the randomizer engine does internally before de/scrambling OOB data.
 526  *
 527  * Those tables are statically defined to avoid calculating randomizer state
 528  * at runtime.
 529  */
 530 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 531         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 532         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 533         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 534         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 535         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 536         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 537         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 538         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 539         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 540         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 541         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 542         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 543         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 544         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 545         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 546         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 547 };
 548
 549 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 550         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 551         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 552         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 553         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 554         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 555         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 556         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 557         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 558         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 559         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 560         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 561         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 562         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 563         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 564         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 565         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 566 };
 567
 568 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 569 {
 570         state &= 0x7fff;
 571
 572         /*
 573          * This loop is just a simple implementation of a Fibonacci LFSR using
 574          * the x16 + x15 + 1 polynomial.
 575          */
 576         while (count--)
 577                 state = ((state >> 1) |
 578                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 579
 580         return state;
 581 }
 582
 583 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
 584                                       bool ecc)
 585 {
 586         struct mtd_info *mtd = nand_to_mtd(nand);
 587         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 588         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 589
 590         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 591                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 592
 593         if (ecc) {
 594                 if (mtd->ecc_step_size == 512)
 595                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 596                 else
 597                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 598         }
 599
 600         return seeds[page % mod];
 601 }
 602
 603 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
 604                                         bool ecc)
 605 {
 606         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 607         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 608         u16 state;
 609
 610         if (!(nand->options & NAND_NEED_SCRAMBLING))
 611                 return;
 612
 613         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 614         state = sunxi_nfc_randomizer_state(nand, page, ecc);
 615         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 616         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 617 }
 618
 619 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
 620 {
 621         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 622
 623         if (!(nand->options & NAND_NEED_SCRAMBLING))
 624                 return;
 625
 626         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 627                nfc->regs + NFC_REG_ECC_CTL);
 628 }
 629
 630 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
 631 {
 632         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 633
 634         if (!(nand->options & NAND_NEED_SCRAMBLING))
 635                 return;
 636
 637         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 638                nfc->regs + NFC_REG_ECC_CTL);
 639 }
 640
 641 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
 642 {
 643         u16 state = sunxi_nfc_randomizer_state(nand, page, true);
 644
 645         bbm[0] ^= state;
 646         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 647 }
 648
 649 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
 650                                            const uint8_t *buf, int len,
 651                                            bool ecc, int page)
 652 {
 653         sunxi_nfc_randomizer_config(nand, page, ecc);
 654         sunxi_nfc_randomizer_enable(nand);
 655         sunxi_nfc_write_buf(nand, buf, len);
 656         sunxi_nfc_randomizer_disable(nand);
 657 }
 658
 659 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
 660                                           int len, bool ecc, int page)
 661 {
 662         sunxi_nfc_randomizer_config(nand, page, ecc);
 663         sunxi_nfc_randomizer_enable(nand);
 664         sunxi_nfc_read_buf(nand, buf, len);
 665         sunxi_nfc_randomizer_disable(nand);
 666 }
 667
 668 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
 669 {
 670         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 671         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 672         u32 ecc_ctl;
 673
 674         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 675         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 676                      NFC_ECC_BLOCK_SIZE_MSK);
 677         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 678                    NFC_ECC_PIPELINE;
 679
 680         if (nand->ecc.size == 512)
 681                 ecc_ctl |= NFC_ECC_BLOCK_512;
 682
 683         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 684 }
 685
 686 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
 687 {
 688         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 689
 690         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 691                nfc->regs + NFC_REG_ECC_CTL);
 692 }
 693
 694 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 695 {
 696         buf[0] = user_data;
 697         buf[1] = user_data >> 8;
 698         buf[2] = user_data >> 16;
 699         buf[3] = user_data >> 24;
 700 }
 701
 702 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 703 {
 704         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 705 }
 706
 707 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
 708                                                 int step, bool bbm, int page)
 709 {
 710         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 711
 712         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 713                                    oob);
 714
 715         /* De-randomize the Bad Block Marker. */
 716         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 717                 sunxi_nfc_randomize_bbm(nand, page, oob);
 718 }
 719
 720 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 721                                                 const u8 *oob, int step,
 722                                                 bool bbm, int page)
 723 {
 724         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 725         u8 user_data[4];
 726
 727         /* Randomize the Bad Block Marker. */
 728         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 729                 memcpy(user_data, oob, sizeof(user_data));
 730                 sunxi_nfc_randomize_bbm(nand, page, user_data);
 731                 oob = user_data;
 732         }
 733
 734         writel(sunxi_nfc_buf_to_user_data(oob),
 735                nfc->regs + NFC_REG_USER_DATA(step));
 736 }
 737
 738 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
 739                                           unsigned int *max_bitflips, int ret)
 740 {
 741         struct mtd_info *mtd = nand_to_mtd(nand);
 742
 743         if (ret < 0) {
 744                 mtd->ecc_stats.failed++;
 745         } else {
 746                 mtd->ecc_stats.corrected += ret;
 747                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 748         }
 749 }
 750
 751 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
 752                                     int step, u32 status, bool *erased)
 753 {
 754         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 755         struct nand_ecc_ctrl *ecc = &nand->ecc;
 756         u32 tmp;
 757
 758         *erased = false;
 759
 760         if (status & NFC_ECC_ERR(step))
 761                 return -EBADMSG;
 762
 763         if (status & NFC_ECC_PAT_FOUND(step)) {
 764                 u8 pattern;
 765
 766                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 767                         pattern = 0x0;
 768                 } else {
 769                         pattern = 0xff;
 770                         *erased = true;
 771                 }
 772
 773                 if (data)
 774                         memset(data, pattern, ecc->size);
 775
 776                 if (oob)
 777                         memset(oob, pattern, ecc->bytes + 4);
 778
 779                 return 0;
 780         }
 781
 782         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 783
 784         return NFC_ECC_ERR_CNT(step, tmp);
 785 }
 786
 787 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
 788                                        u8 *data, int data_off,
 789                                        u8 *oob, int oob_off,
 790                                        int *cur_off,
 791                                        unsigned int *max_bitflips,
 792                                        bool bbm, bool oob_required, int page)
 793 {
 794         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 795         struct nand_ecc_ctrl *ecc = &nand->ecc;
 796         int raw_mode = 0;
 797         bool erased;
 798         int ret;
 799
 800         if (*cur_off != data_off)
 801                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 802
 803         sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
 804
 805         if (data_off + ecc->size != oob_off)
 806                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 807
 808         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 809         if (ret)
 810                 return ret;
 811
 812         sunxi_nfc_randomizer_enable(nand);
 813         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 814                nfc->regs + NFC_REG_CMD);
 815
 816         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 817         sunxi_nfc_randomizer_disable(nand);
 818         if (ret)
 819                 return ret;
 820
 821         *cur_off = oob_off + ecc->bytes + 4;
 822
 823         ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
 824                                        readl(nfc->regs + NFC_REG_ECC_ST),
 825                                        &erased);
 826         if (erased)
 827                 return 1;
 828
 829         if (ret < 0) {
 830                 /*
 831                  * Re-read the data with the randomizer disabled to identify
 832                  * bitflips in erased pages.
 833                  */
 834                 if (nand->options & NAND_NEED_SCRAMBLING)
 835                         nand_change_read_column_op(nand, data_off, data,
 836                                                    ecc->size, false);
 837                 else
 838                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 839                                       ecc->size);
 840
 841                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 842                                            false);
 843
 844                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 845                                                   oob, ecc->bytes + 4,
 846                                                   NULL, 0, ecc->strength);
 847                 if (ret >= 0)
 848                         raw_mode = 1;
 849         } else {
 850                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 851
 852                 if (oob_required) {
 853                         nand_change_read_column_op(nand, oob_off, NULL, 0,
 854                                                    false);
 855                         sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
 856                                                       true, page);
 857
 858                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
 859                                                             bbm, page);
 860                 }
 861         }
 862
 863         sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
 864
 865         return raw_mode;
 866 }
 867
 868 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
 869                                             u8 *oob, int *cur_off,
 870                                             bool randomize, int page)
 871 {
 872         struct mtd_info *mtd = nand_to_mtd(nand);
 873         struct nand_ecc_ctrl *ecc = &nand->ecc;
 874         int offset = ((ecc->bytes + 4) * ecc->steps);
 875         int len = mtd->oobsize - offset;
 876
 877         if (len <= 0)
 878                 return;
 879
 880         if (!cur_off || *cur_off != offset)
 881                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 882                                            false);
 883
 884         if (!randomize)
 885                 sunxi_nfc_read_buf(nand, oob + offset, len);
 886         else
 887                 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
 888                                               false, page);
 889
 890         if (cur_off)
 891                 *cur_off = mtd->oobsize + mtd->writesize;
 892 }
 893
 894 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
 895                                             int oob_required, int page,
 896                                             int nchunks)
 897 {
 898         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
 899         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 900         struct mtd_info *mtd = nand_to_mtd(nand);
 901         struct nand_ecc_ctrl *ecc = &nand->ecc;
 902         unsigned int max_bitflips = 0;
 903         int ret, i, raw_mode = 0;
 904         struct scatterlist sg;
 905         u32 status;
 906
 907         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 908         if (ret)
 909                 return ret;
 910
 911         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
 912                                        DMA_FROM_DEVICE, &sg);
 913         if (ret)
 914                 return ret;
 915
 916         sunxi_nfc_hw_ecc_enable(nand);
 917         sunxi_nfc_randomizer_config(nand, page, false);
 918         sunxi_nfc_randomizer_enable(nand);
 919
 920         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
 921                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
 922
 923         dma_async_issue_pending(nfc->dmac);
 924
 925         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
 926                nfc->regs + NFC_REG_CMD);
 927
 928         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 929         if (ret)
 930                 dmaengine_terminate_all(nfc->dmac);
 931
 932         sunxi_nfc_randomizer_disable(nand);
 933         sunxi_nfc_hw_ecc_disable(nand);
 934
 935         sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
 936
 937         if (ret)
 938                 return ret;
 939
 940         status = readl(nfc->regs + NFC_REG_ECC_ST);
 941
 942         for (i = 0; i < nchunks; i++) {
 943                 int data_off = i * ecc->size;
 944                 int oob_off = i * (ecc->bytes + 4);
 945                 u8 *data = buf + data_off;
 946                 u8 *oob = nand->oob_poi + oob_off;
 947                 bool erased;
 948
 949                 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
 950                                                oob_required ? oob : NULL,
 951                                                i, status, &erased);
 952
 953                 /* ECC errors are handled in the second loop. */
 954                 if (ret < 0)
 955                         continue;
 956
 957                 if (oob_required && !erased) {
 958                         /* TODO: use DMA to retrieve OOB */
 959                         nand_change_read_column_op(nand,
 960                                                    mtd->writesize + oob_off,
 961                                                    oob, ecc->bytes + 4, false);
 962
 963                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
 964                                                             !i, page);
 965                 }
 966
 967                 if (erased)
 968                         raw_mode = 1;
 969
 970                 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
 971         }
 972
 973         if (status & NFC_ECC_ERR_MSK) {
 974                 for (i = 0; i < nchunks; i++) {
 975                         int data_off = i * ecc->size;
 976                         int oob_off = i * (ecc->bytes + 4);
 977                         u8 *data = buf + data_off;
 978                         u8 *oob = nand->oob_poi + oob_off;
 979
 980                         if (!(status & NFC_ECC_ERR(i)))
 981                                 continue;
 982
 983                         /*
 984                          * Re-read the data with the randomizer disabled to
 985                          * identify bitflips in erased pages.
 986                          * TODO: use DMA to read page in raw mode
 987                          */
 988                         if (randomized)
 989                                 nand_change_read_column_op(nand, data_off,
 990                                                            data, ecc->size,
 991                                                            false);
 992
 993                         /* TODO: use DMA to retrieve OOB */
 994                         nand_change_read_column_op(nand,
 995                                                    mtd->writesize + oob_off,
 996                                                    oob, ecc->bytes + 4, false);
 997
 998                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
 999                                                           oob, ecc->bytes + 4,
1000                                                           NULL, 0,
1001                                                           ecc->strength);
1002                         if (ret >= 0)
1003                                 raw_mode = 1;
1004
1005                         sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
1006                 }
1007         }
1008
1009         if (oob_required)
1010                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
1011                                                 NULL, !raw_mode,
1012                                                 page);
1013
1014         return max_bitflips;
1015 }
1016
1017 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1018                                         const u8 *data, int data_off,
1019                                         const u8 *oob, int oob_off,
1020                                         int *cur_off, bool bbm,
1021                                         int page)
1022 {
1023         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1024         struct nand_ecc_ctrl *ecc = &nand->ecc;
1025         int ret;
1026
1027         if (data_off != *cur_off)
1028                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1029
1030         sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1031
1032         if (data_off + ecc->size != oob_off)
1033                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1034
1035         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1036         if (ret)
1037                 return ret;
1038
1039         sunxi_nfc_randomizer_enable(nand);
1040         sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1041
1042         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1043                NFC_ACCESS_DIR | NFC_ECC_OP,
1044                nfc->regs + NFC_REG_CMD);
1045
1046         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1047         sunxi_nfc_randomizer_disable(nand);
1048         if (ret)
1049                 return ret;
1050
1051         *cur_off = oob_off + ecc->bytes + 4;
1052
1053         return 0;
1054 }
1055
1056 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1057                                              u8 *oob, int *cur_off,
1058                                              int page)
1059 {
1060         struct mtd_info *mtd = nand_to_mtd(nand);
1061         struct nand_ecc_ctrl *ecc = &nand->ecc;
1062         int offset = ((ecc->bytes + 4) * ecc->steps);
1063         int len = mtd->oobsize - offset;
1064
1065         if (len <= 0)
1066                 return;
1067
1068         if (!cur_off || *cur_off != offset)
1069                 nand_change_write_column_op(nand, offset + mtd->writesize,
1070                                             NULL, 0, false);
1071
1072         sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1073
1074         if (cur_off)
1075                 *cur_off = mtd->oobsize + mtd->writesize;
1076 }
1077
1078 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1079                                       int oob_required, int page)
1080 {
1081         struct mtd_info *mtd = nand_to_mtd(nand);
1082         struct nand_ecc_ctrl *ecc = &nand->ecc;
1083         unsigned int max_bitflips = 0;
1084         int ret, i, cur_off = 0;
1085         bool raw_mode = false;
1086
1087         sunxi_nfc_select_chip(nand, nand->cur_cs);
1088
1089         nand_read_page_op(nand, page, 0, NULL, 0);
1090
1091         sunxi_nfc_hw_ecc_enable(nand);
1092
1093         for (i = 0; i < ecc->steps; i++) {
1094                 int data_off = i * ecc->size;
1095                 int oob_off = i * (ecc->bytes + 4);
1096                 u8 *data = buf + data_off;
1097                 u8 *oob = nand->oob_poi + oob_off;
1098
1099                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1100                                                   oob_off + mtd->writesize,
1101                                                   &cur_off, &max_bitflips,
1102                                                   !i, oob_required, page);
1103                 if (ret < 0)
1104                         return ret;
1105                 else if (ret)
1106                         raw_mode = true;
1107         }
1108
1109         if (oob_required)
1110                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1111                                                 !raw_mode, page);
1112
1113         sunxi_nfc_hw_ecc_disable(nand);
1114
1115         return max_bitflips;
1116 }
1117
1118 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1119                                           int oob_required, int page)
1120 {
1121         int ret;
1122
1123         sunxi_nfc_select_chip(nand, nand->cur_cs);
1124
1125         nand_read_page_op(nand, page, 0, NULL, 0);
1126
1127         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1128                                                nand->ecc.steps);
1129         if (ret >= 0)
1130                 return ret;
1131
1132         /* Fallback to PIO mode */
1133         return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1134 }
1135
1136 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1137                                          u32 data_offs, u32 readlen,
1138                                          u8 *bufpoi, int page)
1139 {
1140         struct mtd_info *mtd = nand_to_mtd(nand);
1141         struct nand_ecc_ctrl *ecc = &nand->ecc;
1142         int ret, i, cur_off = 0;
1143         unsigned int max_bitflips = 0;
1144
1145         sunxi_nfc_select_chip(nand, nand->cur_cs);
1146
1147         nand_read_page_op(nand, page, 0, NULL, 0);
1148
1149         sunxi_nfc_hw_ecc_enable(nand);
1150
1151         for (i = data_offs / ecc->size;
1152              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1153                 int data_off = i * ecc->size;
1154                 int oob_off = i * (ecc->bytes + 4);
1155                 u8 *data = bufpoi + data_off;
1156                 u8 *oob = nand->oob_poi + oob_off;
1157
1158                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1159                                                   oob,
1160                                                   oob_off + mtd->writesize,
1161                                                   &cur_off, &max_bitflips, !i,
1162                                                   false, page);
1163                 if (ret < 0)
1164                         return ret;
1165         }
1166
1167         sunxi_nfc_hw_ecc_disable(nand);
1168
1169         return max_bitflips;
1170 }
1171
1172 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1173                                              u32 data_offs, u32 readlen,
1174                                              u8 *buf, int page)
1175 {
1176         int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1177         int ret;
1178
1179         sunxi_nfc_select_chip(nand, nand->cur_cs);
1180
1181         nand_read_page_op(nand, page, 0, NULL, 0);
1182
1183         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1184         if (ret >= 0)
1185                 return ret;
1186
1187         /* Fallback to PIO mode */
1188         return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1189                                              buf, page);
1190 }
1191
1192 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1193                                        const uint8_t *buf, int oob_required,
1194                                        int page)
1195 {
1196         struct mtd_info *mtd = nand_to_mtd(nand);
1197         struct nand_ecc_ctrl *ecc = &nand->ecc;
1198         int ret, i, cur_off = 0;
1199
1200         sunxi_nfc_select_chip(nand, nand->cur_cs);
1201
1202         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1203
1204         sunxi_nfc_hw_ecc_enable(nand);
1205
1206         for (i = 0; i < ecc->steps; i++) {
1207                 int data_off = i * ecc->size;
1208                 int oob_off = i * (ecc->bytes + 4);
1209                 const u8 *data = buf + data_off;
1210                 const u8 *oob = nand->oob_poi + oob_off;
1211
1212                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1213                                                    oob_off + mtd->writesize,
1214                                                    &cur_off, !i, page);
1215                 if (ret)
1216                         return ret;
1217         }
1218
1219         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1220                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1221                                                  &cur_off, page);
1222
1223         sunxi_nfc_hw_ecc_disable(nand);
1224
1225         return nand_prog_page_end_op(nand);
1226 }
1227
1228 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1229                                           u32 data_offs, u32 data_len,
1230                                           const u8 *buf, int oob_required,
1231                                           int page)
1232 {
1233         struct mtd_info *mtd = nand_to_mtd(nand);
1234         struct nand_ecc_ctrl *ecc = &nand->ecc;
1235         int ret, i, cur_off = 0;
1236
1237         sunxi_nfc_select_chip(nand, nand->cur_cs);
1238
1239         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1240
1241         sunxi_nfc_hw_ecc_enable(nand);
1242
1243         for (i = data_offs / ecc->size;
1244              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1245                 int data_off = i * ecc->size;
1246                 int oob_off = i * (ecc->bytes + 4);
1247                 const u8 *data = buf + data_off;
1248                 const u8 *oob = nand->oob_poi + oob_off;
1249
1250                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1251                                                    oob_off + mtd->writesize,
1252                                                    &cur_off, !i, page);
1253                 if (ret)
1254                         return ret;
1255         }
1256
1257         sunxi_nfc_hw_ecc_disable(nand);
1258
1259         return nand_prog_page_end_op(nand);
1260 }
1261
1262 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1263                                            const u8 *buf,
1264                                            int oob_required,
1265                                            int page)
1266 {
1267         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1268         struct nand_ecc_ctrl *ecc = &nand->ecc;
1269         struct scatterlist sg;
1270         int ret, i;
1271
1272         sunxi_nfc_select_chip(nand, nand->cur_cs);
1273
1274         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1275         if (ret)
1276                 return ret;
1277
1278         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1279                                        DMA_TO_DEVICE, &sg);
1280         if (ret)
1281                 goto pio_fallback;
1282
1283         for (i = 0; i < ecc->steps; i++) {
1284                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1285
1286                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1287         }
1288
1289         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1290
1291         sunxi_nfc_hw_ecc_enable(nand);
1292         sunxi_nfc_randomizer_config(nand, page, false);
1293         sunxi_nfc_randomizer_enable(nand);
1294
1295         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1296                nfc->regs + NFC_REG_WCMD_SET);
1297
1298         dma_async_issue_pending(nfc->dmac);
1299
1300         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1301                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1302                nfc->regs + NFC_REG_CMD);
1303
1304         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1305         if (ret)
1306                 dmaengine_terminate_all(nfc->dmac);
1307
1308         sunxi_nfc_randomizer_disable(nand);
1309         sunxi_nfc_hw_ecc_disable(nand);
1310
1311         sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1312
1313         if (ret)
1314                 return ret;
1315
1316         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1317                 /* TODO: use DMA to transfer extra OOB bytes ? */
1318                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1319                                                  NULL, page);
1320
1321         return nand_prog_page_end_op(nand);
1322
1323 pio_fallback:
1324         return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1325 }
1326
1327 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1328 {
1329         u8 *buf = nand_get_data_buf(nand);
1330
1331         return nand->ecc.read_page(nand, buf, 1, page);
1332 }
1333
1334 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1335 {
1336         struct mtd_info *mtd = nand_to_mtd(nand);
1337         u8 *buf = nand_get_data_buf(nand);
1338         int ret;
1339
1340         memset(buf, 0xff, mtd->writesize);
1341         ret = nand->ecc.write_page(nand, buf, 1, page);
1342         if (ret)
1343                 return ret;
1344
1345         /* Send command to program the OOB data */
1346         return nand_prog_page_end_op(nand);
1347 }
1348
1349 static const s32 tWB_lut[] = {6, 12, 16, 20};
1350 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1351
1352 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1353                 u32 clk_period)
1354 {
1355         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1356         int i;
1357
1358         for (i = 0; i < lut_size; i++) {
1359                 if (clk_cycles <= lut[i])
1360                         return i;
1361         }
1362
1363         /* Doesn't fit */
1364         return -EINVAL;
1365 }
1366
1367 #define sunxi_nand_lookup_timing(l, p, c) \
1368                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1369
1370 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline,
1371                                         const struct nand_data_interface *conf)
1372 {
1373         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1374         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1375         const struct nand_sdr_timings *timings;
1376         u32 min_clk_period = 0;
1377         s32 tWB, tADL, tWHR, tRHW, tCAD;
1378         long real_clk_rate;
1379
1380         timings = nand_get_sdr_timings(conf);
1381         if (IS_ERR(timings))
1382                 return -ENOTSUPP;
1383
1384         /* T1 <=> tCLS */
1385         if (timings->tCLS_min > min_clk_period)
1386                 min_clk_period = timings->tCLS_min;
1387
1388         /* T2 <=> tCLH */
1389         if (timings->tCLH_min > min_clk_period)
1390                 min_clk_period = timings->tCLH_min;
1391
1392         /* T3 <=> tCS */
1393         if (timings->tCS_min > min_clk_period)
1394                 min_clk_period = timings->tCS_min;
1395
1396         /* T4 <=> tCH */
1397         if (timings->tCH_min > min_clk_period)
1398                 min_clk_period = timings->tCH_min;
1399
1400         /* T5 <=> tWP */
1401         if (timings->tWP_min > min_clk_period)
1402                 min_clk_period = timings->tWP_min;
1403
1404         /* T6 <=> tWH */
1405         if (timings->tWH_min > min_clk_period)
1406                 min_clk_period = timings->tWH_min;
1407
1408         /* T7 <=> tALS */
1409         if (timings->tALS_min > min_clk_period)
1410                 min_clk_period = timings->tALS_min;
1411
1412         /* T8 <=> tDS */
1413         if (timings->tDS_min > min_clk_period)
1414                 min_clk_period = timings->tDS_min;
1415
1416         /* T9 <=> tDH */
1417         if (timings->tDH_min > min_clk_period)
1418                 min_clk_period = timings->tDH_min;
1419
1420         /* T10 <=> tRR */
1421         if (timings->tRR_min > (min_clk_period * 3))
1422                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1423
1424         /* T11 <=> tALH */
1425         if (timings->tALH_min > min_clk_period)
1426                 min_clk_period = timings->tALH_min;
1427
1428         /* T12 <=> tRP */
1429         if (timings->tRP_min > min_clk_period)
1430                 min_clk_period = timings->tRP_min;
1431
1432         /* T13 <=> tREH */
1433         if (timings->tREH_min > min_clk_period)
1434                 min_clk_period = timings->tREH_min;
1435
1436         /* T14 <=> tRC */
1437         if (timings->tRC_min > (min_clk_period * 2))
1438                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1439
1440         /* T15 <=> tWC */
1441         if (timings->tWC_min > (min_clk_period * 2))
1442                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1443
1444         /* T16 - T19 + tCAD */
1445         if (timings->tWB_max > (min_clk_period * 20))
1446                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1447
1448         if (timings->tADL_min > (min_clk_period * 32))
1449                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1450
1451         if (timings->tWHR_min > (min_clk_period * 32))
1452                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1453
1454         if (timings->tRHW_min > (min_clk_period * 20))
1455                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1456
1457         /*
1458          * In non-EDO, tREA should be less than tRP to guarantee that the
1459          * controller does not sample the IO lines too early. Unfortunately,
1460          * the sunxi NAND controller does not allow us to have different
1461          * values for tRP and tREH (tRP = tREH = tRW / 2).
1462          *
1463          * We have 2 options to overcome this limitation:
1464          *
1465          * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1466          * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1467          */
1468         if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1469                 min_clk_period = timings->tREA_max;
1470
1471         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1472                                         min_clk_period);
1473         if (tWB < 0) {
1474                 dev_err(nfc->dev, "unsupported tWB\n");
1475                 return tWB;
1476         }
1477
1478         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1479         if (tADL > 3) {
1480                 dev_err(nfc->dev, "unsupported tADL\n");
1481                 return -EINVAL;
1482         }
1483
1484         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1485         if (tWHR > 3) {
1486                 dev_err(nfc->dev, "unsupported tWHR\n");
1487                 return -EINVAL;
1488         }
1489
1490         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1491                                         min_clk_period);
1492         if (tRHW < 0) {
1493                 dev_err(nfc->dev, "unsupported tRHW\n");
1494                 return tRHW;
1495         }
1496
1497         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1498                 return 0;
1499
1500         /*
1501          * TODO: according to ONFI specs this value only applies for DDR NAND,
1502          * but Allwinner seems to set this to 0x7. Mimic them for now.
1503          */
1504         tCAD = 0x7;
1505
1506         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1507         sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1508
1509         /* Convert min_clk_period from picoseconds to nanoseconds */
1510         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1511
1512         /*
1513          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1514          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1515          * This new formula was verified with a scope and validated by
1516          * Allwinner engineers.
1517          */
1518         sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1519         real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1520         if (real_clk_rate <= 0) {
1521                 dev_err(nfc->dev, "Unable to round clk %lu\n",
1522                         sunxi_nand->clk_rate);
1523                 return -EINVAL;
1524         }
1525
1526         sunxi_nand->timing_ctl = 0;
1527
1528         /*
1529          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1530          * output cycle timings shall be used if the host drives tRC less than
1531          * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1532          */
1533         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1534         if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1535                 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1536
1537         return 0;
1538 }
1539
1540 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1541                                     struct mtd_oob_region *oobregion)
1542 {
1543         struct nand_chip *nand = mtd_to_nand(mtd);
1544         struct nand_ecc_ctrl *ecc = &nand->ecc;
1545
1546         if (section >= ecc->steps)
1547                 return -ERANGE;
1548
1549         oobregion->offset = section * (ecc->bytes + 4) + 4;
1550         oobregion->length = ecc->bytes;
1551
1552         return 0;
1553 }
1554
1555 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1556                                      struct mtd_oob_region *oobregion)
1557 {
1558         struct nand_chip *nand = mtd_to_nand(mtd);
1559         struct nand_ecc_ctrl *ecc = &nand->ecc;
1560
1561         if (section > ecc->steps)
1562                 return -ERANGE;
1563
1564         /*
1565          * The first 2 bytes are used for BB markers, hence we
1566          * only have 2 bytes available in the first user data
1567          * section.
1568          */
1569         if (!section && ecc->mode == NAND_ECC_HW) {
1570                 oobregion->offset = 2;
1571                 oobregion->length = 2;
1572
1573                 return 0;
1574         }
1575
1576         oobregion->offset = section * (ecc->bytes + 4);
1577
1578         if (section < ecc->steps)
1579                 oobregion->length = 4;
1580         else
1581                 oobregion->offset = mtd->oobsize - oobregion->offset;
1582
1583         return 0;
1584 }
1585
1586 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1587         .ecc = sunxi_nand_ooblayout_ecc,
1588         .free = sunxi_nand_ooblayout_free,
1589 };
1590
1591 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1592 {
1593         kfree(ecc->priv);
1594 }
1595
1596 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1597                                        struct nand_ecc_ctrl *ecc,
1598                                        struct device_node *np)
1599 {
1600         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1601         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1602         struct mtd_info *mtd = nand_to_mtd(nand);
1603         struct sunxi_nand_hw_ecc *data;
1604         int nsectors;
1605         int ret;
1606         int i;
1607
1608         if (ecc->options & NAND_ECC_MAXIMIZE) {
1609                 int bytes;
1610
1611                 ecc->size = 1024;
1612                 nsectors = mtd->writesize / ecc->size;
1613
1614                 /* Reserve 2 bytes for the BBM */
1615                 bytes = (mtd->oobsize - 2) / nsectors;
1616
1617                 /* 4 non-ECC bytes are added before each ECC bytes section */
1618                 bytes -= 4;
1619
1620                 /* and bytes has to be even. */
1621                 if (bytes % 2)
1622                         bytes--;
1623
1624                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1625
1626                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1627                         if (strengths[i] > ecc->strength)
1628                                 break;
1629                 }
1630
1631                 if (!i)
1632                         ecc->strength = 0;
1633                 else
1634                         ecc->strength = strengths[i - 1];
1635         }
1636
1637         if (ecc->size != 512 && ecc->size != 1024)
1638                 return -EINVAL;
1639
1640         data = kzalloc(sizeof(*data), GFP_KERNEL);
1641         if (!data)
1642                 return -ENOMEM;
1643
1644         /* Prefer 1k ECC chunk over 512 ones */
1645         if (ecc->size == 512 && mtd->writesize > 512) {
1646                 ecc->size = 1024;
1647                 ecc->strength *= 2;
1648         }
1649
1650         /* Add ECC info retrieval from DT */
1651         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1652                 if (ecc->strength <= strengths[i]) {
1653                         /*
1654                          * Update ecc->strength value with the actual strength
1655                          * that will be used by the ECC engine.
1656                          */
1657                         ecc->strength = strengths[i];
1658                         break;
1659                 }
1660         }
1661
1662         if (i >= ARRAY_SIZE(strengths)) {
1663                 dev_err(nfc->dev, "unsupported strength\n");
1664                 ret = -ENOTSUPP;
1665                 goto err;
1666         }
1667
1668         data->mode = i;
1669
1670         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1671         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1672
1673         /* HW ECC always work with even numbers of ECC bytes */
1674         ecc->bytes = ALIGN(ecc->bytes, 2);
1675
1676         nsectors = mtd->writesize / ecc->size;
1677
1678         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1679                 ret = -EINVAL;
1680                 goto err;
1681         }
1682
1683         ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1684         ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1685         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1686         ecc->priv = data;
1687
1688         if (nfc->dmac) {
1689                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1690                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1691                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1692                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1693         } else {
1694                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1695                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1696                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1697         }
1698
1699         /* TODO: support DMA for raw accesses and subpage write */
1700         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1701         ecc->read_oob_raw = nand_read_oob_std;
1702         ecc->write_oob_raw = nand_write_oob_std;
1703
1704         return 0;
1705
1706 err:
1707         kfree(data);
1708
1709         return ret;
1710 }
1711
1712 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1713 {
1714         switch (ecc->mode) {
1715         case NAND_ECC_HW:
1716                 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1717                 break;
1718         case NAND_ECC_NONE:
1719         default:
1720                 break;
1721         }
1722 }
1723
1724 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1725 {
1726         struct nand_ecc_ctrl *ecc = &nand->ecc;
1727         struct device_node *np = nand_get_flash_node(nand);
1728         int ret;
1729
1730         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1731                 nand->bbt_options |= NAND_BBT_NO_OOB;
1732
1733         if (nand->options & NAND_NEED_SCRAMBLING)
1734                 nand->options |= NAND_NO_SUBPAGE_WRITE;
1735
1736         nand->options |= NAND_SUBPAGE_READ;
1737
1738         if (!ecc->size) {
1739                 ecc->size = nand->base.eccreq.step_size;
1740                 ecc->strength = nand->base.eccreq.strength;
1741         }
1742
1743         if (!ecc->size || !ecc->strength)
1744                 return -EINVAL;
1745
1746         switch (ecc->mode) {
1747         case NAND_ECC_HW:
1748                 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1749                 if (ret)
1750                         return ret;
1751                 break;
1752         case NAND_ECC_NONE:
1753         case NAND_ECC_SOFT:
1754                 break;
1755         default:
1756                 return -EINVAL;
1757         }
1758
1759         return 0;
1760 }
1761
1762 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1763                                 const struct nand_subop *subop)
1764 {
1765         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1766         u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1767         unsigned int i, j, remaining, start;
1768         void *inbuf = NULL;
1769         int ret;
1770
1771         for (i = 0; i < subop->ninstrs; i++) {
1772                 const struct nand_op_instr *instr = &subop->instrs[i];
1773
1774                 switch (instr->type) {
1775                 case NAND_OP_CMD_INSTR:
1776                         if (cmd & NFC_SEND_CMD1) {
1777                                 if (WARN_ON(cmd & NFC_SEND_CMD2))
1778                                         return -EINVAL;
1779
1780                                 cmd |= NFC_SEND_CMD2;
1781                                 extcmd |= instr->ctx.cmd.opcode;
1782                         } else {
1783                                 cmd |= NFC_SEND_CMD1 |
1784                                        NFC_CMD(instr->ctx.cmd.opcode);
1785                         }
1786                         break;
1787
1788                 case NAND_OP_ADDR_INSTR:
1789                         remaining = nand_subop_get_num_addr_cyc(subop, i);
1790                         start = nand_subop_get_addr_start_off(subop, i);
1791                         for (j = 0; j < 8 && j + start < remaining; j++) {
1792                                 u32 addr = instr->ctx.addr.addrs[j + start];
1793
1794                                 addrs[j / 4] |= addr << (j % 4) * 8;
1795                         }
1796
1797                         if (j)
1798                                 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1799
1800                         break;
1801
1802                 case NAND_OP_DATA_IN_INSTR:
1803                 case NAND_OP_DATA_OUT_INSTR:
1804                         start = nand_subop_get_data_start_off(subop, i);
1805                         remaining = nand_subop_get_data_len(subop, i);
1806                         cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1807                         cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1808
1809                         if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1810                                 cmd |= NFC_ACCESS_DIR;
1811                                 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1812                                             instr->ctx.data.buf.out + start,
1813                                             cnt);
1814                         } else {
1815                                 inbuf = instr->ctx.data.buf.in + start;
1816                         }
1817
1818                         break;
1819
1820                 case NAND_OP_WAITRDY_INSTR:
1821                         cmd |= NFC_WAIT_FLAG;
1822                         break;
1823                 }
1824         }
1825
1826         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1827         if (ret)
1828                 return ret;
1829
1830         if (cmd & NFC_SEND_ADR) {
1831                 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1832                 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1833         }
1834
1835         if (cmd & NFC_SEND_CMD2)
1836                 writel(extcmd,
1837                        nfc->regs +
1838                        (cmd & NFC_ACCESS_DIR ?
1839                         NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1840
1841         if (cmd & NFC_DATA_TRANS)
1842                 writel(cnt, nfc->regs + NFC_REG_CNT);
1843
1844         writel(cmd, nfc->regs + NFC_REG_CMD);
1845
1846         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1847                                     !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1848                                     0);
1849         if (ret)
1850                 return ret;
1851
1852         if (inbuf)
1853                 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1854
1855         return 0;
1856 }
1857
1858 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1859                                   const struct nand_subop *subop)
1860 {
1861         return nand_soft_waitrdy(nand,
1862                                  subop->instrs[0].ctx.waitrdy.timeout_ms);
1863 }
1864
1865 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1866         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1867                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1868                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1869                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1870                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1871                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1872         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1873                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1874                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1875                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1876                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1877                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1878 );
1879
1880 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1881         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1882                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1883                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1884                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1885                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1886         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1887                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1888                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1889                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1890                                NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1891         NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1892                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1893 );
1894
1895 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1896                              const struct nand_operation *op, bool check_only)
1897 {
1898         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1899         const struct nand_op_parser *parser;
1900
1901         sunxi_nfc_select_chip(nand, op->cs);
1902
1903         if (sunxi_nand->sels[op->cs].rb >= 0)
1904                 parser = &sunxi_nfc_op_parser;
1905         else
1906                 parser = &sunxi_nfc_norb_op_parser;
1907
1908         return nand_op_parser_exec_op(nand, parser, op, check_only);
1909 }
1910
1911 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1912         .attach_chip = sunxi_nand_attach_chip,
1913         .setup_data_interface = sunxi_nfc_setup_data_interface,
1914         .exec_op = sunxi_nfc_exec_op,
1915 };
1916
1917 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1918                                 struct device_node *np)
1919 {
1920         struct sunxi_nand_chip *sunxi_nand;
1921         struct mtd_info *mtd;
1922         struct nand_chip *nand;
1923         int nsels;
1924         int ret;
1925         int i;
1926         u32 tmp;
1927
1928         if (!of_get_property(np, "reg", &nsels))
1929                 return -EINVAL;
1930
1931         nsels /= sizeof(u32);
1932         if (!nsels) {
1933                 dev_err(dev, "invalid reg property size\n");
1934                 return -EINVAL;
1935         }
1936
1937         sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1938                                   GFP_KERNEL);
1939         if (!sunxi_nand) {
1940                 dev_err(dev, "could not allocate chip\n");
1941                 return -ENOMEM;
1942         }
1943
1944         sunxi_nand->nsels = nsels;
1945
1946         for (i = 0; i < nsels; i++) {
1947                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1948                 if (ret) {
1949                         dev_err(dev, "could not retrieve reg property: %d\n",
1950                                 ret);
1951                         return ret;
1952                 }
1953
1954                 if (tmp > NFC_MAX_CS) {
1955                         dev_err(dev,
1956                                 "invalid reg value: %u (max CS = 7)\n",
1957                                 tmp);
1958                         return -EINVAL;
1959                 }
1960
1961                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1962                         dev_err(dev, "CS %d already assigned\n", tmp);
1963                         return -EINVAL;
1964                 }
1965
1966                 sunxi_nand->sels[i].cs = tmp;
1967
1968                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1969                     tmp < 2)
1970                         sunxi_nand->sels[i].rb = tmp;
1971                 else
1972                         sunxi_nand->sels[i].rb = -1;
1973         }
1974
1975         nand = &sunxi_nand->nand;
1976         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1977         nand->controller = &nfc->controller;
1978         nand->controller->ops = &sunxi_nand_controller_ops;
1979
1980         /*
1981          * Set the ECC mode to the default value in case nothing is specified
1982          * in the DT.
1983          */
1984         nand->ecc.mode = NAND_ECC_HW;
1985         nand_set_flash_node(nand, np);
1986
1987         mtd = nand_to_mtd(nand);
1988         mtd->dev.parent = dev;
1989
1990         ret = nand_scan(nand, nsels);
1991         if (ret)
1992                 return ret;
1993
1994         ret = mtd_device_register(mtd, NULL, 0);
1995         if (ret) {
1996                 dev_err(dev, "failed to register mtd device: %d\n", ret);
1997                 nand_release(nand);
1998                 return ret;
1999         }
2000
2001         list_add_tail(&sunxi_nand->node, &nfc->chips);
2002
2003         return 0;
2004 }
2005
2006 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2007 {
2008         struct device_node *np = dev->of_node;
2009         struct device_node *nand_np;
2010         int nchips = of_get_child_count(np);
2011         int ret;
2012
2013         if (nchips > 8) {
2014                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2015                 return -EINVAL;
2016         }
2017
2018         for_each_child_of_node(np, nand_np) {
2019                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2020                 if (ret) {
2021                         of_node_put(nand_np);
2022                         return ret;
2023                 }
2024         }
2025
2026         return 0;
2027 }
2028
2029 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2030 {
2031         struct sunxi_nand_chip *sunxi_nand;
2032
2033         while (!list_empty(&nfc->chips)) {
2034                 sunxi_nand = list_first_entry(&nfc->chips,
2035                                               struct sunxi_nand_chip,
2036                                               node);
2037                 nand_release(&sunxi_nand->nand);
2038                 sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc);
2039                 list_del(&sunxi_nand->node);
2040         }
2041 }
2042
2043 static int sunxi_nfc_probe(struct platform_device *pdev)
2044 {
2045         struct device *dev = &pdev->dev;
2046         struct resource *r;
2047         struct sunxi_nfc *nfc;
2048         int irq;
2049         int ret;
2050
2051         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2052         if (!nfc)
2053                 return -ENOMEM;
2054
2055         nfc->dev = dev;
2056         nand_controller_init(&nfc->controller);
2057         INIT_LIST_HEAD(&nfc->chips);
2058
2059         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2060         nfc->regs = devm_ioremap_resource(dev, r);
2061         if (IS_ERR(nfc->regs))
2062                 return PTR_ERR(nfc->regs);
2063
2064         irq = platform_get_irq(pdev, 0);
2065         if (irq < 0) {
2066                 dev_err(dev, "failed to retrieve irq\n");
2067                 return irq;
2068         }
2069
2070         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2071         if (IS_ERR(nfc->ahb_clk)) {
2072                 dev_err(dev, "failed to retrieve ahb clk\n");
2073                 return PTR_ERR(nfc->ahb_clk);
2074         }
2075
2076         ret = clk_prepare_enable(nfc->ahb_clk);
2077         if (ret)
2078                 return ret;
2079
2080         nfc->mod_clk = devm_clk_get(dev, "mod");
2081         if (IS_ERR(nfc->mod_clk)) {
2082                 dev_err(dev, "failed to retrieve mod clk\n");
2083                 ret = PTR_ERR(nfc->mod_clk);
2084                 goto out_ahb_clk_unprepare;
2085         }
2086
2087         ret = clk_prepare_enable(nfc->mod_clk);
2088         if (ret)
2089                 goto out_ahb_clk_unprepare;
2090
2091         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2092         if (IS_ERR(nfc->reset)) {
2093                 ret = PTR_ERR(nfc->reset);
2094                 goto out_mod_clk_unprepare;
2095         }
2096
2097         ret = reset_control_deassert(nfc->reset);
2098         if (ret) {
2099                 dev_err(dev, "reset err %d\n", ret);
2100                 goto out_mod_clk_unprepare;
2101         }
2102
2103         nfc->caps = of_device_get_match_data(&pdev->dev);
2104         if (!nfc->caps) {
2105                 ret = -EINVAL;
2106                 goto out_ahb_reset_reassert;
2107         }
2108
2109         ret = sunxi_nfc_rst(nfc);
2110         if (ret)
2111                 goto out_ahb_reset_reassert;
2112
2113         writel(0, nfc->regs + NFC_REG_INT);
2114         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2115                                0, "sunxi-nand", nfc);
2116         if (ret)
2117                 goto out_ahb_reset_reassert;
2118
2119         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2120         if (nfc->dmac) {
2121                 struct dma_slave_config dmac_cfg = { };
2122
2123                 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2124                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2125                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2126                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2127                 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2128                 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2129                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2130         } else {
2131                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2132         }
2133
2134         platform_set_drvdata(pdev, nfc);
2135
2136         ret = sunxi_nand_chips_init(dev, nfc);
2137         if (ret) {
2138                 dev_err(dev, "failed to init nand chips\n");
2139                 goto out_release_dmac;
2140         }
2141
2142         return 0;
2143
2144 out_release_dmac:
2145         if (nfc->dmac)
2146                 dma_release_channel(nfc->dmac);
2147 out_ahb_reset_reassert:
2148         reset_control_assert(nfc->reset);
2149 out_mod_clk_unprepare:
2150         clk_disable_unprepare(nfc->mod_clk);
2151 out_ahb_clk_unprepare:
2152         clk_disable_unprepare(nfc->ahb_clk);
2153
2154         return ret;
2155 }
2156
2157 static int sunxi_nfc_remove(struct platform_device *pdev)
2158 {
2159         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2160
2161         sunxi_nand_chips_cleanup(nfc);
2162
2163         reset_control_assert(nfc->reset);
2164
2165         if (nfc->dmac)
2166                 dma_release_channel(nfc->dmac);
2167         clk_disable_unprepare(nfc->mod_clk);
2168         clk_disable_unprepare(nfc->ahb_clk);
2169
2170         return 0;
2171 }
2172
2173 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2174         .reg_io_data = NFC_REG_A10_IO_DATA,
2175         .dma_maxburst = 4,
2176 };
2177
2178 static const struct of_device_id sunxi_nfc_ids[] = {
2179         {
2180                 .compatible = "allwinner,sun4i-a10-nand",
2181                 .data = &sunxi_nfc_a10_caps,
2182         },
2183         { /* sentinel */ }
2184 };
2185 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2186
2187 static struct platform_driver sunxi_nfc_driver = {
2188         .driver = {
2189                 .name = "sunxi_nand",
2190                 .of_match_table = sunxi_nfc_ids,
2191         },
2192         .probe = sunxi_nfc_probe,
2193         .remove = sunxi_nfc_remove,
2194 };
2195 module_platform_driver(sunxi_nfc_driver);
2196
2197 MODULE_LICENSE("GPL");
2198 MODULE_AUTHOR("Boris BREZILLON");
2199 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2200 MODULE_ALIAS("platform:sunxi_nand");