drivers/mtd/nand/raw/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/mtd/mtd.h>
  33 #include <linux/mtd/rawnand.h>
  34 #include <linux/mtd/partitions.h>
  35 #include <linux/clk.h>
  36 #include <linux/delay.h>
  37 #include <linux/dmaengine.h>
  38 #include <linux/interrupt.h>
  39 #include <linux/iopoll.h>
  40 #include <linux/reset.h>
  41
  42 #define NFC_REG_CTL             0x0000
  43 #define NFC_REG_ST              0x0004
  44 #define NFC_REG_INT             0x0008
  45 #define NFC_REG_TIMING_CTL      0x000C
  46 #define NFC_REG_TIMING_CFG      0x0010
  47 #define NFC_REG_ADDR_LOW        0x0014
  48 #define NFC_REG_ADDR_HIGH       0x0018
  49 #define NFC_REG_SECTOR_NUM      0x001C
  50 #define NFC_REG_CNT             0x0020
  51 #define NFC_REG_CMD             0x0024
  52 #define NFC_REG_RCMD_SET        0x0028
  53 #define NFC_REG_WCMD_SET        0x002C
  54 #define NFC_REG_IO_DATA         0x0030
  55 #define NFC_REG_ECC_CTL         0x0034
  56 #define NFC_REG_ECC_ST          0x0038
  57 #define NFC_REG_DEBUG           0x003C
  58 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  59 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  60 #define NFC_REG_SPARE_AREA      0x00A0
  61 #define NFC_REG_PAT_ID          0x00A4
  62 #define NFC_RAM0_BASE           0x0400
  63 #define NFC_RAM1_BASE           0x0800
  64
  65 /* define bit use in NFC_CTL */
  66 #define NFC_EN                  BIT(0)
  67 #define NFC_RESET               BIT(1)
  68 #define NFC_BUS_WIDTH_MSK       BIT(2)
  69 #define NFC_BUS_WIDTH_8         (0 << 2)
  70 #define NFC_BUS_WIDTH_16        (1 << 2)
  71 #define NFC_RB_SEL_MSK          BIT(3)
  72 #define NFC_RB_SEL(x)           ((x) << 3)
  73 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  74 #define NFC_CE_SEL(x)           ((x) << 24)
  75 #define NFC_CE_CTL              BIT(6)
  76 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  77 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  78 #define NFC_SAM                 BIT(12)
  79 #define NFC_RAM_METHOD          BIT(14)
  80 #define NFC_DEBUG_CTL           BIT(31)
  81
  82 /* define bit use in NFC_ST */
  83 #define NFC_RB_B2R              BIT(0)
  84 #define NFC_CMD_INT_FLAG        BIT(1)
  85 #define NFC_DMA_INT_FLAG        BIT(2)
  86 #define NFC_CMD_FIFO_STATUS     BIT(3)
  87 #define NFC_STA                 BIT(4)
  88 #define NFC_NATCH_INT_FLAG      BIT(5)
  89 #define NFC_RB_STATE(x)         BIT(x + 8)
  90
  91 /* define bit use in NFC_INT */
  92 #define NFC_B2R_INT_ENABLE      BIT(0)
  93 #define NFC_CMD_INT_ENABLE      BIT(1)
  94 #define NFC_DMA_INT_ENABLE      BIT(2)
  95 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  96                                  NFC_CMD_INT_ENABLE | \
  97                                  NFC_DMA_INT_ENABLE)
  98
  99 /* define bit use in NFC_TIMING_CTL */
 100 #define NFC_TIMING_CTL_EDO      BIT(8)
 101
 102 /* define NFC_TIMING_CFG register layout */
 103 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 104         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 105         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 106         (((tCAD) & 0x7) << 8))
 107
 108 /* define bit use in NFC_CMD */
 109 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 110 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 111 #define NFC_CMD(x)              (x)
 112 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 113 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 114 #define NFC_SEND_ADR            BIT(19)
 115 #define NFC_ACCESS_DIR          BIT(20)
 116 #define NFC_DATA_TRANS          BIT(21)
 117 #define NFC_SEND_CMD1           BIT(22)
 118 #define NFC_WAIT_FLAG           BIT(23)
 119 #define NFC_SEND_CMD2           BIT(24)
 120 #define NFC_SEQ                 BIT(25)
 121 #define NFC_DATA_SWAP_METHOD    BIT(26)
 122 #define NFC_ROW_AUTO_INC        BIT(27)
 123 #define NFC_SEND_CMD3           BIT(28)
 124 #define NFC_SEND_CMD4           BIT(29)
 125 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 126 #define NFC_NORMAL_OP           (0 << 30)
 127 #define NFC_ECC_OP              (1 << 30)
 128 #define NFC_PAGE_OP             (2U << 30)
 129
 130 /* define bit use in NFC_RCMD_SET */
 131 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 132 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 133 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 134
 135 /* define bit use in NFC_WCMD_SET */
 136 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 137 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 138 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 139 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 140
 141 /* define bit use in NFC_ECC_CTL */
 142 #define NFC_ECC_EN              BIT(0)
 143 #define NFC_ECC_PIPELINE        BIT(3)
 144 #define NFC_ECC_EXCEPTION       BIT(4)
 145 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 146 #define NFC_ECC_BLOCK_512       BIT(5)
 147 #define NFC_RANDOM_EN           BIT(9)
 148 #define NFC_RANDOM_DIRECTION    BIT(10)
 149 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 150 #define NFC_ECC_MODE(x)         ((x) << 12)
 151 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 152 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 153
 154 /* define bit use in NFC_ECC_ST */
 155 #define NFC_ECC_ERR(x)          BIT(x)
 156 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 157 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 158 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 159
 160 #define NFC_DEFAULT_TIMEOUT_MS  1000
 161
 162 #define NFC_SRAM_SIZE           1024
 163
 164 #define NFC_MAX_CS              7
 165
 166 /*
 167  * Chip Select structure: stores information related to NAND Chip Select
 168  *
 169  * @cs:         the NAND CS id used to communicate with a NAND Chip
 170  * @rb:         the Ready/Busy pin ID. -1 means no R/B pin connected to the
 171  *              NFC
 172  */
 173 struct sunxi_nand_chip_sel {
 174         u8 cs;
 175         s8 rb;
 176 };
 177
 178 /*
 179  * sunxi HW ECC infos: stores information related to HW ECC support
 180  *
 181  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 182  */
 183 struct sunxi_nand_hw_ecc {
 184         int mode;
 185 };
 186
 187 /*
 188  * NAND chip structure: stores NAND chip device related information
 189  *
 190  * @node:               used to store NAND chips into a list
 191  * @nand:               base NAND chip structure
 192  * @mtd:                base MTD structure
 193  * @clk_rate:           clk_rate required for this NAND chip
 194  * @timing_cfg          TIMING_CFG register value for this NAND chip
 195  * @selected:           current active CS
 196  * @nsels:              number of CS lines required by the NAND chip
 197  * @sels:               array of CS lines descriptions
 198  */
 199 struct sunxi_nand_chip {
 200         struct list_head node;
 201         struct nand_chip nand;
 202         unsigned long clk_rate;
 203         u32 timing_cfg;
 204         u32 timing_ctl;
 205         int selected;
 206         int addr_cycles;
 207         u32 addr[2];
 208         int cmd_cycles;
 209         u8 cmd[2];
 210         int nsels;
 211         struct sunxi_nand_chip_sel sels[0];
 212 };
 213
 214 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 215 {
 216         return container_of(nand, struct sunxi_nand_chip, nand);
 217 }
 218
 219 /*
 220  * NAND Controller structure: stores sunxi NAND controller information
 221  *
 222  * @controller:         base controller structure
 223  * @dev:                parent device (used to print error messages)
 224  * @regs:               NAND controller registers
 225  * @ahb_clk:            NAND Controller AHB clock
 226  * @mod_clk:            NAND Controller mod clock
 227  * @assigned_cs:        bitmask describing already assigned CS lines
 228  * @clk_rate:           NAND controller current clock rate
 229  * @chips:              a list containing all the NAND chips attached to
 230  *                      this NAND controller
 231  * @complete:           a completion object used to wait for NAND
 232  *                      controller events
 233  */
 234 struct sunxi_nfc {
 235         struct nand_controller controller;
 236         struct device *dev;
 237         void __iomem *regs;
 238         struct clk *ahb_clk;
 239         struct clk *mod_clk;
 240         struct reset_control *reset;
 241         unsigned long assigned_cs;
 242         unsigned long clk_rate;
 243         struct list_head chips;
 244         struct completion complete;
 245         struct dma_chan *dmac;
 246 };
 247
 248 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 249 {
 250         return container_of(ctrl, struct sunxi_nfc, controller);
 251 }
 252
 253 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 254 {
 255         struct sunxi_nfc *nfc = dev_id;
 256         u32 st = readl(nfc->regs + NFC_REG_ST);
 257         u32 ien = readl(nfc->regs + NFC_REG_INT);
 258
 259         if (!(ien & st))
 260                 return IRQ_NONE;
 261
 262         if ((ien & st) == ien)
 263                 complete(&nfc->complete);
 264
 265         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 266         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 267
 268         return IRQ_HANDLED;
 269 }
 270
 271 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 272                                  bool use_polling, unsigned int timeout_ms)
 273 {
 274         int ret;
 275
 276         if (events & ~NFC_INT_MASK)
 277                 return -EINVAL;
 278
 279         if (!timeout_ms)
 280                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 281
 282         if (!use_polling) {
 283                 init_completion(&nfc->complete);
 284
 285                 writel(events, nfc->regs + NFC_REG_INT);
 286
 287                 ret = wait_for_completion_timeout(&nfc->complete,
 288                                                 msecs_to_jiffies(timeout_ms));
 289                 if (!ret)
 290                         ret = -ETIMEDOUT;
 291                 else
 292                         ret = 0;
 293
 294                 writel(0, nfc->regs + NFC_REG_INT);
 295         } else {
 296                 u32 status;
 297
 298                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 299                                          (status & events) == events, 1,
 300                                          timeout_ms * 1000);
 301         }
 302
 303         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 304
 305         if (ret)
 306                 dev_err(nfc->dev, "wait interrupt timedout\n");
 307
 308         return ret;
 309 }
 310
 311 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 312 {
 313         u32 status;
 314         int ret;
 315
 316         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 317                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 318                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 319         if (ret)
 320                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 321
 322         return ret;
 323 }
 324
 325 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 326 {
 327         u32 ctl;
 328         int ret;
 329
 330         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 331         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 332
 333         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 334                                  !(ctl & NFC_RESET), 1,
 335                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 336         if (ret)
 337                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 338
 339         return ret;
 340 }
 341
 342 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 343                                     int chunksize, int nchunks,
 344                                     enum dma_data_direction ddir,
 345                                     struct scatterlist *sg)
 346 {
 347         struct nand_chip *nand = mtd_to_nand(mtd);
 348         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 349         struct dma_async_tx_descriptor *dmad;
 350         enum dma_transfer_direction tdir;
 351         dma_cookie_t dmat;
 352         int ret;
 353
 354         if (ddir == DMA_FROM_DEVICE)
 355                 tdir = DMA_DEV_TO_MEM;
 356         else
 357                 tdir = DMA_MEM_TO_DEV;
 358
 359         sg_init_one(sg, buf, nchunks * chunksize);
 360         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 361         if (!ret)
 362                 return -ENOMEM;
 363
 364         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 365         if (!dmad) {
 366                 ret = -EINVAL;
 367                 goto err_unmap_buf;
 368         }
 369
 370         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 371                nfc->regs + NFC_REG_CTL);
 372         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 373         writel(chunksize, nfc->regs + NFC_REG_CNT);
 374         dmat = dmaengine_submit(dmad);
 375
 376         ret = dma_submit_error(dmat);
 377         if (ret)
 378                 goto err_clr_dma_flag;
 379
 380         return 0;
 381
 382 err_clr_dma_flag:
 383         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 384                nfc->regs + NFC_REG_CTL);
 385
 386 err_unmap_buf:
 387         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 388         return ret;
 389 }
 390
 391 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 392                                      enum dma_data_direction ddir,
 393                                      struct scatterlist *sg)
 394 {
 395         struct nand_chip *nand = mtd_to_nand(mtd);
 396         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 397
 398         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 399         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 400                nfc->regs + NFC_REG_CTL);
 401 }
 402
 403 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 404 {
 405         struct nand_chip *nand = mtd_to_nand(mtd);
 406         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 407         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 408         u32 mask;
 409
 410         if (sunxi_nand->selected < 0)
 411                 return 0;
 412
 413         if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) {
 414                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 415                 return 0;
 416         }
 417
 418         mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb);
 419
 420         return !!(readl(nfc->regs + NFC_REG_ST) & mask);
 421 }
 422
 423 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 424 {
 425         struct nand_chip *nand = mtd_to_nand(mtd);
 426         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 427         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 428         struct sunxi_nand_chip_sel *sel;
 429         u32 ctl;
 430
 431         if (chip > 0 && chip >= sunxi_nand->nsels)
 432                 return;
 433
 434         if (chip == sunxi_nand->selected)
 435                 return;
 436
 437         ctl = readl(nfc->regs + NFC_REG_CTL) &
 438               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 439
 440         if (chip >= 0) {
 441                 sel = &sunxi_nand->sels[chip];
 442
 443                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 444                        NFC_PAGE_SHIFT(nand->page_shift);
 445                 if (sel->rb < 0) {
 446                         nand->dev_ready = NULL;
 447                 } else {
 448                         nand->dev_ready = sunxi_nfc_dev_ready;
 449                         ctl |= NFC_RB_SEL(sel->rb);
 450                 }
 451
 452                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 453
 454                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 455                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 456                         nfc->clk_rate = sunxi_nand->clk_rate;
 457                 }
 458         }
 459
 460         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 461         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 462         writel(ctl, nfc->regs + NFC_REG_CTL);
 463
 464         sunxi_nand->selected = chip;
 465 }
 466
 467 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 468 {
 469         struct nand_chip *nand = mtd_to_nand(mtd);
 470         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 471         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 472         int ret;
 473         int cnt;
 474         int offs = 0;
 475         u32 tmp;
 476
 477         while (len > offs) {
 478                 bool poll = false;
 479
 480                 cnt = min(len - offs, NFC_SRAM_SIZE);
 481
 482                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 483                 if (ret)
 484                         break;
 485
 486                 writel(cnt, nfc->regs + NFC_REG_CNT);
 487                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 488                 writel(tmp, nfc->regs + NFC_REG_CMD);
 489
 490                 /* Arbitrary limit for polling mode */
 491                 if (cnt < 64)
 492                         poll = true;
 493
 494                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 495                 if (ret)
 496                         break;
 497
 498                 if (buf)
 499                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 500                                       cnt);
 501                 offs += cnt;
 502         }
 503 }
 504
 505 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 506                                 int len)
 507 {
 508         struct nand_chip *nand = mtd_to_nand(mtd);
 509         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 510         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 511         int ret;
 512         int cnt;
 513         int offs = 0;
 514         u32 tmp;
 515
 516         while (len > offs) {
 517                 bool poll = false;
 518
 519                 cnt = min(len - offs, NFC_SRAM_SIZE);
 520
 521                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 522                 if (ret)
 523                         break;
 524
 525                 writel(cnt, nfc->regs + NFC_REG_CNT);
 526                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 527                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 528                       NFC_ACCESS_DIR;
 529                 writel(tmp, nfc->regs + NFC_REG_CMD);
 530
 531                 /* Arbitrary limit for polling mode */
 532                 if (cnt < 64)
 533                         poll = true;
 534
 535                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 536                 if (ret)
 537                         break;
 538
 539                 offs += cnt;
 540         }
 541 }
 542
 543 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 544 {
 545         uint8_t ret = 0;
 546
 547         sunxi_nfc_read_buf(mtd, &ret, 1);
 548
 549         return ret;
 550 }
 551
 552 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 553                                unsigned int ctrl)
 554 {
 555         struct nand_chip *nand = mtd_to_nand(mtd);
 556         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 557         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 558         int ret;
 559
 560         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 561             !(ctrl & (NAND_CLE | NAND_ALE))) {
 562                 u32 cmd = 0;
 563
 564                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 565                         return;
 566
 567                 if (sunxi_nand->cmd_cycles--)
 568                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 569
 570                 if (sunxi_nand->cmd_cycles--) {
 571                         cmd |= NFC_SEND_CMD2;
 572                         writel(sunxi_nand->cmd[1],
 573                                nfc->regs + NFC_REG_RCMD_SET);
 574                 }
 575
 576                 sunxi_nand->cmd_cycles = 0;
 577
 578                 if (sunxi_nand->addr_cycles) {
 579                         cmd |= NFC_SEND_ADR |
 580                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 581                         writel(sunxi_nand->addr[0],
 582                                nfc->regs + NFC_REG_ADDR_LOW);
 583                 }
 584
 585                 if (sunxi_nand->addr_cycles > 4)
 586                         writel(sunxi_nand->addr[1],
 587                                nfc->regs + NFC_REG_ADDR_HIGH);
 588
 589                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 590                 if (ret)
 591                         return;
 592
 593                 writel(cmd, nfc->regs + NFC_REG_CMD);
 594                 sunxi_nand->addr[0] = 0;
 595                 sunxi_nand->addr[1] = 0;
 596                 sunxi_nand->addr_cycles = 0;
 597                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 598         }
 599
 600         if (ctrl & NAND_CLE) {
 601                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 602         } else if (ctrl & NAND_ALE) {
 603                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 604                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 605                 sunxi_nand->addr_cycles++;
 606         }
 607 }
 608
 609 /* These seed values have been extracted from Allwinner's BSP */
 610 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 611         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 612         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 613         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 614         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 615         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 616         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 617         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 618         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 619         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 620         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 621         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 622         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 623         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 624         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 625         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 626         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 627 };
 628
 629 /*
 630  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 631  * have been generated using
 632  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 633  * the randomizer engine does internally before de/scrambling OOB data.
 634  *
 635  * Those tables are statically defined to avoid calculating randomizer state
 636  * at runtime.
 637  */
 638 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 639         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 640         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 641         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 642         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 643         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 644         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 645         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 646         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 647         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 648         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 649         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 650         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 651         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 652         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 653         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 654         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 655 };
 656
 657 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 658         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 659         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 660         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 661         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 662         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 663         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 664         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 665         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 666         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 667         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 668         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 669         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 670         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 671         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 672         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 673         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 674 };
 675
 676 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 677 {
 678         state &= 0x7fff;
 679
 680         /*
 681          * This loop is just a simple implementation of a Fibonacci LFSR using
 682          * the x16 + x15 + 1 polynomial.
 683          */
 684         while (count--)
 685                 state = ((state >> 1) |
 686                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 687
 688         return state;
 689 }
 690
 691 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 692 {
 693         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 694         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 695
 696         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 697                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 698
 699         if (ecc) {
 700                 if (mtd->ecc_step_size == 512)
 701                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 702                 else
 703                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 704         }
 705
 706         return seeds[page % mod];
 707 }
 708
 709 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 710                                         int page, bool ecc)
 711 {
 712         struct nand_chip *nand = mtd_to_nand(mtd);
 713         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 714         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 715         u16 state;
 716
 717         if (!(nand->options & NAND_NEED_SCRAMBLING))
 718                 return;
 719
 720         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 721         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 722         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 723         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 724 }
 725
 726 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 727 {
 728         struct nand_chip *nand = mtd_to_nand(mtd);
 729         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 730
 731         if (!(nand->options & NAND_NEED_SCRAMBLING))
 732                 return;
 733
 734         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 735                nfc->regs + NFC_REG_ECC_CTL);
 736 }
 737
 738 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 739 {
 740         struct nand_chip *nand = mtd_to_nand(mtd);
 741         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 742
 743         if (!(nand->options & NAND_NEED_SCRAMBLING))
 744                 return;
 745
 746         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 747                nfc->regs + NFC_REG_ECC_CTL);
 748 }
 749
 750 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 751 {
 752         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 753
 754         bbm[0] ^= state;
 755         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 756 }
 757
 758 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 759                                            const uint8_t *buf, int len,
 760                                            bool ecc, int page)
 761 {
 762         sunxi_nfc_randomizer_config(mtd, page, ecc);
 763         sunxi_nfc_randomizer_enable(mtd);
 764         sunxi_nfc_write_buf(mtd, buf, len);
 765         sunxi_nfc_randomizer_disable(mtd);
 766 }
 767
 768 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 769                                           int len, bool ecc, int page)
 770 {
 771         sunxi_nfc_randomizer_config(mtd, page, ecc);
 772         sunxi_nfc_randomizer_enable(mtd);
 773         sunxi_nfc_read_buf(mtd, buf, len);
 774         sunxi_nfc_randomizer_disable(mtd);
 775 }
 776
 777 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 778 {
 779         struct nand_chip *nand = mtd_to_nand(mtd);
 780         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 781         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 782         u32 ecc_ctl;
 783
 784         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 785         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 786                      NFC_ECC_BLOCK_SIZE_MSK);
 787         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 788                    NFC_ECC_PIPELINE;
 789
 790         if (nand->ecc.size == 512)
 791                 ecc_ctl |= NFC_ECC_BLOCK_512;
 792
 793         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 794 }
 795
 796 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 797 {
 798         struct nand_chip *nand = mtd_to_nand(mtd);
 799         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 800
 801         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 802                nfc->regs + NFC_REG_ECC_CTL);
 803 }
 804
 805 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 806 {
 807         buf[0] = user_data;
 808         buf[1] = user_data >> 8;
 809         buf[2] = user_data >> 16;
 810         buf[3] = user_data >> 24;
 811 }
 812
 813 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 814 {
 815         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 816 }
 817
 818 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 819                                                 int step, bool bbm, int page)
 820 {
 821         struct nand_chip *nand = mtd_to_nand(mtd);
 822         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 823
 824         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 825                                    oob);
 826
 827         /* De-randomize the Bad Block Marker. */
 828         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 829                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 830 }
 831
 832 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 833                                                 const u8 *oob, int step,
 834                                                 bool bbm, int page)
 835 {
 836         struct nand_chip *nand = mtd_to_nand(mtd);
 837         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 838         u8 user_data[4];
 839
 840         /* Randomize the Bad Block Marker. */
 841         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 842                 memcpy(user_data, oob, sizeof(user_data));
 843                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 844                 oob = user_data;
 845         }
 846
 847         writel(sunxi_nfc_buf_to_user_data(oob),
 848                nfc->regs + NFC_REG_USER_DATA(step));
 849 }
 850
 851 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 852                                           unsigned int *max_bitflips, int ret)
 853 {
 854         if (ret < 0) {
 855                 mtd->ecc_stats.failed++;
 856         } else {
 857                 mtd->ecc_stats.corrected += ret;
 858                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 859         }
 860 }
 861
 862 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 863                                     int step, u32 status, bool *erased)
 864 {
 865         struct nand_chip *nand = mtd_to_nand(mtd);
 866         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 867         struct nand_ecc_ctrl *ecc = &nand->ecc;
 868         u32 tmp;
 869
 870         *erased = false;
 871
 872         if (status & NFC_ECC_ERR(step))
 873                 return -EBADMSG;
 874
 875         if (status & NFC_ECC_PAT_FOUND(step)) {
 876                 u8 pattern;
 877
 878                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 879                         pattern = 0x0;
 880                 } else {
 881                         pattern = 0xff;
 882                         *erased = true;
 883                 }
 884
 885                 if (data)
 886                         memset(data, pattern, ecc->size);
 887
 888                 if (oob)
 889                         memset(oob, pattern, ecc->bytes + 4);
 890
 891                 return 0;
 892         }
 893
 894         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 895
 896         return NFC_ECC_ERR_CNT(step, tmp);
 897 }
 898
 899 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 900                                        u8 *data, int data_off,
 901                                        u8 *oob, int oob_off,
 902                                        int *cur_off,
 903                                        unsigned int *max_bitflips,
 904                                        bool bbm, bool oob_required, int page)
 905 {
 906         struct nand_chip *nand = mtd_to_nand(mtd);
 907         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 908         struct nand_ecc_ctrl *ecc = &nand->ecc;
 909         int raw_mode = 0;
 910         bool erased;
 911         int ret;
 912
 913         if (*cur_off != data_off)
 914                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 915
 916         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 917
 918         if (data_off + ecc->size != oob_off)
 919                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 920
 921         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 922         if (ret)
 923                 return ret;
 924
 925         sunxi_nfc_randomizer_enable(mtd);
 926         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 927                nfc->regs + NFC_REG_CMD);
 928
 929         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 930         sunxi_nfc_randomizer_disable(mtd);
 931         if (ret)
 932                 return ret;
 933
 934         *cur_off = oob_off + ecc->bytes + 4;
 935
 936         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 937                                        readl(nfc->regs + NFC_REG_ECC_ST),
 938                                        &erased);
 939         if (erased)
 940                 return 1;
 941
 942         if (ret < 0) {
 943                 /*
 944                  * Re-read the data with the randomizer disabled to identify
 945                  * bitflips in erased pages.
 946                  */
 947                 if (nand->options & NAND_NEED_SCRAMBLING)
 948                         nand_change_read_column_op(nand, data_off, data,
 949                                                    ecc->size, false);
 950                 else
 951                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 952                                       ecc->size);
 953
 954                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 955                                            false);
 956
 957                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 958                                                   oob, ecc->bytes + 4,
 959                                                   NULL, 0, ecc->strength);
 960                 if (ret >= 0)
 961                         raw_mode = 1;
 962         } else {
 963                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 964
 965                 if (oob_required) {
 966                         nand_change_read_column_op(nand, oob_off, NULL, 0,
 967                                                    false);
 968                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
 969                                                       true, page);
 970
 971                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
 972                                                             bbm, page);
 973                 }
 974         }
 975
 976         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
 977
 978         return raw_mode;
 979 }
 980
 981 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
 982                                             u8 *oob, int *cur_off,
 983                                             bool randomize, int page)
 984 {
 985         struct nand_chip *nand = mtd_to_nand(mtd);
 986         struct nand_ecc_ctrl *ecc = &nand->ecc;
 987         int offset = ((ecc->bytes + 4) * ecc->steps);
 988         int len = mtd->oobsize - offset;
 989
 990         if (len <= 0)
 991                 return;
 992
 993         if (!cur_off || *cur_off != offset)
 994                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 995                                            false);
 996
 997         if (!randomize)
 998                 sunxi_nfc_read_buf(mtd, oob + offset, len);
 999         else
1000                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1001                                               false, page);
1002
1003         if (cur_off)
1004                 *cur_off = mtd->oobsize + mtd->writesize;
1005 }
1006
1007 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1008                                             int oob_required, int page,
1009                                             int nchunks)
1010 {
1011         struct nand_chip *nand = mtd_to_nand(mtd);
1012         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1013         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1014         struct nand_ecc_ctrl *ecc = &nand->ecc;
1015         unsigned int max_bitflips = 0;
1016         int ret, i, raw_mode = 0;
1017         struct scatterlist sg;
1018         u32 status;
1019
1020         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1021         if (ret)
1022                 return ret;
1023
1024         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1025                                        DMA_FROM_DEVICE, &sg);
1026         if (ret)
1027                 return ret;
1028
1029         sunxi_nfc_hw_ecc_enable(mtd);
1030         sunxi_nfc_randomizer_config(mtd, page, false);
1031         sunxi_nfc_randomizer_enable(mtd);
1032
1033         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1034                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1035
1036         dma_async_issue_pending(nfc->dmac);
1037
1038         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1039                nfc->regs + NFC_REG_CMD);
1040
1041         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1042         if (ret)
1043                 dmaengine_terminate_all(nfc->dmac);
1044
1045         sunxi_nfc_randomizer_disable(mtd);
1046         sunxi_nfc_hw_ecc_disable(mtd);
1047
1048         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1049
1050         if (ret)
1051                 return ret;
1052
1053         status = readl(nfc->regs + NFC_REG_ECC_ST);
1054
1055         for (i = 0; i < nchunks; i++) {
1056                 int data_off = i * ecc->size;
1057                 int oob_off = i * (ecc->bytes + 4);
1058                 u8 *data = buf + data_off;
1059                 u8 *oob = nand->oob_poi + oob_off;
1060                 bool erased;
1061
1062                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1063                                                oob_required ? oob : NULL,
1064                                                i, status, &erased);
1065
1066                 /* ECC errors are handled in the second loop. */
1067                 if (ret < 0)
1068                         continue;
1069
1070                 if (oob_required && !erased) {
1071                         /* TODO: use DMA to retrieve OOB */
1072                         nand_change_read_column_op(nand,
1073                                                    mtd->writesize + oob_off,
1074                                                    oob, ecc->bytes + 4, false);
1075
1076                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1077                                                             !i, page);
1078                 }
1079
1080                 if (erased)
1081                         raw_mode = 1;
1082
1083                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1084         }
1085
1086         if (status & NFC_ECC_ERR_MSK) {
1087                 for (i = 0; i < nchunks; i++) {
1088                         int data_off = i * ecc->size;
1089                         int oob_off = i * (ecc->bytes + 4);
1090                         u8 *data = buf + data_off;
1091                         u8 *oob = nand->oob_poi + oob_off;
1092
1093                         if (!(status & NFC_ECC_ERR(i)))
1094                                 continue;
1095
1096                         /*
1097                          * Re-read the data with the randomizer disabled to
1098                          * identify bitflips in erased pages.
1099                          * TODO: use DMA to read page in raw mode
1100                          */
1101                         if (randomized)
1102                                 nand_change_read_column_op(nand, data_off,
1103                                                            data, ecc->size,
1104                                                            false);
1105
1106                         /* TODO: use DMA to retrieve OOB */
1107                         nand_change_read_column_op(nand,
1108                                                    mtd->writesize + oob_off,
1109                                                    oob, ecc->bytes + 4, false);
1110
1111                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1112                                                           oob, ecc->bytes + 4,
1113                                                           NULL, 0,
1114                                                           ecc->strength);
1115                         if (ret >= 0)
1116                                 raw_mode = 1;
1117
1118                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1119                 }
1120         }
1121
1122         if (oob_required)
1123                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1124                                                 NULL, !raw_mode,
1125                                                 page);
1126
1127         return max_bitflips;
1128 }
1129
1130 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1131                                         const u8 *data, int data_off,
1132                                         const u8 *oob, int oob_off,
1133                                         int *cur_off, bool bbm,
1134                                         int page)
1135 {
1136         struct nand_chip *nand = mtd_to_nand(mtd);
1137         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1138         struct nand_ecc_ctrl *ecc = &nand->ecc;
1139         int ret;
1140
1141         if (data_off != *cur_off)
1142                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1143
1144         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1145
1146         if (data_off + ecc->size != oob_off)
1147                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1148
1149         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1150         if (ret)
1151                 return ret;
1152
1153         sunxi_nfc_randomizer_enable(mtd);
1154         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1155
1156         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1157                NFC_ACCESS_DIR | NFC_ECC_OP,
1158                nfc->regs + NFC_REG_CMD);
1159
1160         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1161         sunxi_nfc_randomizer_disable(mtd);
1162         if (ret)
1163                 return ret;
1164
1165         *cur_off = oob_off + ecc->bytes + 4;
1166
1167         return 0;
1168 }
1169
1170 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1171                                              u8 *oob, int *cur_off,
1172                                              int page)
1173 {
1174         struct nand_chip *nand = mtd_to_nand(mtd);
1175         struct nand_ecc_ctrl *ecc = &nand->ecc;
1176         int offset = ((ecc->bytes + 4) * ecc->steps);
1177         int len = mtd->oobsize - offset;
1178
1179         if (len <= 0)
1180                 return;
1181
1182         if (!cur_off || *cur_off != offset)
1183                 nand_change_write_column_op(nand, offset + mtd->writesize,
1184                                             NULL, 0, false);
1185
1186         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1187
1188         if (cur_off)
1189                 *cur_off = mtd->oobsize + mtd->writesize;
1190 }
1191
1192 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1193                                       struct nand_chip *chip, uint8_t *buf,
1194                                       int oob_required, int page)
1195 {
1196         struct nand_ecc_ctrl *ecc = &chip->ecc;
1197         unsigned int max_bitflips = 0;
1198         int ret, i, cur_off = 0;
1199         bool raw_mode = false;
1200
1201         nand_read_page_op(chip, page, 0, NULL, 0);
1202
1203         sunxi_nfc_hw_ecc_enable(mtd);
1204
1205         for (i = 0; i < ecc->steps; i++) {
1206                 int data_off = i * ecc->size;
1207                 int oob_off = i * (ecc->bytes + 4);
1208                 u8 *data = buf + data_off;
1209                 u8 *oob = chip->oob_poi + oob_off;
1210
1211                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1212                                                   oob_off + mtd->writesize,
1213                                                   &cur_off, &max_bitflips,
1214                                                   !i, oob_required, page);
1215                 if (ret < 0)
1216                         return ret;
1217                 else if (ret)
1218                         raw_mode = true;
1219         }
1220
1221         if (oob_required)
1222                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1223                                                 !raw_mode, page);
1224
1225         sunxi_nfc_hw_ecc_disable(mtd);
1226
1227         return max_bitflips;
1228 }
1229
1230 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1231                                           struct nand_chip *chip, u8 *buf,
1232                                           int oob_required, int page)
1233 {
1234         int ret;
1235
1236         nand_read_page_op(chip, page, 0, NULL, 0);
1237
1238         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1239                                                chip->ecc.steps);
1240         if (ret >= 0)
1241                 return ret;
1242
1243         /* Fallback to PIO mode */
1244         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1245 }
1246
1247 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1248                                          struct nand_chip *chip,
1249                                          u32 data_offs, u32 readlen,
1250                                          u8 *bufpoi, int page)
1251 {
1252         struct nand_ecc_ctrl *ecc = &chip->ecc;
1253         int ret, i, cur_off = 0;
1254         unsigned int max_bitflips = 0;
1255
1256         nand_read_page_op(chip, page, 0, NULL, 0);
1257
1258         sunxi_nfc_hw_ecc_enable(mtd);
1259
1260         for (i = data_offs / ecc->size;
1261              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1262                 int data_off = i * ecc->size;
1263                 int oob_off = i * (ecc->bytes + 4);
1264                 u8 *data = bufpoi + data_off;
1265                 u8 *oob = chip->oob_poi + oob_off;
1266
1267                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1268                                                   oob,
1269                                                   oob_off + mtd->writesize,
1270                                                   &cur_off, &max_bitflips, !i,
1271                                                   false, page);
1272                 if (ret < 0)
1273                         return ret;
1274         }
1275
1276         sunxi_nfc_hw_ecc_disable(mtd);
1277
1278         return max_bitflips;
1279 }
1280
1281 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1282                                              struct nand_chip *chip,
1283                                              u32 data_offs, u32 readlen,
1284                                              u8 *buf, int page)
1285 {
1286         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1287         int ret;
1288
1289         nand_read_page_op(chip, page, 0, NULL, 0);
1290
1291         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1292         if (ret >= 0)
1293                 return ret;
1294
1295         /* Fallback to PIO mode */
1296         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1297                                              buf, page);
1298 }
1299
1300 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1301                                        struct nand_chip *chip,
1302                                        const uint8_t *buf, int oob_required,
1303                                        int page)
1304 {
1305         struct nand_ecc_ctrl *ecc = &chip->ecc;
1306         int ret, i, cur_off = 0;
1307
1308         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1309
1310         sunxi_nfc_hw_ecc_enable(mtd);
1311
1312         for (i = 0; i < ecc->steps; i++) {
1313                 int data_off = i * ecc->size;
1314                 int oob_off = i * (ecc->bytes + 4);
1315                 const u8 *data = buf + data_off;
1316                 const u8 *oob = chip->oob_poi + oob_off;
1317
1318                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1319                                                    oob_off + mtd->writesize,
1320                                                    &cur_off, !i, page);
1321                 if (ret)
1322                         return ret;
1323         }
1324
1325         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1326                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1327                                                  &cur_off, page);
1328
1329         sunxi_nfc_hw_ecc_disable(mtd);
1330
1331         return nand_prog_page_end_op(chip);
1332 }
1333
1334 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1335                                           struct nand_chip *chip,
1336                                           u32 data_offs, u32 data_len,
1337                                           const u8 *buf, int oob_required,
1338                                           int page)
1339 {
1340         struct nand_ecc_ctrl *ecc = &chip->ecc;
1341         int ret, i, cur_off = 0;
1342
1343         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1344
1345         sunxi_nfc_hw_ecc_enable(mtd);
1346
1347         for (i = data_offs / ecc->size;
1348              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1349                 int data_off = i * ecc->size;
1350                 int oob_off = i * (ecc->bytes + 4);
1351                 const u8 *data = buf + data_off;
1352                 const u8 *oob = chip->oob_poi + oob_off;
1353
1354                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1355                                                    oob_off + mtd->writesize,
1356                                                    &cur_off, !i, page);
1357                 if (ret)
1358                         return ret;
1359         }
1360
1361         sunxi_nfc_hw_ecc_disable(mtd);
1362
1363         return nand_prog_page_end_op(chip);
1364 }
1365
1366 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1367                                            struct nand_chip *chip,
1368                                            const u8 *buf,
1369                                            int oob_required,
1370                                            int page)
1371 {
1372         struct nand_chip *nand = mtd_to_nand(mtd);
1373         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1374         struct nand_ecc_ctrl *ecc = &nand->ecc;
1375         struct scatterlist sg;
1376         int ret, i;
1377
1378         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1379         if (ret)
1380                 return ret;
1381
1382         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1383                                        DMA_TO_DEVICE, &sg);
1384         if (ret)
1385                 goto pio_fallback;
1386
1387         for (i = 0; i < ecc->steps; i++) {
1388                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1389
1390                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1391         }
1392
1393         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1394
1395         sunxi_nfc_hw_ecc_enable(mtd);
1396         sunxi_nfc_randomizer_config(mtd, page, false);
1397         sunxi_nfc_randomizer_enable(mtd);
1398
1399         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1400                nfc->regs + NFC_REG_RCMD_SET);
1401
1402         dma_async_issue_pending(nfc->dmac);
1403
1404         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1405                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1406                nfc->regs + NFC_REG_CMD);
1407
1408         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1409         if (ret)
1410                 dmaengine_terminate_all(nfc->dmac);
1411
1412         sunxi_nfc_randomizer_disable(mtd);
1413         sunxi_nfc_hw_ecc_disable(mtd);
1414
1415         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1416
1417         if (ret)
1418                 return ret;
1419
1420         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1421                 /* TODO: use DMA to transfer extra OOB bytes ? */
1422                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1423                                                  NULL, page);
1424
1425         return nand_prog_page_end_op(chip);
1426
1427 pio_fallback:
1428         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1429 }
1430
1431 static int sunxi_nfc_hw_ecc_read_oob(struct mtd_info *mtd,
1432                                      struct nand_chip *chip,
1433                                      int page)
1434 {
1435         chip->pagebuf = -1;
1436
1437         return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page);
1438 }
1439
1440 static int sunxi_nfc_hw_ecc_write_oob(struct mtd_info *mtd,
1441                                       struct nand_chip *chip,
1442                                       int page)
1443 {
1444         int ret;
1445
1446         chip->pagebuf = -1;
1447
1448         memset(chip->data_buf, 0xff, mtd->writesize);
1449         ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page);
1450         if (ret)
1451                 return ret;
1452
1453         /* Send command to program the OOB data */
1454         return nand_prog_page_end_op(chip);
1455 }
1456
1457 static const s32 tWB_lut[] = {6, 12, 16, 20};
1458 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1459
1460 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1461                 u32 clk_period)
1462 {
1463         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1464         int i;
1465
1466         for (i = 0; i < lut_size; i++) {
1467                 if (clk_cycles <= lut[i])
1468                         return i;
1469         }
1470
1471         /* Doesn't fit */
1472         return -EINVAL;
1473 }
1474
1475 #define sunxi_nand_lookup_timing(l, p, c) \
1476                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1477
1478 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline,
1479                                         const struct nand_data_interface *conf)
1480 {
1481         struct nand_chip *nand = mtd_to_nand(mtd);
1482         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1483         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1484         const struct nand_sdr_timings *timings;
1485         u32 min_clk_period = 0;
1486         s32 tWB, tADL, tWHR, tRHW, tCAD;
1487         long real_clk_rate;
1488
1489         timings = nand_get_sdr_timings(conf);
1490         if (IS_ERR(timings))
1491                 return -ENOTSUPP;
1492
1493         /* T1 <=> tCLS */
1494         if (timings->tCLS_min > min_clk_period)
1495                 min_clk_period = timings->tCLS_min;
1496
1497         /* T2 <=> tCLH */
1498         if (timings->tCLH_min > min_clk_period)
1499                 min_clk_period = timings->tCLH_min;
1500
1501         /* T3 <=> tCS */
1502         if (timings->tCS_min > min_clk_period)
1503                 min_clk_period = timings->tCS_min;
1504
1505         /* T4 <=> tCH */
1506         if (timings->tCH_min > min_clk_period)
1507                 min_clk_period = timings->tCH_min;
1508
1509         /* T5 <=> tWP */
1510         if (timings->tWP_min > min_clk_period)
1511                 min_clk_period = timings->tWP_min;
1512
1513         /* T6 <=> tWH */
1514         if (timings->tWH_min > min_clk_period)
1515                 min_clk_period = timings->tWH_min;
1516
1517         /* T7 <=> tALS */
1518         if (timings->tALS_min > min_clk_period)
1519                 min_clk_period = timings->tALS_min;
1520
1521         /* T8 <=> tDS */
1522         if (timings->tDS_min > min_clk_period)
1523                 min_clk_period = timings->tDS_min;
1524
1525         /* T9 <=> tDH */
1526         if (timings->tDH_min > min_clk_period)
1527                 min_clk_period = timings->tDH_min;
1528
1529         /* T10 <=> tRR */
1530         if (timings->tRR_min > (min_clk_period * 3))
1531                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1532
1533         /* T11 <=> tALH */
1534         if (timings->tALH_min > min_clk_period)
1535                 min_clk_period = timings->tALH_min;
1536
1537         /* T12 <=> tRP */
1538         if (timings->tRP_min > min_clk_period)
1539                 min_clk_period = timings->tRP_min;
1540
1541         /* T13 <=> tREH */
1542         if (timings->tREH_min > min_clk_period)
1543                 min_clk_period = timings->tREH_min;
1544
1545         /* T14 <=> tRC */
1546         if (timings->tRC_min > (min_clk_period * 2))
1547                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1548
1549         /* T15 <=> tWC */
1550         if (timings->tWC_min > (min_clk_period * 2))
1551                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1552
1553         /* T16 - T19 + tCAD */
1554         if (timings->tWB_max > (min_clk_period * 20))
1555                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1556
1557         if (timings->tADL_min > (min_clk_period * 32))
1558                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1559
1560         if (timings->tWHR_min > (min_clk_period * 32))
1561                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1562
1563         if (timings->tRHW_min > (min_clk_period * 20))
1564                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1565
1566         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1567                                         min_clk_period);
1568         if (tWB < 0) {
1569                 dev_err(nfc->dev, "unsupported tWB\n");
1570                 return tWB;
1571         }
1572
1573         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1574         if (tADL > 3) {
1575                 dev_err(nfc->dev, "unsupported tADL\n");
1576                 return -EINVAL;
1577         }
1578
1579         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1580         if (tWHR > 3) {
1581                 dev_err(nfc->dev, "unsupported tWHR\n");
1582                 return -EINVAL;
1583         }
1584
1585         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1586                                         min_clk_period);
1587         if (tRHW < 0) {
1588                 dev_err(nfc->dev, "unsupported tRHW\n");
1589                 return tRHW;
1590         }
1591
1592         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1593                 return 0;
1594
1595         /*
1596          * TODO: according to ONFI specs this value only applies for DDR NAND,
1597          * but Allwinner seems to set this to 0x7. Mimic them for now.
1598          */
1599         tCAD = 0x7;
1600
1601         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1602         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1603
1604         /* Convert min_clk_period from picoseconds to nanoseconds */
1605         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1606
1607         /*
1608          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1609          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1610          * This new formula was verified with a scope and validated by
1611          * Allwinner engineers.
1612          */
1613         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1614         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1615         if (real_clk_rate <= 0) {
1616                 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate);
1617                 return -EINVAL;
1618         }
1619
1620         /*
1621          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1622          * output cycle timings shall be used if the host drives tRC less than
1623          * 30 ns.
1624          */
1625         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1626         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1627                            NFC_TIMING_CTL_EDO : 0;
1628
1629         return 0;
1630 }
1631
1632 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1633                                     struct mtd_oob_region *oobregion)
1634 {
1635         struct nand_chip *nand = mtd_to_nand(mtd);
1636         struct nand_ecc_ctrl *ecc = &nand->ecc;
1637
1638         if (section >= ecc->steps)
1639                 return -ERANGE;
1640
1641         oobregion->offset = section * (ecc->bytes + 4) + 4;
1642         oobregion->length = ecc->bytes;
1643
1644         return 0;
1645 }
1646
1647 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1648                                      struct mtd_oob_region *oobregion)
1649 {
1650         struct nand_chip *nand = mtd_to_nand(mtd);
1651         struct nand_ecc_ctrl *ecc = &nand->ecc;
1652
1653         if (section > ecc->steps)
1654                 return -ERANGE;
1655
1656         /*
1657          * The first 2 bytes are used for BB markers, hence we
1658          * only have 2 bytes available in the first user data
1659          * section.
1660          */
1661         if (!section && ecc->mode == NAND_ECC_HW) {
1662                 oobregion->offset = 2;
1663                 oobregion->length = 2;
1664
1665                 return 0;
1666         }
1667
1668         oobregion->offset = section * (ecc->bytes + 4);
1669
1670         if (section < ecc->steps)
1671                 oobregion->length = 4;
1672         else
1673                 oobregion->offset = mtd->oobsize - oobregion->offset;
1674
1675         return 0;
1676 }
1677
1678 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1679         .ecc = sunxi_nand_ooblayout_ecc,
1680         .free = sunxi_nand_ooblayout_free,
1681 };
1682
1683 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1684 {
1685         kfree(ecc->priv);
1686 }
1687
1688 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1689                                        struct nand_ecc_ctrl *ecc,
1690                                        struct device_node *np)
1691 {
1692         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1693         struct nand_chip *nand = mtd_to_nand(mtd);
1694         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1695         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1696         struct sunxi_nand_hw_ecc *data;
1697         int nsectors;
1698         int ret;
1699         int i;
1700
1701         if (ecc->options & NAND_ECC_MAXIMIZE) {
1702                 int bytes;
1703
1704                 ecc->size = 1024;
1705                 nsectors = mtd->writesize / ecc->size;
1706
1707                 /* Reserve 2 bytes for the BBM */
1708                 bytes = (mtd->oobsize - 2) / nsectors;
1709
1710                 /* 4 non-ECC bytes are added before each ECC bytes section */
1711                 bytes -= 4;
1712
1713                 /* and bytes has to be even. */
1714                 if (bytes % 2)
1715                         bytes--;
1716
1717                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1718
1719                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1720                         if (strengths[i] > ecc->strength)
1721                                 break;
1722                 }
1723
1724                 if (!i)
1725                         ecc->strength = 0;
1726                 else
1727                         ecc->strength = strengths[i - 1];
1728         }
1729
1730         if (ecc->size != 512 && ecc->size != 1024)
1731                 return -EINVAL;
1732
1733         data = kzalloc(sizeof(*data), GFP_KERNEL);
1734         if (!data)
1735                 return -ENOMEM;
1736
1737         /* Prefer 1k ECC chunk over 512 ones */
1738         if (ecc->size == 512 && mtd->writesize > 512) {
1739                 ecc->size = 1024;
1740                 ecc->strength *= 2;
1741         }
1742
1743         /* Add ECC info retrieval from DT */
1744         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1745                 if (ecc->strength <= strengths[i]) {
1746                         /*
1747                          * Update ecc->strength value with the actual strength
1748                          * that will be used by the ECC engine.
1749                          */
1750                         ecc->strength = strengths[i];
1751                         break;
1752                 }
1753         }
1754
1755         if (i >= ARRAY_SIZE(strengths)) {
1756                 dev_err(nfc->dev, "unsupported strength\n");
1757                 ret = -ENOTSUPP;
1758                 goto err;
1759         }
1760
1761         data->mode = i;
1762
1763         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1764         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1765
1766         /* HW ECC always work with even numbers of ECC bytes */
1767         ecc->bytes = ALIGN(ecc->bytes, 2);
1768
1769         nsectors = mtd->writesize / ecc->size;
1770
1771         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1772                 ret = -EINVAL;
1773                 goto err;
1774         }
1775
1776         ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1777         ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1778         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1779         ecc->priv = data;
1780
1781         if (nfc->dmac) {
1782                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1783                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1784                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1785                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1786         } else {
1787                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1788                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1789                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1790         }
1791
1792         /* TODO: support DMA for raw accesses and subpage write */
1793         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1794         ecc->read_oob_raw = nand_read_oob_std;
1795         ecc->write_oob_raw = nand_write_oob_std;
1796
1797         return 0;
1798
1799 err:
1800         kfree(data);
1801
1802         return ret;
1803 }
1804
1805 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1806 {
1807         switch (ecc->mode) {
1808         case NAND_ECC_HW:
1809                 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1810                 break;
1811         case NAND_ECC_NONE:
1812         default:
1813                 break;
1814         }
1815 }
1816
1817 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1818 {
1819         struct mtd_info *mtd = nand_to_mtd(nand);
1820         struct nand_ecc_ctrl *ecc = &nand->ecc;
1821         struct device_node *np = nand_get_flash_node(nand);
1822         int ret;
1823
1824         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1825                 nand->bbt_options |= NAND_BBT_NO_OOB;
1826
1827         if (nand->options & NAND_NEED_SCRAMBLING)
1828                 nand->options |= NAND_NO_SUBPAGE_WRITE;
1829
1830         nand->options |= NAND_SUBPAGE_READ;
1831
1832         if (!ecc->size) {
1833                 ecc->size = nand->ecc_step_ds;
1834                 ecc->strength = nand->ecc_strength_ds;
1835         }
1836
1837         if (!ecc->size || !ecc->strength)
1838                 return -EINVAL;
1839
1840         switch (ecc->mode) {
1841         case NAND_ECC_HW:
1842                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1843                 if (ret)
1844                         return ret;
1845                 break;
1846         case NAND_ECC_NONE:
1847         case NAND_ECC_SOFT:
1848                 break;
1849         default:
1850                 return -EINVAL;
1851         }
1852
1853         return 0;
1854 }
1855
1856 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1857         .attach_chip = sunxi_nand_attach_chip,
1858 };
1859
1860 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1861                                 struct device_node *np)
1862 {
1863         struct sunxi_nand_chip *chip;
1864         struct mtd_info *mtd;
1865         struct nand_chip *nand;
1866         int nsels;
1867         int ret;
1868         int i;
1869         u32 tmp;
1870
1871         if (!of_get_property(np, "reg", &nsels))
1872                 return -EINVAL;
1873
1874         nsels /= sizeof(u32);
1875         if (!nsels) {
1876                 dev_err(dev, "invalid reg property size\n");
1877                 return -EINVAL;
1878         }
1879
1880         chip = devm_kzalloc(dev,
1881                             sizeof(*chip) +
1882                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
1883                             GFP_KERNEL);
1884         if (!chip) {
1885                 dev_err(dev, "could not allocate chip\n");
1886                 return -ENOMEM;
1887         }
1888
1889         chip->nsels = nsels;
1890         chip->selected = -1;
1891
1892         for (i = 0; i < nsels; i++) {
1893                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1894                 if (ret) {
1895                         dev_err(dev, "could not retrieve reg property: %d\n",
1896                                 ret);
1897                         return ret;
1898                 }
1899
1900                 if (tmp > NFC_MAX_CS) {
1901                         dev_err(dev,
1902                                 "invalid reg value: %u (max CS = 7)\n",
1903                                 tmp);
1904                         return -EINVAL;
1905                 }
1906
1907                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1908                         dev_err(dev, "CS %d already assigned\n", tmp);
1909                         return -EINVAL;
1910                 }
1911
1912                 chip->sels[i].cs = tmp;
1913
1914                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1915                     tmp < 2)
1916                         chip->sels[i].rb = tmp;
1917                 else
1918                         chip->sels[i].rb = -1;
1919         }
1920
1921         nand = &chip->nand;
1922         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1923         nand->chip_delay = 200;
1924         nand->controller = &nfc->controller;
1925         nand->controller->ops = &sunxi_nand_controller_ops;
1926
1927         /*
1928          * Set the ECC mode to the default value in case nothing is specified
1929          * in the DT.
1930          */
1931         nand->ecc.mode = NAND_ECC_HW;
1932         nand_set_flash_node(nand, np);
1933         nand->select_chip = sunxi_nfc_select_chip;
1934         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
1935         nand->read_buf = sunxi_nfc_read_buf;
1936         nand->write_buf = sunxi_nfc_write_buf;
1937         nand->read_byte = sunxi_nfc_read_byte;
1938         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
1939
1940         mtd = nand_to_mtd(nand);
1941         mtd->dev.parent = dev;
1942
1943         ret = nand_scan(mtd, nsels);
1944         if (ret)
1945                 return ret;
1946
1947         ret = mtd_device_register(mtd, NULL, 0);
1948         if (ret) {
1949                 dev_err(dev, "failed to register mtd device: %d\n", ret);
1950                 nand_release(mtd);
1951                 return ret;
1952         }
1953
1954         list_add_tail(&chip->node, &nfc->chips);
1955
1956         return 0;
1957 }
1958
1959 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
1960 {
1961         struct device_node *np = dev->of_node;
1962         struct device_node *nand_np;
1963         int nchips = of_get_child_count(np);
1964         int ret;
1965
1966         if (nchips > 8) {
1967                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
1968                 return -EINVAL;
1969         }
1970
1971         for_each_child_of_node(np, nand_np) {
1972                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
1973                 if (ret) {
1974                         of_node_put(nand_np);
1975                         return ret;
1976                 }
1977         }
1978
1979         return 0;
1980 }
1981
1982 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
1983 {
1984         struct sunxi_nand_chip *chip;
1985
1986         while (!list_empty(&nfc->chips)) {
1987                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
1988                                         node);
1989                 nand_release(nand_to_mtd(&chip->nand));
1990                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
1991                 list_del(&chip->node);
1992         }
1993 }
1994
1995 static int sunxi_nfc_probe(struct platform_device *pdev)
1996 {
1997         struct device *dev = &pdev->dev;
1998         struct resource *r;
1999         struct sunxi_nfc *nfc;
2000         int irq;
2001         int ret;
2002
2003         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2004         if (!nfc)
2005                 return -ENOMEM;
2006
2007         nfc->dev = dev;
2008         nand_controller_init(&nfc->controller);
2009         INIT_LIST_HEAD(&nfc->chips);
2010
2011         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2012         nfc->regs = devm_ioremap_resource(dev, r);
2013         if (IS_ERR(nfc->regs))
2014                 return PTR_ERR(nfc->regs);
2015
2016         irq = platform_get_irq(pdev, 0);
2017         if (irq < 0) {
2018                 dev_err(dev, "failed to retrieve irq\n");
2019                 return irq;
2020         }
2021
2022         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2023         if (IS_ERR(nfc->ahb_clk)) {
2024                 dev_err(dev, "failed to retrieve ahb clk\n");
2025                 return PTR_ERR(nfc->ahb_clk);
2026         }
2027
2028         ret = clk_prepare_enable(nfc->ahb_clk);
2029         if (ret)
2030                 return ret;
2031
2032         nfc->mod_clk = devm_clk_get(dev, "mod");
2033         if (IS_ERR(nfc->mod_clk)) {
2034                 dev_err(dev, "failed to retrieve mod clk\n");
2035                 ret = PTR_ERR(nfc->mod_clk);
2036                 goto out_ahb_clk_unprepare;
2037         }
2038
2039         ret = clk_prepare_enable(nfc->mod_clk);
2040         if (ret)
2041                 goto out_ahb_clk_unprepare;
2042
2043         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2044         if (IS_ERR(nfc->reset)) {
2045                 ret = PTR_ERR(nfc->reset);
2046                 goto out_mod_clk_unprepare;
2047         }
2048
2049         ret = reset_control_deassert(nfc->reset);
2050         if (ret) {
2051                 dev_err(dev, "reset err %d\n", ret);
2052                 goto out_mod_clk_unprepare;
2053         }
2054
2055         ret = sunxi_nfc_rst(nfc);
2056         if (ret)
2057                 goto out_ahb_reset_reassert;
2058
2059         writel(0, nfc->regs + NFC_REG_INT);
2060         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2061                                0, "sunxi-nand", nfc);
2062         if (ret)
2063                 goto out_ahb_reset_reassert;
2064
2065         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2066         if (nfc->dmac) {
2067                 struct dma_slave_config dmac_cfg = { };
2068
2069                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2070                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2071                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2072                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2073                 dmac_cfg.src_maxburst = 4;
2074                 dmac_cfg.dst_maxburst = 4;
2075                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2076         } else {
2077                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2078         }
2079
2080         platform_set_drvdata(pdev, nfc);
2081
2082         ret = sunxi_nand_chips_init(dev, nfc);
2083         if (ret) {
2084                 dev_err(dev, "failed to init nand chips\n");
2085                 goto out_release_dmac;
2086         }
2087
2088         return 0;
2089
2090 out_release_dmac:
2091         if (nfc->dmac)
2092                 dma_release_channel(nfc->dmac);
2093 out_ahb_reset_reassert:
2094         reset_control_assert(nfc->reset);
2095 out_mod_clk_unprepare:
2096         clk_disable_unprepare(nfc->mod_clk);
2097 out_ahb_clk_unprepare:
2098         clk_disable_unprepare(nfc->ahb_clk);
2099
2100         return ret;
2101 }
2102
2103 static int sunxi_nfc_remove(struct platform_device *pdev)
2104 {
2105         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2106
2107         sunxi_nand_chips_cleanup(nfc);
2108
2109         reset_control_assert(nfc->reset);
2110
2111         if (nfc->dmac)
2112                 dma_release_channel(nfc->dmac);
2113         clk_disable_unprepare(nfc->mod_clk);
2114         clk_disable_unprepare(nfc->ahb_clk);
2115
2116         return 0;
2117 }
2118
2119 static const struct of_device_id sunxi_nfc_ids[] = {
2120         { .compatible = "allwinner,sun4i-a10-nand" },
2121         { /* sentinel */ }
2122 };
2123 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2124
2125 static struct platform_driver sunxi_nfc_driver = {
2126         .driver = {
2127                 .name = "sunxi_nand",
2128                 .of_match_table = sunxi_nfc_ids,
2129         },
2130         .probe = sunxi_nfc_probe,
2131         .remove = sunxi_nfc_remove,
2132 };
2133 module_platform_driver(sunxi_nfc_driver);
2134
2135 MODULE_LICENSE("GPL v2");
2136 MODULE_AUTHOR("Boris BREZILLON");
2137 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2138 MODULE_ALIAS("platform:sunxi_nand");