drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/rawnand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_ECC_BLOCK_512       BIT(5)
 149 #define NFC_RANDOM_EN           BIT(9)
 150 #define NFC_RANDOM_DIRECTION    BIT(10)
 151 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 152 #define NFC_ECC_MODE(x)         ((x) << 12)
 153 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 154 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 155
 156 /* define bit use in NFC_ECC_ST */
 157 #define NFC_ECC_ERR(x)          BIT(x)
 158 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 159 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 160 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 161
 162 #define NFC_DEFAULT_TIMEOUT_MS  1000
 163
 164 #define NFC_SRAM_SIZE           1024
 165
 166 #define NFC_MAX_CS              7
 167
 168 /*
 169  * Ready/Busy detection type: describes the Ready/Busy detection modes
 170  *
 171  * @RB_NONE:    no external detection available, rely on STATUS command
 172  *              and software timeouts
 173  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 174  *              pin of the NAND flash chip must be connected to one of the
 175  *              native NAND R/B pins (those which can be muxed to the NAND
 176  *              Controller)
 177  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 178  *              pin of the NAND flash chip must be connected to a GPIO capable
 179  *              pin.
 180  */
 181 enum sunxi_nand_rb_type {
 182         RB_NONE,
 183         RB_NATIVE,
 184         RB_GPIO,
 185 };
 186
 187 /*
 188  * Ready/Busy structure: stores information related to Ready/Busy detection
 189  *
 190  * @type:       the Ready/Busy detection mode
 191  * @info:       information related to the R/B detection mode. Either a gpio
 192  *              id or a native R/B id (those supported by the NAND controller).
 193  */
 194 struct sunxi_nand_rb {
 195         enum sunxi_nand_rb_type type;
 196         union {
 197                 int gpio;
 198                 int nativeid;
 199         } info;
 200 };
 201
 202 /*
 203  * Chip Select structure: stores information related to NAND Chip Select
 204  *
 205  * @cs:         the NAND CS id used to communicate with a NAND Chip
 206  * @rb:         the Ready/Busy description
 207  */
 208 struct sunxi_nand_chip_sel {
 209         u8 cs;
 210         struct sunxi_nand_rb rb;
 211 };
 212
 213 /*
 214  * sunxi HW ECC infos: stores information related to HW ECC support
 215  *
 216  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 217  */
 218 struct sunxi_nand_hw_ecc {
 219         int mode;
 220 };
 221
 222 /*
 223  * NAND chip structure: stores NAND chip device related information
 224  *
 225  * @node:               used to store NAND chips into a list
 226  * @nand:               base NAND chip structure
 227  * @mtd:                base MTD structure
 228  * @clk_rate:           clk_rate required for this NAND chip
 229  * @timing_cfg          TIMING_CFG register value for this NAND chip
 230  * @selected:           current active CS
 231  * @nsels:              number of CS lines required by the NAND chip
 232  * @sels:               array of CS lines descriptions
 233  */
 234 struct sunxi_nand_chip {
 235         struct list_head node;
 236         struct nand_chip nand;
 237         unsigned long clk_rate;
 238         u32 timing_cfg;
 239         u32 timing_ctl;
 240         int selected;
 241         int addr_cycles;
 242         u32 addr[2];
 243         int cmd_cycles;
 244         u8 cmd[2];
 245         int nsels;
 246         struct sunxi_nand_chip_sel sels[0];
 247 };
 248
 249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 250 {
 251         return container_of(nand, struct sunxi_nand_chip, nand);
 252 }
 253
 254 /*
 255  * NAND Controller structure: stores sunxi NAND controller information
 256  *
 257  * @controller:         base controller structure
 258  * @dev:                parent device (used to print error messages)
 259  * @regs:               NAND controller registers
 260  * @ahb_clk:            NAND Controller AHB clock
 261  * @mod_clk:            NAND Controller mod clock
 262  * @assigned_cs:        bitmask describing already assigned CS lines
 263  * @clk_rate:           NAND controller current clock rate
 264  * @chips:              a list containing all the NAND chips attached to
 265  *                      this NAND controller
 266  * @complete:           a completion object used to wait for NAND
 267  *                      controller events
 268  */
 269 struct sunxi_nfc {
 270         struct nand_hw_control controller;
 271         struct device *dev;
 272         void __iomem *regs;
 273         struct clk *ahb_clk;
 274         struct clk *mod_clk;
 275         struct reset_control *reset;
 276         unsigned long assigned_cs;
 277         unsigned long clk_rate;
 278         struct list_head chips;
 279         struct completion complete;
 280         struct dma_chan *dmac;
 281 };
 282
 283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 284 {
 285         return container_of(ctrl, struct sunxi_nfc, controller);
 286 }
 287
 288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 289 {
 290         struct sunxi_nfc *nfc = dev_id;
 291         u32 st = readl(nfc->regs + NFC_REG_ST);
 292         u32 ien = readl(nfc->regs + NFC_REG_INT);
 293
 294         if (!(ien & st))
 295                 return IRQ_NONE;
 296
 297         if ((ien & st) == ien)
 298                 complete(&nfc->complete);
 299
 300         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 301         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 307                                  bool use_polling, unsigned int timeout_ms)
 308 {
 309         int ret;
 310
 311         if (events & ~NFC_INT_MASK)
 312                 return -EINVAL;
 313
 314         if (!timeout_ms)
 315                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 316
 317         if (!use_polling) {
 318                 init_completion(&nfc->complete);
 319
 320                 writel(events, nfc->regs + NFC_REG_INT);
 321
 322                 ret = wait_for_completion_timeout(&nfc->complete,
 323                                                 msecs_to_jiffies(timeout_ms));
 324                 if (!ret)
 325                         ret = -ETIMEDOUT;
 326                 else
 327                         ret = 0;
 328
 329                 writel(0, nfc->regs + NFC_REG_INT);
 330         } else {
 331                 u32 status;
 332
 333                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 334                                          (status & events) == events, 1,
 335                                          timeout_ms * 1000);
 336         }
 337
 338         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 339
 340         if (ret)
 341                 dev_err(nfc->dev, "wait interrupt timedout\n");
 342
 343         return ret;
 344 }
 345
 346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 347 {
 348         u32 status;
 349         int ret;
 350
 351         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 352                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 353                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 354         if (ret)
 355                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 356
 357         return ret;
 358 }
 359
 360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 361 {
 362         u32 ctl;
 363         int ret;
 364
 365         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 366         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 367
 368         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 369                                  !(ctl & NFC_RESET), 1,
 370                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 371         if (ret)
 372                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 373
 374         return ret;
 375 }
 376
 377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 378                                     int chunksize, int nchunks,
 379                                     enum dma_data_direction ddir,
 380                                     struct scatterlist *sg)
 381 {
 382         struct nand_chip *nand = mtd_to_nand(mtd);
 383         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 384         struct dma_async_tx_descriptor *dmad;
 385         enum dma_transfer_direction tdir;
 386         dma_cookie_t dmat;
 387         int ret;
 388
 389         if (ddir == DMA_FROM_DEVICE)
 390                 tdir = DMA_DEV_TO_MEM;
 391         else
 392                 tdir = DMA_MEM_TO_DEV;
 393
 394         sg_init_one(sg, buf, nchunks * chunksize);
 395         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 396         if (!ret)
 397                 return -ENOMEM;
 398
 399         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 400         if (!dmad) {
 401                 ret = -EINVAL;
 402                 goto err_unmap_buf;
 403         }
 404
 405         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 406                nfc->regs + NFC_REG_CTL);
 407         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 408         writel(chunksize, nfc->regs + NFC_REG_CNT);
 409         dmat = dmaengine_submit(dmad);
 410
 411         ret = dma_submit_error(dmat);
 412         if (ret)
 413                 goto err_clr_dma_flag;
 414
 415         return 0;
 416
 417 err_clr_dma_flag:
 418         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 419                nfc->regs + NFC_REG_CTL);
 420
 421 err_unmap_buf:
 422         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 423         return ret;
 424 }
 425
 426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 427                                      enum dma_data_direction ddir,
 428                                      struct scatterlist *sg)
 429 {
 430         struct nand_chip *nand = mtd_to_nand(mtd);
 431         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 432
 433         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 434         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 435                nfc->regs + NFC_REG_CTL);
 436 }
 437
 438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 439 {
 440         struct nand_chip *nand = mtd_to_nand(mtd);
 441         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 442         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 443         struct sunxi_nand_rb *rb;
 444         int ret;
 445
 446         if (sunxi_nand->selected < 0)
 447                 return 0;
 448
 449         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 450
 451         switch (rb->type) {
 452         case RB_NATIVE:
 453                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 454                          NFC_RB_STATE(rb->info.nativeid));
 455                 break;
 456         case RB_GPIO:
 457                 ret = gpio_get_value(rb->info.gpio);
 458                 break;
 459         case RB_NONE:
 460         default:
 461                 ret = 0;
 462                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 463                 break;
 464         }
 465
 466         return ret;
 467 }
 468
 469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 470 {
 471         struct nand_chip *nand = mtd_to_nand(mtd);
 472         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 473         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 474         struct sunxi_nand_chip_sel *sel;
 475         u32 ctl;
 476
 477         if (chip > 0 && chip >= sunxi_nand->nsels)
 478                 return;
 479
 480         if (chip == sunxi_nand->selected)
 481                 return;
 482
 483         ctl = readl(nfc->regs + NFC_REG_CTL) &
 484               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 485
 486         if (chip >= 0) {
 487                 sel = &sunxi_nand->sels[chip];
 488
 489                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 490                        NFC_PAGE_SHIFT(nand->page_shift);
 491                 if (sel->rb.type == RB_NONE) {
 492                         nand->dev_ready = NULL;
 493                 } else {
 494                         nand->dev_ready = sunxi_nfc_dev_ready;
 495                         if (sel->rb.type == RB_NATIVE)
 496                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 497                 }
 498
 499                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 500
 501                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 502                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 503                         nfc->clk_rate = sunxi_nand->clk_rate;
 504                 }
 505         }
 506
 507         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 508         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 509         writel(ctl, nfc->regs + NFC_REG_CTL);
 510
 511         sunxi_nand->selected = chip;
 512 }
 513
 514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 515 {
 516         struct nand_chip *nand = mtd_to_nand(mtd);
 517         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 518         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 519         int ret;
 520         int cnt;
 521         int offs = 0;
 522         u32 tmp;
 523
 524         while (len > offs) {
 525                 bool poll = false;
 526
 527                 cnt = min(len - offs, NFC_SRAM_SIZE);
 528
 529                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 530                 if (ret)
 531                         break;
 532
 533                 writel(cnt, nfc->regs + NFC_REG_CNT);
 534                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 535                 writel(tmp, nfc->regs + NFC_REG_CMD);
 536
 537                 /* Arbitrary limit for polling mode */
 538                 if (cnt < 64)
 539                         poll = true;
 540
 541                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 542                 if (ret)
 543                         break;
 544
 545                 if (buf)
 546                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 547                                       cnt);
 548                 offs += cnt;
 549         }
 550 }
 551
 552 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 553                                 int len)
 554 {
 555         struct nand_chip *nand = mtd_to_nand(mtd);
 556         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 557         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 558         int ret;
 559         int cnt;
 560         int offs = 0;
 561         u32 tmp;
 562
 563         while (len > offs) {
 564                 bool poll = false;
 565
 566                 cnt = min(len - offs, NFC_SRAM_SIZE);
 567
 568                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 569                 if (ret)
 570                         break;
 571
 572                 writel(cnt, nfc->regs + NFC_REG_CNT);
 573                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 574                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 575                       NFC_ACCESS_DIR;
 576                 writel(tmp, nfc->regs + NFC_REG_CMD);
 577
 578                 /* Arbitrary limit for polling mode */
 579                 if (cnt < 64)
 580                         poll = true;
 581
 582                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 583                 if (ret)
 584                         break;
 585
 586                 offs += cnt;
 587         }
 588 }
 589
 590 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 591 {
 592         uint8_t ret;
 593
 594         sunxi_nfc_read_buf(mtd, &ret, 1);
 595
 596         return ret;
 597 }
 598
 599 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 600                                unsigned int ctrl)
 601 {
 602         struct nand_chip *nand = mtd_to_nand(mtd);
 603         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 604         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 605         int ret;
 606
 607         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 608             !(ctrl & (NAND_CLE | NAND_ALE))) {
 609                 u32 cmd = 0;
 610
 611                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 612                         return;
 613
 614                 if (sunxi_nand->cmd_cycles--)
 615                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 616
 617                 if (sunxi_nand->cmd_cycles--) {
 618                         cmd |= NFC_SEND_CMD2;
 619                         writel(sunxi_nand->cmd[1],
 620                                nfc->regs + NFC_REG_RCMD_SET);
 621                 }
 622
 623                 sunxi_nand->cmd_cycles = 0;
 624
 625                 if (sunxi_nand->addr_cycles) {
 626                         cmd |= NFC_SEND_ADR |
 627                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 628                         writel(sunxi_nand->addr[0],
 629                                nfc->regs + NFC_REG_ADDR_LOW);
 630                 }
 631
 632                 if (sunxi_nand->addr_cycles > 4)
 633                         writel(sunxi_nand->addr[1],
 634                                nfc->regs + NFC_REG_ADDR_HIGH);
 635
 636                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 637                 if (ret)
 638                         return;
 639
 640                 writel(cmd, nfc->regs + NFC_REG_CMD);
 641                 sunxi_nand->addr[0] = 0;
 642                 sunxi_nand->addr[1] = 0;
 643                 sunxi_nand->addr_cycles = 0;
 644                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 645         }
 646
 647         if (ctrl & NAND_CLE) {
 648                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 649         } else if (ctrl & NAND_ALE) {
 650                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 651                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 652                 sunxi_nand->addr_cycles++;
 653         }
 654 }
 655
 656 /* These seed values have been extracted from Allwinner's BSP */
 657 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 658         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 659         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 660         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 661         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 662         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 663         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 664         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 665         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 666         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 667         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 668         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 669         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 670         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 671         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 672         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 673         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 674 };
 675
 676 /*
 677  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 678  * have been generated using
 679  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 680  * the randomizer engine does internally before de/scrambling OOB data.
 681  *
 682  * Those tables are statically defined to avoid calculating randomizer state
 683  * at runtime.
 684  */
 685 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 686         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 687         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 688         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 689         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 690         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 691         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 692         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 693         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 694         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 695         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 696         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 697         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 698         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 699         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 700         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 701         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 702 };
 703
 704 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 705         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 706         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 707         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 708         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 709         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 710         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 711         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 712         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 713         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 714         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 715         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 716         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 717         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 718         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 719         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 720         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 721 };
 722
 723 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 724 {
 725         state &= 0x7fff;
 726
 727         /*
 728          * This loop is just a simple implementation of a Fibonacci LFSR using
 729          * the x16 + x15 + 1 polynomial.
 730          */
 731         while (count--)
 732                 state = ((state >> 1) |
 733                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 734
 735         return state;
 736 }
 737
 738 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 739 {
 740         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 741         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 742
 743         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 744                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 745
 746         if (ecc) {
 747                 if (mtd->ecc_step_size == 512)
 748                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 749                 else
 750                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 751         }
 752
 753         return seeds[page % mod];
 754 }
 755
 756 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 757                                         int page, bool ecc)
 758 {
 759         struct nand_chip *nand = mtd_to_nand(mtd);
 760         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 761         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 762         u16 state;
 763
 764         if (!(nand->options & NAND_NEED_SCRAMBLING))
 765                 return;
 766
 767         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 768         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 769         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 770         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 771 }
 772
 773 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 774 {
 775         struct nand_chip *nand = mtd_to_nand(mtd);
 776         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 777
 778         if (!(nand->options & NAND_NEED_SCRAMBLING))
 779                 return;
 780
 781         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 782                nfc->regs + NFC_REG_ECC_CTL);
 783 }
 784
 785 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 786 {
 787         struct nand_chip *nand = mtd_to_nand(mtd);
 788         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 789
 790         if (!(nand->options & NAND_NEED_SCRAMBLING))
 791                 return;
 792
 793         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 794                nfc->regs + NFC_REG_ECC_CTL);
 795 }
 796
 797 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 798 {
 799         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 800
 801         bbm[0] ^= state;
 802         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 803 }
 804
 805 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 806                                            const uint8_t *buf, int len,
 807                                            bool ecc, int page)
 808 {
 809         sunxi_nfc_randomizer_config(mtd, page, ecc);
 810         sunxi_nfc_randomizer_enable(mtd);
 811         sunxi_nfc_write_buf(mtd, buf, len);
 812         sunxi_nfc_randomizer_disable(mtd);
 813 }
 814
 815 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 816                                           int len, bool ecc, int page)
 817 {
 818         sunxi_nfc_randomizer_config(mtd, page, ecc);
 819         sunxi_nfc_randomizer_enable(mtd);
 820         sunxi_nfc_read_buf(mtd, buf, len);
 821         sunxi_nfc_randomizer_disable(mtd);
 822 }
 823
 824 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 825 {
 826         struct nand_chip *nand = mtd_to_nand(mtd);
 827         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 828         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 829         u32 ecc_ctl;
 830
 831         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 832         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 833                      NFC_ECC_BLOCK_SIZE_MSK);
 834         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 835                    NFC_ECC_PIPELINE;
 836
 837         if (nand->ecc.size == 512)
 838                 ecc_ctl |= NFC_ECC_BLOCK_512;
 839
 840         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 841 }
 842
 843 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 844 {
 845         struct nand_chip *nand = mtd_to_nand(mtd);
 846         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 847
 848         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 849                nfc->regs + NFC_REG_ECC_CTL);
 850 }
 851
 852 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 853 {
 854         buf[0] = user_data;
 855         buf[1] = user_data >> 8;
 856         buf[2] = user_data >> 16;
 857         buf[3] = user_data >> 24;
 858 }
 859
 860 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 861 {
 862         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 863 }
 864
 865 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 866                                                 int step, bool bbm, int page)
 867 {
 868         struct nand_chip *nand = mtd_to_nand(mtd);
 869         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 870
 871         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 872                                    oob);
 873
 874         /* De-randomize the Bad Block Marker. */
 875         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 876                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 877 }
 878
 879 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 880                                                 const u8 *oob, int step,
 881                                                 bool bbm, int page)
 882 {
 883         struct nand_chip *nand = mtd_to_nand(mtd);
 884         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 885         u8 user_data[4];
 886
 887         /* Randomize the Bad Block Marker. */
 888         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 889                 memcpy(user_data, oob, sizeof(user_data));
 890                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 891                 oob = user_data;
 892         }
 893
 894         writel(sunxi_nfc_buf_to_user_data(oob),
 895                nfc->regs + NFC_REG_USER_DATA(step));
 896 }
 897
 898 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 899                                           unsigned int *max_bitflips, int ret)
 900 {
 901         if (ret < 0) {
 902                 mtd->ecc_stats.failed++;
 903         } else {
 904                 mtd->ecc_stats.corrected += ret;
 905                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 906         }
 907 }
 908
 909 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 910                                     int step, u32 status, bool *erased)
 911 {
 912         struct nand_chip *nand = mtd_to_nand(mtd);
 913         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 914         struct nand_ecc_ctrl *ecc = &nand->ecc;
 915         u32 tmp;
 916
 917         *erased = false;
 918
 919         if (status & NFC_ECC_ERR(step))
 920                 return -EBADMSG;
 921
 922         if (status & NFC_ECC_PAT_FOUND(step)) {
 923                 u8 pattern;
 924
 925                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 926                         pattern = 0x0;
 927                 } else {
 928                         pattern = 0xff;
 929                         *erased = true;
 930                 }
 931
 932                 if (data)
 933                         memset(data, pattern, ecc->size);
 934
 935                 if (oob)
 936                         memset(oob, pattern, ecc->bytes + 4);
 937
 938                 return 0;
 939         }
 940
 941         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 942
 943         return NFC_ECC_ERR_CNT(step, tmp);
 944 }
 945
 946 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 947                                        u8 *data, int data_off,
 948                                        u8 *oob, int oob_off,
 949                                        int *cur_off,
 950                                        unsigned int *max_bitflips,
 951                                        bool bbm, bool oob_required, int page)
 952 {
 953         struct nand_chip *nand = mtd_to_nand(mtd);
 954         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 955         struct nand_ecc_ctrl *ecc = &nand->ecc;
 956         int raw_mode = 0;
 957         bool erased;
 958         int ret;
 959
 960         if (*cur_off != data_off)
 961                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 962
 963         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 964
 965         if (data_off + ecc->size != oob_off)
 966                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 967
 968         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 969         if (ret)
 970                 return ret;
 971
 972         sunxi_nfc_randomizer_enable(mtd);
 973         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 974                nfc->regs + NFC_REG_CMD);
 975
 976         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 977         sunxi_nfc_randomizer_disable(mtd);
 978         if (ret)
 979                 return ret;
 980
 981         *cur_off = oob_off + ecc->bytes + 4;
 982
 983         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 984                                        readl(nfc->regs + NFC_REG_ECC_ST),
 985                                        &erased);
 986         if (erased)
 987                 return 1;
 988
 989         if (ret < 0) {
 990                 /*
 991                  * Re-read the data with the randomizer disabled to identify
 992                  * bitflips in erased pages.
 993                  */
 994                 if (nand->options & NAND_NEED_SCRAMBLING)
 995                         nand_change_read_column_op(nand, data_off, data,
 996                                                    ecc->size, false);
 997                 else
 998                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 999                                       ecc->size);
1000
1001                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
1002                                            false);
1003
1004                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1005                                                   oob, ecc->bytes + 4,
1006                                                   NULL, 0, ecc->strength);
1007                 if (ret >= 0)
1008                         raw_mode = 1;
1009         } else {
1010                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
1011
1012                 if (oob_required) {
1013                         nand_change_read_column_op(nand, oob_off, NULL, 0,
1014                                                    false);
1015                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1016                                                       true, page);
1017
1018                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1019                                                             bbm, page);
1020                 }
1021         }
1022
1023         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1024
1025         return raw_mode;
1026 }
1027
1028 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1029                                             u8 *oob, int *cur_off,
1030                                             bool randomize, int page)
1031 {
1032         struct nand_chip *nand = mtd_to_nand(mtd);
1033         struct nand_ecc_ctrl *ecc = &nand->ecc;
1034         int offset = ((ecc->bytes + 4) * ecc->steps);
1035         int len = mtd->oobsize - offset;
1036
1037         if (len <= 0)
1038                 return;
1039
1040         if (!cur_off || *cur_off != offset)
1041                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
1042                                            false);
1043
1044         if (!randomize)
1045                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1046         else
1047                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1048                                               false, page);
1049
1050         if (cur_off)
1051                 *cur_off = mtd->oobsize + mtd->writesize;
1052 }
1053
1054 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1055                                             int oob_required, int page,
1056                                             int nchunks)
1057 {
1058         struct nand_chip *nand = mtd_to_nand(mtd);
1059         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1060         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1061         struct nand_ecc_ctrl *ecc = &nand->ecc;
1062         unsigned int max_bitflips = 0;
1063         int ret, i, raw_mode = 0;
1064         struct scatterlist sg;
1065         u32 status;
1066
1067         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1068         if (ret)
1069                 return ret;
1070
1071         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1072                                        DMA_FROM_DEVICE, &sg);
1073         if (ret)
1074                 return ret;
1075
1076         sunxi_nfc_hw_ecc_enable(mtd);
1077         sunxi_nfc_randomizer_config(mtd, page, false);
1078         sunxi_nfc_randomizer_enable(mtd);
1079
1080         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1081                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1082
1083         dma_async_issue_pending(nfc->dmac);
1084
1085         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1086                nfc->regs + NFC_REG_CMD);
1087
1088         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1089         if (ret)
1090                 dmaengine_terminate_all(nfc->dmac);
1091
1092         sunxi_nfc_randomizer_disable(mtd);
1093         sunxi_nfc_hw_ecc_disable(mtd);
1094
1095         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1096
1097         if (ret)
1098                 return ret;
1099
1100         status = readl(nfc->regs + NFC_REG_ECC_ST);
1101
1102         for (i = 0; i < nchunks; i++) {
1103                 int data_off = i * ecc->size;
1104                 int oob_off = i * (ecc->bytes + 4);
1105                 u8 *data = buf + data_off;
1106                 u8 *oob = nand->oob_poi + oob_off;
1107                 bool erased;
1108
1109                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1110                                                oob_required ? oob : NULL,
1111                                                i, status, &erased);
1112
1113                 /* ECC errors are handled in the second loop. */
1114                 if (ret < 0)
1115                         continue;
1116
1117                 if (oob_required && !erased) {
1118                         /* TODO: use DMA to retrieve OOB */
1119                         nand_change_read_column_op(nand,
1120                                                    mtd->writesize + oob_off,
1121                                                    oob, ecc->bytes + 4, false);
1122
1123                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1124                                                             !i, page);
1125                 }
1126
1127                 if (erased)
1128                         raw_mode = 1;
1129
1130                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1131         }
1132
1133         if (status & NFC_ECC_ERR_MSK) {
1134                 for (i = 0; i < nchunks; i++) {
1135                         int data_off = i * ecc->size;
1136                         int oob_off = i * (ecc->bytes + 4);
1137                         u8 *data = buf + data_off;
1138                         u8 *oob = nand->oob_poi + oob_off;
1139
1140                         if (!(status & NFC_ECC_ERR(i)))
1141                                 continue;
1142
1143                         /*
1144                          * Re-read the data with the randomizer disabled to
1145                          * identify bitflips in erased pages.
1146                          * TODO: use DMA to read page in raw mode
1147                          */
1148                         if (randomized)
1149                                 nand_change_read_column_op(nand, data_off,
1150                                                            data, ecc->size,
1151                                                            false);
1152
1153                         /* TODO: use DMA to retrieve OOB */
1154                         nand_change_read_column_op(nand,
1155                                                    mtd->writesize + oob_off,
1156                                                    oob, ecc->bytes + 4, false);
1157
1158                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1159                                                           oob, ecc->bytes + 4,
1160                                                           NULL, 0,
1161                                                           ecc->strength);
1162                         if (ret >= 0)
1163                                 raw_mode = 1;
1164
1165                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1166                 }
1167         }
1168
1169         if (oob_required)
1170                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1171                                                 NULL, !raw_mode,
1172                                                 page);
1173
1174         return max_bitflips;
1175 }
1176
1177 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1178                                         const u8 *data, int data_off,
1179                                         const u8 *oob, int oob_off,
1180                                         int *cur_off, bool bbm,
1181                                         int page)
1182 {
1183         struct nand_chip *nand = mtd_to_nand(mtd);
1184         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1185         struct nand_ecc_ctrl *ecc = &nand->ecc;
1186         int ret;
1187
1188         if (data_off != *cur_off)
1189                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1190
1191         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1192
1193         if (data_off + ecc->size != oob_off)
1194                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1195
1196         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1197         if (ret)
1198                 return ret;
1199
1200         sunxi_nfc_randomizer_enable(mtd);
1201         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1202
1203         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1204                NFC_ACCESS_DIR | NFC_ECC_OP,
1205                nfc->regs + NFC_REG_CMD);
1206
1207         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1208         sunxi_nfc_randomizer_disable(mtd);
1209         if (ret)
1210                 return ret;
1211
1212         *cur_off = oob_off + ecc->bytes + 4;
1213
1214         return 0;
1215 }
1216
1217 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1218                                              u8 *oob, int *cur_off,
1219                                              int page)
1220 {
1221         struct nand_chip *nand = mtd_to_nand(mtd);
1222         struct nand_ecc_ctrl *ecc = &nand->ecc;
1223         int offset = ((ecc->bytes + 4) * ecc->steps);
1224         int len = mtd->oobsize - offset;
1225
1226         if (len <= 0)
1227                 return;
1228
1229         if (!cur_off || *cur_off != offset)
1230                 nand_change_write_column_op(nand, offset + mtd->writesize,
1231                                             NULL, 0, false);
1232
1233         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1234
1235         if (cur_off)
1236                 *cur_off = mtd->oobsize + mtd->writesize;
1237 }
1238
1239 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1240                                       struct nand_chip *chip, uint8_t *buf,
1241                                       int oob_required, int page)
1242 {
1243         struct nand_ecc_ctrl *ecc = &chip->ecc;
1244         unsigned int max_bitflips = 0;
1245         int ret, i, cur_off = 0;
1246         bool raw_mode = false;
1247
1248         nand_read_page_op(chip, page, 0, NULL, 0);
1249
1250         sunxi_nfc_hw_ecc_enable(mtd);
1251
1252         for (i = 0; i < ecc->steps; i++) {
1253                 int data_off = i * ecc->size;
1254                 int oob_off = i * (ecc->bytes + 4);
1255                 u8 *data = buf + data_off;
1256                 u8 *oob = chip->oob_poi + oob_off;
1257
1258                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1259                                                   oob_off + mtd->writesize,
1260                                                   &cur_off, &max_bitflips,
1261                                                   !i, oob_required, page);
1262                 if (ret < 0)
1263                         return ret;
1264                 else if (ret)
1265                         raw_mode = true;
1266         }
1267
1268         if (oob_required)
1269                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1270                                                 !raw_mode, page);
1271
1272         sunxi_nfc_hw_ecc_disable(mtd);
1273
1274         return max_bitflips;
1275 }
1276
1277 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1278                                           struct nand_chip *chip, u8 *buf,
1279                                           int oob_required, int page)
1280 {
1281         int ret;
1282
1283         nand_read_page_op(chip, page, 0, NULL, 0);
1284
1285         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1286                                                chip->ecc.steps);
1287         if (ret >= 0)
1288                 return ret;
1289
1290         /* Fallback to PIO mode */
1291         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1292 }
1293
1294 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1295                                          struct nand_chip *chip,
1296                                          u32 data_offs, u32 readlen,
1297                                          u8 *bufpoi, int page)
1298 {
1299         struct nand_ecc_ctrl *ecc = &chip->ecc;
1300         int ret, i, cur_off = 0;
1301         unsigned int max_bitflips = 0;
1302
1303         nand_read_page_op(chip, page, 0, NULL, 0);
1304
1305         sunxi_nfc_hw_ecc_enable(mtd);
1306
1307         for (i = data_offs / ecc->size;
1308              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1309                 int data_off = i * ecc->size;
1310                 int oob_off = i * (ecc->bytes + 4);
1311                 u8 *data = bufpoi + data_off;
1312                 u8 *oob = chip->oob_poi + oob_off;
1313
1314                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1315                                                   oob,
1316                                                   oob_off + mtd->writesize,
1317                                                   &cur_off, &max_bitflips, !i,
1318                                                   false, page);
1319                 if (ret < 0)
1320                         return ret;
1321         }
1322
1323         sunxi_nfc_hw_ecc_disable(mtd);
1324
1325         return max_bitflips;
1326 }
1327
1328 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1329                                              struct nand_chip *chip,
1330                                              u32 data_offs, u32 readlen,
1331                                              u8 *buf, int page)
1332 {
1333         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1334         int ret;
1335
1336         nand_read_page_op(chip, page, 0, NULL, 0);
1337
1338         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1339         if (ret >= 0)
1340                 return ret;
1341
1342         /* Fallback to PIO mode */
1343         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1344                                              buf, page);
1345 }
1346
1347 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1348                                        struct nand_chip *chip,
1349                                        const uint8_t *buf, int oob_required,
1350                                        int page)
1351 {
1352         struct nand_ecc_ctrl *ecc = &chip->ecc;
1353         int ret, i, cur_off = 0;
1354
1355         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1356
1357         sunxi_nfc_hw_ecc_enable(mtd);
1358
1359         for (i = 0; i < ecc->steps; i++) {
1360                 int data_off = i * ecc->size;
1361                 int oob_off = i * (ecc->bytes + 4);
1362                 const u8 *data = buf + data_off;
1363                 const u8 *oob = chip->oob_poi + oob_off;
1364
1365                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1366                                                    oob_off + mtd->writesize,
1367                                                    &cur_off, !i, page);
1368                 if (ret)
1369                         return ret;
1370         }
1371
1372         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1373                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1374                                                  &cur_off, page);
1375
1376         sunxi_nfc_hw_ecc_disable(mtd);
1377
1378         return nand_prog_page_end_op(chip);
1379 }
1380
1381 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1382                                           struct nand_chip *chip,
1383                                           u32 data_offs, u32 data_len,
1384                                           const u8 *buf, int oob_required,
1385                                           int page)
1386 {
1387         struct nand_ecc_ctrl *ecc = &chip->ecc;
1388         int ret, i, cur_off = 0;
1389
1390         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1391
1392         sunxi_nfc_hw_ecc_enable(mtd);
1393
1394         for (i = data_offs / ecc->size;
1395              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1396                 int data_off = i * ecc->size;
1397                 int oob_off = i * (ecc->bytes + 4);
1398                 const u8 *data = buf + data_off;
1399                 const u8 *oob = chip->oob_poi + oob_off;
1400
1401                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1402                                                    oob_off + mtd->writesize,
1403                                                    &cur_off, !i, page);
1404                 if (ret)
1405                         return ret;
1406         }
1407
1408         sunxi_nfc_hw_ecc_disable(mtd);
1409
1410         return nand_prog_page_end_op(chip);
1411 }
1412
1413 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1414                                            struct nand_chip *chip,
1415                                            const u8 *buf,
1416                                            int oob_required,
1417                                            int page)
1418 {
1419         struct nand_chip *nand = mtd_to_nand(mtd);
1420         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1421         struct nand_ecc_ctrl *ecc = &nand->ecc;
1422         struct scatterlist sg;
1423         int ret, i;
1424
1425         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1426         if (ret)
1427                 return ret;
1428
1429         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1430                                        DMA_TO_DEVICE, &sg);
1431         if (ret)
1432                 goto pio_fallback;
1433
1434         for (i = 0; i < ecc->steps; i++) {
1435                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1436
1437                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1438         }
1439
1440         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1441
1442         sunxi_nfc_hw_ecc_enable(mtd);
1443         sunxi_nfc_randomizer_config(mtd, page, false);
1444         sunxi_nfc_randomizer_enable(mtd);
1445
1446         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1447                nfc->regs + NFC_REG_RCMD_SET);
1448
1449         dma_async_issue_pending(nfc->dmac);
1450
1451         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1452                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1453                nfc->regs + NFC_REG_CMD);
1454
1455         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1456         if (ret)
1457                 dmaengine_terminate_all(nfc->dmac);
1458
1459         sunxi_nfc_randomizer_disable(mtd);
1460         sunxi_nfc_hw_ecc_disable(mtd);
1461
1462         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1463
1464         if (ret)
1465                 return ret;
1466
1467         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1468                 /* TODO: use DMA to transfer extra OOB bytes ? */
1469                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1470                                                  NULL, page);
1471
1472         return nand_prog_page_end_op(chip);
1473
1474 pio_fallback:
1475         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1476 }
1477
1478 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1479                                                struct nand_chip *chip,
1480                                                uint8_t *buf, int oob_required,
1481                                                int page)
1482 {
1483         struct nand_ecc_ctrl *ecc = &chip->ecc;
1484         unsigned int max_bitflips = 0;
1485         int ret, i, cur_off = 0;
1486         bool raw_mode = false;
1487
1488         nand_read_page_op(chip, page, 0, NULL, 0);
1489
1490         sunxi_nfc_hw_ecc_enable(mtd);
1491
1492         for (i = 0; i < ecc->steps; i++) {
1493                 int data_off = i * (ecc->size + ecc->bytes + 4);
1494                 int oob_off = data_off + ecc->size;
1495                 u8 *data = buf + (i * ecc->size);
1496                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1497
1498                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1499                                                   oob_off, &cur_off,
1500                                                   &max_bitflips, !i,
1501                                                   oob_required,
1502                                                   page);
1503                 if (ret < 0)
1504                         return ret;
1505                 else if (ret)
1506                         raw_mode = true;
1507         }
1508
1509         if (oob_required)
1510                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1511                                                 !raw_mode, page);
1512
1513         sunxi_nfc_hw_ecc_disable(mtd);
1514
1515         return max_bitflips;
1516 }
1517
1518 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1519                                                 struct nand_chip *chip,
1520                                                 const uint8_t *buf,
1521                                                 int oob_required, int page)
1522 {
1523         struct nand_ecc_ctrl *ecc = &chip->ecc;
1524         int ret, i, cur_off = 0;
1525
1526         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1527
1528         sunxi_nfc_hw_ecc_enable(mtd);
1529
1530         for (i = 0; i < ecc->steps; i++) {
1531                 int data_off = i * (ecc->size + ecc->bytes + 4);
1532                 int oob_off = data_off + ecc->size;
1533                 const u8 *data = buf + (i * ecc->size);
1534                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1535
1536                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1537                                                    oob, oob_off, &cur_off,
1538                                                    false, page);
1539                 if (ret)
1540                         return ret;
1541         }
1542
1543         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1544                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1545                                                  &cur_off, page);
1546
1547         sunxi_nfc_hw_ecc_disable(mtd);
1548
1549         return nand_prog_page_end_op(chip);
1550 }
1551
1552 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1553                                             struct nand_chip *chip,
1554                                             int page)
1555 {
1556         chip->pagebuf = -1;
1557
1558         return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page);
1559 }
1560
1561 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1562                                              struct nand_chip *chip,
1563                                              int page)
1564 {
1565         int ret;
1566
1567         chip->pagebuf = -1;
1568
1569         memset(chip->data_buf, 0xff, mtd->writesize);
1570         ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page);
1571         if (ret)
1572                 return ret;
1573
1574         /* Send command to program the OOB data */
1575         return nand_prog_page_end_op(chip);
1576 }
1577
1578 static const s32 tWB_lut[] = {6, 12, 16, 20};
1579 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1580
1581 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1582                 u32 clk_period)
1583 {
1584         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1585         int i;
1586
1587         for (i = 0; i < lut_size; i++) {
1588                 if (clk_cycles <= lut[i])
1589                         return i;
1590         }
1591
1592         /* Doesn't fit */
1593         return -EINVAL;
1594 }
1595
1596 #define sunxi_nand_lookup_timing(l, p, c) \
1597                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1598
1599 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline,
1600                                         const struct nand_data_interface *conf)
1601 {
1602         struct nand_chip *nand = mtd_to_nand(mtd);
1603         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1604         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1605         const struct nand_sdr_timings *timings;
1606         u32 min_clk_period = 0;
1607         s32 tWB, tADL, tWHR, tRHW, tCAD;
1608         long real_clk_rate;
1609
1610         timings = nand_get_sdr_timings(conf);
1611         if (IS_ERR(timings))
1612                 return -ENOTSUPP;
1613
1614         /* T1 <=> tCLS */
1615         if (timings->tCLS_min > min_clk_period)
1616                 min_clk_period = timings->tCLS_min;
1617
1618         /* T2 <=> tCLH */
1619         if (timings->tCLH_min > min_clk_period)
1620                 min_clk_period = timings->tCLH_min;
1621
1622         /* T3 <=> tCS */
1623         if (timings->tCS_min > min_clk_period)
1624                 min_clk_period = timings->tCS_min;
1625
1626         /* T4 <=> tCH */
1627         if (timings->tCH_min > min_clk_period)
1628                 min_clk_period = timings->tCH_min;
1629
1630         /* T5 <=> tWP */
1631         if (timings->tWP_min > min_clk_period)
1632                 min_clk_period = timings->tWP_min;
1633
1634         /* T6 <=> tWH */
1635         if (timings->tWH_min > min_clk_period)
1636                 min_clk_period = timings->tWH_min;
1637
1638         /* T7 <=> tALS */
1639         if (timings->tALS_min > min_clk_period)
1640                 min_clk_period = timings->tALS_min;
1641
1642         /* T8 <=> tDS */
1643         if (timings->tDS_min > min_clk_period)
1644                 min_clk_period = timings->tDS_min;
1645
1646         /* T9 <=> tDH */
1647         if (timings->tDH_min > min_clk_period)
1648                 min_clk_period = timings->tDH_min;
1649
1650         /* T10 <=> tRR */
1651         if (timings->tRR_min > (min_clk_period * 3))
1652                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1653
1654         /* T11 <=> tALH */
1655         if (timings->tALH_min > min_clk_period)
1656                 min_clk_period = timings->tALH_min;
1657
1658         /* T12 <=> tRP */
1659         if (timings->tRP_min > min_clk_period)
1660                 min_clk_period = timings->tRP_min;
1661
1662         /* T13 <=> tREH */
1663         if (timings->tREH_min > min_clk_period)
1664                 min_clk_period = timings->tREH_min;
1665
1666         /* T14 <=> tRC */
1667         if (timings->tRC_min > (min_clk_period * 2))
1668                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1669
1670         /* T15 <=> tWC */
1671         if (timings->tWC_min > (min_clk_period * 2))
1672                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1673
1674         /* T16 - T19 + tCAD */
1675         if (timings->tWB_max > (min_clk_period * 20))
1676                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1677
1678         if (timings->tADL_min > (min_clk_period * 32))
1679                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1680
1681         if (timings->tWHR_min > (min_clk_period * 32))
1682                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1683
1684         if (timings->tRHW_min > (min_clk_period * 20))
1685                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1686
1687         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1688                                         min_clk_period);
1689         if (tWB < 0) {
1690                 dev_err(nfc->dev, "unsupported tWB\n");
1691                 return tWB;
1692         }
1693
1694         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1695         if (tADL > 3) {
1696                 dev_err(nfc->dev, "unsupported tADL\n");
1697                 return -EINVAL;
1698         }
1699
1700         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1701         if (tWHR > 3) {
1702                 dev_err(nfc->dev, "unsupported tWHR\n");
1703                 return -EINVAL;
1704         }
1705
1706         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1707                                         min_clk_period);
1708         if (tRHW < 0) {
1709                 dev_err(nfc->dev, "unsupported tRHW\n");
1710                 return tRHW;
1711         }
1712
1713         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1714                 return 0;
1715
1716         /*
1717          * TODO: according to ONFI specs this value only applies for DDR NAND,
1718          * but Allwinner seems to set this to 0x7. Mimic them for now.
1719          */
1720         tCAD = 0x7;
1721
1722         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1723         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1724
1725         /* Convert min_clk_period from picoseconds to nanoseconds */
1726         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1727
1728         /*
1729          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1730          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1731          * This new formula was verified with a scope and validated by
1732          * Allwinner engineers.
1733          */
1734         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1735         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1736         if (real_clk_rate <= 0) {
1737                 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate);
1738                 return -EINVAL;
1739         }
1740
1741         /*
1742          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1743          * output cycle timings shall be used if the host drives tRC less than
1744          * 30 ns.
1745          */
1746         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1747         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1748                            NFC_TIMING_CTL_EDO : 0;
1749
1750         return 0;
1751 }
1752
1753 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1754                                     struct mtd_oob_region *oobregion)
1755 {
1756         struct nand_chip *nand = mtd_to_nand(mtd);
1757         struct nand_ecc_ctrl *ecc = &nand->ecc;
1758
1759         if (section >= ecc->steps)
1760                 return -ERANGE;
1761
1762         oobregion->offset = section * (ecc->bytes + 4) + 4;
1763         oobregion->length = ecc->bytes;
1764
1765         return 0;
1766 }
1767
1768 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1769                                      struct mtd_oob_region *oobregion)
1770 {
1771         struct nand_chip *nand = mtd_to_nand(mtd);
1772         struct nand_ecc_ctrl *ecc = &nand->ecc;
1773
1774         if (section > ecc->steps)
1775                 return -ERANGE;
1776
1777         /*
1778          * The first 2 bytes are used for BB markers, hence we
1779          * only have 2 bytes available in the first user data
1780          * section.
1781          */
1782         if (!section && ecc->mode == NAND_ECC_HW) {
1783                 oobregion->offset = 2;
1784                 oobregion->length = 2;
1785
1786                 return 0;
1787         }
1788
1789         oobregion->offset = section * (ecc->bytes + 4);
1790
1791         if (section < ecc->steps)
1792                 oobregion->length = 4;
1793         else
1794                 oobregion->offset = mtd->oobsize - oobregion->offset;
1795
1796         return 0;
1797 }
1798
1799 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1800         .ecc = sunxi_nand_ooblayout_ecc,
1801         .free = sunxi_nand_ooblayout_free,
1802 };
1803
1804 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1805                                               struct nand_ecc_ctrl *ecc,
1806                                               struct device_node *np)
1807 {
1808         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1809         struct nand_chip *nand = mtd_to_nand(mtd);
1810         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1811         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1812         struct sunxi_nand_hw_ecc *data;
1813         int nsectors;
1814         int ret;
1815         int i;
1816
1817         if (ecc->options & NAND_ECC_MAXIMIZE) {
1818                 int bytes;
1819
1820                 ecc->size = 1024;
1821                 nsectors = mtd->writesize / ecc->size;
1822
1823                 /* Reserve 2 bytes for the BBM */
1824                 bytes = (mtd->oobsize - 2) / nsectors;
1825
1826                 /* 4 non-ECC bytes are added before each ECC bytes section */
1827                 bytes -= 4;
1828
1829                 /* and bytes has to be even. */
1830                 if (bytes % 2)
1831                         bytes--;
1832
1833                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1834
1835                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1836                         if (strengths[i] > ecc->strength)
1837                                 break;
1838                 }
1839
1840                 if (!i)
1841                         ecc->strength = 0;
1842                 else
1843                         ecc->strength = strengths[i - 1];
1844         }
1845
1846         if (ecc->size != 512 && ecc->size != 1024)
1847                 return -EINVAL;
1848
1849         data = kzalloc(sizeof(*data), GFP_KERNEL);
1850         if (!data)
1851                 return -ENOMEM;
1852
1853         /* Prefer 1k ECC chunk over 512 ones */
1854         if (ecc->size == 512 && mtd->writesize > 512) {
1855                 ecc->size = 1024;
1856                 ecc->strength *= 2;
1857         }
1858
1859         /* Add ECC info retrieval from DT */
1860         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1861                 if (ecc->strength <= strengths[i]) {
1862                         /*
1863                          * Update ecc->strength value with the actual strength
1864                          * that will be used by the ECC engine.
1865                          */
1866                         ecc->strength = strengths[i];
1867                         break;
1868                 }
1869         }
1870
1871         if (i >= ARRAY_SIZE(strengths)) {
1872                 dev_err(nfc->dev, "unsupported strength\n");
1873                 ret = -ENOTSUPP;
1874                 goto err;
1875         }
1876
1877         data->mode = i;
1878
1879         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1880         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1881
1882         /* HW ECC always work with even numbers of ECC bytes */
1883         ecc->bytes = ALIGN(ecc->bytes, 2);
1884
1885         nsectors = mtd->writesize / ecc->size;
1886
1887         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1888                 ret = -EINVAL;
1889                 goto err;
1890         }
1891
1892         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1893         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1894         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1895         ecc->priv = data;
1896
1897         return 0;
1898
1899 err:
1900         kfree(data);
1901
1902         return ret;
1903 }
1904
1905 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1906 {
1907         kfree(ecc->priv);
1908 }
1909
1910 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1911                                        struct nand_ecc_ctrl *ecc,
1912                                        struct device_node *np)
1913 {
1914         struct nand_chip *nand = mtd_to_nand(mtd);
1915         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1916         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1917         int ret;
1918
1919         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1920         if (ret)
1921                 return ret;
1922
1923         if (nfc->dmac) {
1924                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1925                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1926                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1927                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1928         } else {
1929                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1930                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1931                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1932         }
1933
1934         /* TODO: support DMA for raw accesses and subpage write */
1935         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1936         ecc->read_oob_raw = nand_read_oob_std;
1937         ecc->write_oob_raw = nand_write_oob_std;
1938
1939         return 0;
1940 }
1941
1942 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1943                                                 struct nand_ecc_ctrl *ecc,
1944                                                 struct device_node *np)
1945 {
1946         int ret;
1947
1948         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1949         if (ret)
1950                 return ret;
1951
1952         ecc->prepad = 4;
1953         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1954         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1955         ecc->read_oob_raw = nand_read_oob_syndrome;
1956         ecc->write_oob_raw = nand_write_oob_syndrome;
1957
1958         return 0;
1959 }
1960
1961 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1962 {
1963         switch (ecc->mode) {
1964         case NAND_ECC_HW:
1965         case NAND_ECC_HW_SYNDROME:
1966                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1967                 break;
1968         case NAND_ECC_NONE:
1969         default:
1970                 break;
1971         }
1972 }
1973
1974 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1975                                struct device_node *np)
1976 {
1977         struct nand_chip *nand = mtd_to_nand(mtd);
1978         int ret;
1979
1980         if (!ecc->size) {
1981                 ecc->size = nand->ecc_step_ds;
1982                 ecc->strength = nand->ecc_strength_ds;
1983         }
1984
1985         if (!ecc->size || !ecc->strength)
1986                 return -EINVAL;
1987
1988         switch (ecc->mode) {
1989         case NAND_ECC_HW:
1990                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1991                 if (ret)
1992                         return ret;
1993                 break;
1994         case NAND_ECC_HW_SYNDROME:
1995                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1996                 if (ret)
1997                         return ret;
1998                 break;
1999         case NAND_ECC_NONE:
2000         case NAND_ECC_SOFT:
2001                 break;
2002         default:
2003                 return -EINVAL;
2004         }
2005
2006         return 0;
2007 }
2008
2009 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
2010                                 struct device_node *np)
2011 {
2012         struct sunxi_nand_chip *chip;
2013         struct mtd_info *mtd;
2014         struct nand_chip *nand;
2015         int nsels;
2016         int ret;
2017         int i;
2018         u32 tmp;
2019
2020         if (!of_get_property(np, "reg", &nsels))
2021                 return -EINVAL;
2022
2023         nsels /= sizeof(u32);
2024         if (!nsels) {
2025                 dev_err(dev, "invalid reg property size\n");
2026                 return -EINVAL;
2027         }
2028
2029         chip = devm_kzalloc(dev,
2030                             sizeof(*chip) +
2031                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2032                             GFP_KERNEL);
2033         if (!chip) {
2034                 dev_err(dev, "could not allocate chip\n");
2035                 return -ENOMEM;
2036         }
2037
2038         chip->nsels = nsels;
2039         chip->selected = -1;
2040
2041         for (i = 0; i < nsels; i++) {
2042                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2043                 if (ret) {
2044                         dev_err(dev, "could not retrieve reg property: %d\n",
2045                                 ret);
2046                         return ret;
2047                 }
2048
2049                 if (tmp > NFC_MAX_CS) {
2050                         dev_err(dev,
2051                                 "invalid reg value: %u (max CS = 7)\n",
2052                                 tmp);
2053                         return -EINVAL;
2054                 }
2055
2056                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2057                         dev_err(dev, "CS %d already assigned\n", tmp);
2058                         return -EINVAL;
2059                 }
2060
2061                 chip->sels[i].cs = tmp;
2062
2063                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2064                     tmp < 2) {
2065                         chip->sels[i].rb.type = RB_NATIVE;
2066                         chip->sels[i].rb.info.nativeid = tmp;
2067                 } else {
2068                         ret = of_get_named_gpio(np, "rb-gpios", i);
2069                         if (ret >= 0) {
2070                                 tmp = ret;
2071                                 chip->sels[i].rb.type = RB_GPIO;
2072                                 chip->sels[i].rb.info.gpio = tmp;
2073                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2074                                 if (ret)
2075                                         return ret;
2076
2077                                 ret = gpio_direction_input(tmp);
2078                                 if (ret)
2079                                         return ret;
2080                         } else {
2081                                 chip->sels[i].rb.type = RB_NONE;
2082                         }
2083                 }
2084         }
2085
2086         nand = &chip->nand;
2087         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2088         nand->chip_delay = 200;
2089         nand->controller = &nfc->controller;
2090         /*
2091          * Set the ECC mode to the default value in case nothing is specified
2092          * in the DT.
2093          */
2094         nand->ecc.mode = NAND_ECC_HW;
2095         nand_set_flash_node(nand, np);
2096         nand->select_chip = sunxi_nfc_select_chip;
2097         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2098         nand->read_buf = sunxi_nfc_read_buf;
2099         nand->write_buf = sunxi_nfc_write_buf;
2100         nand->read_byte = sunxi_nfc_read_byte;
2101         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2102
2103         mtd = nand_to_mtd(nand);
2104         mtd->dev.parent = dev;
2105
2106         ret = nand_scan_ident(mtd, nsels, NULL);
2107         if (ret)
2108                 return ret;
2109
2110         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2111                 nand->bbt_options |= NAND_BBT_NO_OOB;
2112
2113         if (nand->options & NAND_NEED_SCRAMBLING)
2114                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2115
2116         nand->options |= NAND_SUBPAGE_READ;
2117
2118         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2119         if (ret) {
2120                 dev_err(dev, "ECC init failed: %d\n", ret);
2121                 return ret;
2122         }
2123
2124         ret = nand_scan_tail(mtd);
2125         if (ret) {
2126                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2127                 return ret;
2128         }
2129
2130         ret = mtd_device_register(mtd, NULL, 0);
2131         if (ret) {
2132                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2133                 nand_release(mtd);
2134                 return ret;
2135         }
2136
2137         list_add_tail(&chip->node, &nfc->chips);
2138
2139         return 0;
2140 }
2141
2142 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2143 {
2144         struct device_node *np = dev->of_node;
2145         struct device_node *nand_np;
2146         int nchips = of_get_child_count(np);
2147         int ret;
2148
2149         if (nchips > 8) {
2150                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2151                 return -EINVAL;
2152         }
2153
2154         for_each_child_of_node(np, nand_np) {
2155                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2156                 if (ret) {
2157                         of_node_put(nand_np);
2158                         return ret;
2159                 }
2160         }
2161
2162         return 0;
2163 }
2164
2165 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2166 {
2167         struct sunxi_nand_chip *chip;
2168
2169         while (!list_empty(&nfc->chips)) {
2170                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2171                                         node);
2172                 nand_release(nand_to_mtd(&chip->nand));
2173                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2174                 list_del(&chip->node);
2175         }
2176 }
2177
2178 static int sunxi_nfc_probe(struct platform_device *pdev)
2179 {
2180         struct device *dev = &pdev->dev;
2181         struct resource *r;
2182         struct sunxi_nfc *nfc;
2183         int irq;
2184         int ret;
2185
2186         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2187         if (!nfc)
2188                 return -ENOMEM;
2189
2190         nfc->dev = dev;
2191         nand_hw_control_init(&nfc->controller);
2192         INIT_LIST_HEAD(&nfc->chips);
2193
2194         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2195         nfc->regs = devm_ioremap_resource(dev, r);
2196         if (IS_ERR(nfc->regs))
2197                 return PTR_ERR(nfc->regs);
2198
2199         irq = platform_get_irq(pdev, 0);
2200         if (irq < 0) {
2201                 dev_err(dev, "failed to retrieve irq\n");
2202                 return irq;
2203         }
2204
2205         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2206         if (IS_ERR(nfc->ahb_clk)) {
2207                 dev_err(dev, "failed to retrieve ahb clk\n");
2208                 return PTR_ERR(nfc->ahb_clk);
2209         }
2210
2211         ret = clk_prepare_enable(nfc->ahb_clk);
2212         if (ret)
2213                 return ret;
2214
2215         nfc->mod_clk = devm_clk_get(dev, "mod");
2216         if (IS_ERR(nfc->mod_clk)) {
2217                 dev_err(dev, "failed to retrieve mod clk\n");
2218                 ret = PTR_ERR(nfc->mod_clk);
2219                 goto out_ahb_clk_unprepare;
2220         }
2221
2222         ret = clk_prepare_enable(nfc->mod_clk);
2223         if (ret)
2224                 goto out_ahb_clk_unprepare;
2225
2226         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2227         if (IS_ERR(nfc->reset)) {
2228                 ret = PTR_ERR(nfc->reset);
2229                 goto out_mod_clk_unprepare;
2230         }
2231
2232         ret = reset_control_deassert(nfc->reset);
2233         if (ret) {
2234                 dev_err(dev, "reset err %d\n", ret);
2235                 goto out_mod_clk_unprepare;
2236         }
2237
2238         ret = sunxi_nfc_rst(nfc);
2239         if (ret)
2240                 goto out_ahb_reset_reassert;
2241
2242         writel(0, nfc->regs + NFC_REG_INT);
2243         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2244                                0, "sunxi-nand", nfc);
2245         if (ret)
2246                 goto out_ahb_reset_reassert;
2247
2248         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2249         if (nfc->dmac) {
2250                 struct dma_slave_config dmac_cfg = { };
2251
2252                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2253                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2254                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2255                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2256                 dmac_cfg.src_maxburst = 4;
2257                 dmac_cfg.dst_maxburst = 4;
2258                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2259         } else {
2260                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2261         }
2262
2263         platform_set_drvdata(pdev, nfc);
2264
2265         ret = sunxi_nand_chips_init(dev, nfc);
2266         if (ret) {
2267                 dev_err(dev, "failed to init nand chips\n");
2268                 goto out_release_dmac;
2269         }
2270
2271         return 0;
2272
2273 out_release_dmac:
2274         if (nfc->dmac)
2275                 dma_release_channel(nfc->dmac);
2276 out_ahb_reset_reassert:
2277         reset_control_assert(nfc->reset);
2278 out_mod_clk_unprepare:
2279         clk_disable_unprepare(nfc->mod_clk);
2280 out_ahb_clk_unprepare:
2281         clk_disable_unprepare(nfc->ahb_clk);
2282
2283         return ret;
2284 }
2285
2286 static int sunxi_nfc_remove(struct platform_device *pdev)
2287 {
2288         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2289
2290         sunxi_nand_chips_cleanup(nfc);
2291
2292         reset_control_assert(nfc->reset);
2293
2294         if (nfc->dmac)
2295                 dma_release_channel(nfc->dmac);
2296         clk_disable_unprepare(nfc->mod_clk);
2297         clk_disable_unprepare(nfc->ahb_clk);
2298
2299         return 0;
2300 }
2301
2302 static const struct of_device_id sunxi_nfc_ids[] = {
2303         { .compatible = "allwinner,sun4i-a10-nand" },
2304         { /* sentinel */ }
2305 };
2306 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2307
2308 static struct platform_driver sunxi_nfc_driver = {
2309         .driver = {
2310                 .name = "sunxi_nand",
2311                 .of_match_table = sunxi_nfc_ids,
2312         },
2313         .probe = sunxi_nfc_probe,
2314         .remove = sunxi_nfc_remove,
2315 };
2316 module_platform_driver(sunxi_nfc_driver);
2317
2318 MODULE_LICENSE("GPL v2");
2319 MODULE_AUTHOR("Boris BREZILLON");
2320 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2321 MODULE_ALIAS("platform:sunxi_nand");