drivers/mtd/nand/raw/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/mtd/mtd.h>
  33 #include <linux/mtd/rawnand.h>
  34 #include <linux/mtd/partitions.h>
  35 #include <linux/clk.h>
  36 #include <linux/delay.h>
  37 #include <linux/dmaengine.h>
  38 #include <linux/interrupt.h>
  39 #include <linux/iopoll.h>
  40 #include <linux/reset.h>
  41
  42 #define NFC_REG_CTL             0x0000
  43 #define NFC_REG_ST              0x0004
  44 #define NFC_REG_INT             0x0008
  45 #define NFC_REG_TIMING_CTL      0x000C
  46 #define NFC_REG_TIMING_CFG      0x0010
  47 #define NFC_REG_ADDR_LOW        0x0014
  48 #define NFC_REG_ADDR_HIGH       0x0018
  49 #define NFC_REG_SECTOR_NUM      0x001C
  50 #define NFC_REG_CNT             0x0020
  51 #define NFC_REG_CMD             0x0024
  52 #define NFC_REG_RCMD_SET        0x0028
  53 #define NFC_REG_WCMD_SET        0x002C
  54 #define NFC_REG_IO_DATA         0x0030
  55 #define NFC_REG_ECC_CTL         0x0034
  56 #define NFC_REG_ECC_ST          0x0038
  57 #define NFC_REG_DEBUG           0x003C
  58 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  59 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  60 #define NFC_REG_SPARE_AREA      0x00A0
  61 #define NFC_REG_PAT_ID          0x00A4
  62 #define NFC_RAM0_BASE           0x0400
  63 #define NFC_RAM1_BASE           0x0800
  64
  65 /* define bit use in NFC_CTL */
  66 #define NFC_EN                  BIT(0)
  67 #define NFC_RESET               BIT(1)
  68 #define NFC_BUS_WIDTH_MSK       BIT(2)
  69 #define NFC_BUS_WIDTH_8         (0 << 2)
  70 #define NFC_BUS_WIDTH_16        (1 << 2)
  71 #define NFC_RB_SEL_MSK          BIT(3)
  72 #define NFC_RB_SEL(x)           ((x) << 3)
  73 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  74 #define NFC_CE_SEL(x)           ((x) << 24)
  75 #define NFC_CE_CTL              BIT(6)
  76 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  77 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  78 #define NFC_SAM                 BIT(12)
  79 #define NFC_RAM_METHOD          BIT(14)
  80 #define NFC_DEBUG_CTL           BIT(31)
  81
  82 /* define bit use in NFC_ST */
  83 #define NFC_RB_B2R              BIT(0)
  84 #define NFC_CMD_INT_FLAG        BIT(1)
  85 #define NFC_DMA_INT_FLAG        BIT(2)
  86 #define NFC_CMD_FIFO_STATUS     BIT(3)
  87 #define NFC_STA                 BIT(4)
  88 #define NFC_NATCH_INT_FLAG      BIT(5)
  89 #define NFC_RB_STATE(x)         BIT(x + 8)
  90
  91 /* define bit use in NFC_INT */
  92 #define NFC_B2R_INT_ENABLE      BIT(0)
  93 #define NFC_CMD_INT_ENABLE      BIT(1)
  94 #define NFC_DMA_INT_ENABLE      BIT(2)
  95 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  96                                  NFC_CMD_INT_ENABLE | \
  97                                  NFC_DMA_INT_ENABLE)
  98
  99 /* define bit use in NFC_TIMING_CTL */
 100 #define NFC_TIMING_CTL_EDO      BIT(8)
 101
 102 /* define NFC_TIMING_CFG register layout */
 103 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 104         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 105         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 106         (((tCAD) & 0x7) << 8))
 107
 108 /* define bit use in NFC_CMD */
 109 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 110 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 111 #define NFC_CMD(x)              (x)
 112 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 113 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 114 #define NFC_SEND_ADR            BIT(19)
 115 #define NFC_ACCESS_DIR          BIT(20)
 116 #define NFC_DATA_TRANS          BIT(21)
 117 #define NFC_SEND_CMD1           BIT(22)
 118 #define NFC_WAIT_FLAG           BIT(23)
 119 #define NFC_SEND_CMD2           BIT(24)
 120 #define NFC_SEQ                 BIT(25)
 121 #define NFC_DATA_SWAP_METHOD    BIT(26)
 122 #define NFC_ROW_AUTO_INC        BIT(27)
 123 #define NFC_SEND_CMD3           BIT(28)
 124 #define NFC_SEND_CMD4           BIT(29)
 125 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 126 #define NFC_NORMAL_OP           (0 << 30)
 127 #define NFC_ECC_OP              (1 << 30)
 128 #define NFC_PAGE_OP             (2U << 30)
 129
 130 /* define bit use in NFC_RCMD_SET */
 131 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 132 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 133 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 134
 135 /* define bit use in NFC_WCMD_SET */
 136 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 137 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 138 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 139 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 140
 141 /* define bit use in NFC_ECC_CTL */
 142 #define NFC_ECC_EN              BIT(0)
 143 #define NFC_ECC_PIPELINE        BIT(3)
 144 #define NFC_ECC_EXCEPTION       BIT(4)
 145 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 146 #define NFC_ECC_BLOCK_512       BIT(5)
 147 #define NFC_RANDOM_EN           BIT(9)
 148 #define NFC_RANDOM_DIRECTION    BIT(10)
 149 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 150 #define NFC_ECC_MODE(x)         ((x) << 12)
 151 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 152 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 153
 154 /* define bit use in NFC_ECC_ST */
 155 #define NFC_ECC_ERR(x)          BIT(x)
 156 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 157 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 158 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 159
 160 #define NFC_DEFAULT_TIMEOUT_MS  1000
 161
 162 #define NFC_SRAM_SIZE           1024
 163
 164 #define NFC_MAX_CS              7
 165
 166 /*
 167  * Chip Select structure: stores information related to NAND Chip Select
 168  *
 169  * @cs:         the NAND CS id used to communicate with a NAND Chip
 170  * @rb:         the Ready/Busy pin ID. -1 means no R/B pin connected to the
 171  *              NFC
 172  */
 173 struct sunxi_nand_chip_sel {
 174         u8 cs;
 175         s8 rb;
 176 };
 177
 178 /*
 179  * sunxi HW ECC infos: stores information related to HW ECC support
 180  *
 181  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 182  */
 183 struct sunxi_nand_hw_ecc {
 184         int mode;
 185 };
 186
 187 /*
 188  * NAND chip structure: stores NAND chip device related information
 189  *
 190  * @node:               used to store NAND chips into a list
 191  * @nand:               base NAND chip structure
 192  * @mtd:                base MTD structure
 193  * @clk_rate:           clk_rate required for this NAND chip
 194  * @timing_cfg          TIMING_CFG register value for this NAND chip
 195  * @selected:           current active CS
 196  * @nsels:              number of CS lines required by the NAND chip
 197  * @sels:               array of CS lines descriptions
 198  */
 199 struct sunxi_nand_chip {
 200         struct list_head node;
 201         struct nand_chip nand;
 202         unsigned long clk_rate;
 203         u32 timing_cfg;
 204         u32 timing_ctl;
 205         int selected;
 206         int addr_cycles;
 207         u32 addr[2];
 208         int cmd_cycles;
 209         u8 cmd[2];
 210         int nsels;
 211         struct sunxi_nand_chip_sel sels[0];
 212 };
 213
 214 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 215 {
 216         return container_of(nand, struct sunxi_nand_chip, nand);
 217 }
 218
 219 /*
 220  * NAND Controller structure: stores sunxi NAND controller information
 221  *
 222  * @controller:         base controller structure
 223  * @dev:                parent device (used to print error messages)
 224  * @regs:               NAND controller registers
 225  * @ahb_clk:            NAND Controller AHB clock
 226  * @mod_clk:            NAND Controller mod clock
 227  * @assigned_cs:        bitmask describing already assigned CS lines
 228  * @clk_rate:           NAND controller current clock rate
 229  * @chips:              a list containing all the NAND chips attached to
 230  *                      this NAND controller
 231  * @complete:           a completion object used to wait for NAND
 232  *                      controller events
 233  */
 234 struct sunxi_nfc {
 235         struct nand_controller controller;
 236         struct device *dev;
 237         void __iomem *regs;
 238         struct clk *ahb_clk;
 239         struct clk *mod_clk;
 240         struct reset_control *reset;
 241         unsigned long assigned_cs;
 242         unsigned long clk_rate;
 243         struct list_head chips;
 244         struct completion complete;
 245         struct dma_chan *dmac;
 246 };
 247
 248 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 249 {
 250         return container_of(ctrl, struct sunxi_nfc, controller);
 251 }
 252
 253 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 254 {
 255         struct sunxi_nfc *nfc = dev_id;
 256         u32 st = readl(nfc->regs + NFC_REG_ST);
 257         u32 ien = readl(nfc->regs + NFC_REG_INT);
 258
 259         if (!(ien & st))
 260                 return IRQ_NONE;
 261
 262         if ((ien & st) == ien)
 263                 complete(&nfc->complete);
 264
 265         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 266         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 267
 268         return IRQ_HANDLED;
 269 }
 270
 271 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 272                                  bool use_polling, unsigned int timeout_ms)
 273 {
 274         int ret;
 275
 276         if (events & ~NFC_INT_MASK)
 277                 return -EINVAL;
 278
 279         if (!timeout_ms)
 280                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 281
 282         if (!use_polling) {
 283                 init_completion(&nfc->complete);
 284
 285                 writel(events, nfc->regs + NFC_REG_INT);
 286
 287                 ret = wait_for_completion_timeout(&nfc->complete,
 288                                                 msecs_to_jiffies(timeout_ms));
 289                 if (!ret)
 290                         ret = -ETIMEDOUT;
 291                 else
 292                         ret = 0;
 293
 294                 writel(0, nfc->regs + NFC_REG_INT);
 295         } else {
 296                 u32 status;
 297
 298                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 299                                          (status & events) == events, 1,
 300                                          timeout_ms * 1000);
 301         }
 302
 303         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 304
 305         if (ret)
 306                 dev_err(nfc->dev, "wait interrupt timedout\n");
 307
 308         return ret;
 309 }
 310
 311 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 312 {
 313         u32 status;
 314         int ret;
 315
 316         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 317                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 318                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 319         if (ret)
 320                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 321
 322         return ret;
 323 }
 324
 325 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 326 {
 327         u32 ctl;
 328         int ret;
 329
 330         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 331         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 332
 333         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 334                                  !(ctl & NFC_RESET), 1,
 335                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 336         if (ret)
 337                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 338
 339         return ret;
 340 }
 341
 342 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 343                                     int chunksize, int nchunks,
 344                                     enum dma_data_direction ddir,
 345                                     struct scatterlist *sg)
 346 {
 347         struct nand_chip *nand = mtd_to_nand(mtd);
 348         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 349         struct dma_async_tx_descriptor *dmad;
 350         enum dma_transfer_direction tdir;
 351         dma_cookie_t dmat;
 352         int ret;
 353
 354         if (ddir == DMA_FROM_DEVICE)
 355                 tdir = DMA_DEV_TO_MEM;
 356         else
 357                 tdir = DMA_MEM_TO_DEV;
 358
 359         sg_init_one(sg, buf, nchunks * chunksize);
 360         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 361         if (!ret)
 362                 return -ENOMEM;
 363
 364         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 365         if (!dmad) {
 366                 ret = -EINVAL;
 367                 goto err_unmap_buf;
 368         }
 369
 370         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 371                nfc->regs + NFC_REG_CTL);
 372         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 373         writel(chunksize, nfc->regs + NFC_REG_CNT);
 374         dmat = dmaengine_submit(dmad);
 375
 376         ret = dma_submit_error(dmat);
 377         if (ret)
 378                 goto err_clr_dma_flag;
 379
 380         return 0;
 381
 382 err_clr_dma_flag:
 383         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 384                nfc->regs + NFC_REG_CTL);
 385
 386 err_unmap_buf:
 387         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 388         return ret;
 389 }
 390
 391 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 392                                      enum dma_data_direction ddir,
 393                                      struct scatterlist *sg)
 394 {
 395         struct nand_chip *nand = mtd_to_nand(mtd);
 396         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 397
 398         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 399         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 400                nfc->regs + NFC_REG_CTL);
 401 }
 402
 403 static int sunxi_nfc_dev_ready(struct nand_chip *nand)
 404 {
 405         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 406         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 407         u32 mask;
 408
 409         if (sunxi_nand->selected < 0)
 410                 return 0;
 411
 412         if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) {
 413                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 414                 return 0;
 415         }
 416
 417         mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb);
 418
 419         return !!(readl(nfc->regs + NFC_REG_ST) & mask);
 420 }
 421
 422 static void sunxi_nfc_select_chip(struct nand_chip *nand, int chip)
 423 {
 424         struct mtd_info *mtd = nand_to_mtd(nand);
 425         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 426         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 427         struct sunxi_nand_chip_sel *sel;
 428         u32 ctl;
 429
 430         if (chip > 0 && chip >= sunxi_nand->nsels)
 431                 return;
 432
 433         if (chip == sunxi_nand->selected)
 434                 return;
 435
 436         ctl = readl(nfc->regs + NFC_REG_CTL) &
 437               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 438
 439         if (chip >= 0) {
 440                 sel = &sunxi_nand->sels[chip];
 441
 442                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 443                        NFC_PAGE_SHIFT(nand->page_shift);
 444                 if (sel->rb < 0) {
 445                         nand->legacy.dev_ready = NULL;
 446                 } else {
 447                         nand->legacy.dev_ready = sunxi_nfc_dev_ready;
 448                         ctl |= NFC_RB_SEL(sel->rb);
 449                 }
 450
 451                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 452
 453                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 454                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 455                         nfc->clk_rate = sunxi_nand->clk_rate;
 456                 }
 457         }
 458
 459         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 460         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 461         writel(ctl, nfc->regs + NFC_REG_CTL);
 462
 463         sunxi_nand->selected = chip;
 464 }
 465
 466 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
 467 {
 468         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 469         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 470         int ret;
 471         int cnt;
 472         int offs = 0;
 473         u32 tmp;
 474
 475         while (len > offs) {
 476                 bool poll = false;
 477
 478                 cnt = min(len - offs, NFC_SRAM_SIZE);
 479
 480                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 481                 if (ret)
 482                         break;
 483
 484                 writel(cnt, nfc->regs + NFC_REG_CNT);
 485                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 486                 writel(tmp, nfc->regs + NFC_REG_CMD);
 487
 488                 /* Arbitrary limit for polling mode */
 489                 if (cnt < 64)
 490                         poll = true;
 491
 492                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 493                 if (ret)
 494                         break;
 495
 496                 if (buf)
 497                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 498                                       cnt);
 499                 offs += cnt;
 500         }
 501 }
 502
 503 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
 504                                 int len)
 505 {
 506         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 507         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 508         int ret;
 509         int cnt;
 510         int offs = 0;
 511         u32 tmp;
 512
 513         while (len > offs) {
 514                 bool poll = false;
 515
 516                 cnt = min(len - offs, NFC_SRAM_SIZE);
 517
 518                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 519                 if (ret)
 520                         break;
 521
 522                 writel(cnt, nfc->regs + NFC_REG_CNT);
 523                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 524                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 525                       NFC_ACCESS_DIR;
 526                 writel(tmp, nfc->regs + NFC_REG_CMD);
 527
 528                 /* Arbitrary limit for polling mode */
 529                 if (cnt < 64)
 530                         poll = true;
 531
 532                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 533                 if (ret)
 534                         break;
 535
 536                 offs += cnt;
 537         }
 538 }
 539
 540 static uint8_t sunxi_nfc_read_byte(struct nand_chip *nand)
 541 {
 542         uint8_t ret = 0;
 543
 544         sunxi_nfc_read_buf(nand, &ret, 1);
 545
 546         return ret;
 547 }
 548
 549 static void sunxi_nfc_cmd_ctrl(struct nand_chip *nand, int dat,
 550                                unsigned int ctrl)
 551 {
 552         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 553         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 554         int ret;
 555
 556         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 557             !(ctrl & (NAND_CLE | NAND_ALE))) {
 558                 u32 cmd = 0;
 559
 560                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 561                         return;
 562
 563                 if (sunxi_nand->cmd_cycles--)
 564                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 565
 566                 if (sunxi_nand->cmd_cycles--) {
 567                         cmd |= NFC_SEND_CMD2;
 568                         writel(sunxi_nand->cmd[1],
 569                                nfc->regs + NFC_REG_RCMD_SET);
 570                 }
 571
 572                 sunxi_nand->cmd_cycles = 0;
 573
 574                 if (sunxi_nand->addr_cycles) {
 575                         cmd |= NFC_SEND_ADR |
 576                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 577                         writel(sunxi_nand->addr[0],
 578                                nfc->regs + NFC_REG_ADDR_LOW);
 579                 }
 580
 581                 if (sunxi_nand->addr_cycles > 4)
 582                         writel(sunxi_nand->addr[1],
 583                                nfc->regs + NFC_REG_ADDR_HIGH);
 584
 585                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 586                 if (ret)
 587                         return;
 588
 589                 writel(cmd, nfc->regs + NFC_REG_CMD);
 590                 sunxi_nand->addr[0] = 0;
 591                 sunxi_nand->addr[1] = 0;
 592                 sunxi_nand->addr_cycles = 0;
 593                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 594         }
 595
 596         if (ctrl & NAND_CLE) {
 597                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 598         } else if (ctrl & NAND_ALE) {
 599                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 600                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 601                 sunxi_nand->addr_cycles++;
 602         }
 603 }
 604
 605 /* These seed values have been extracted from Allwinner's BSP */
 606 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 607         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 608         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 609         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 610         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 611         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 612         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 613         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 614         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 615         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 616         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 617         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 618         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 619         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 620         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 621         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 622         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 623 };
 624
 625 /*
 626  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 627  * have been generated using
 628  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 629  * the randomizer engine does internally before de/scrambling OOB data.
 630  *
 631  * Those tables are statically defined to avoid calculating randomizer state
 632  * at runtime.
 633  */
 634 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 635         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 636         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 637         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 638         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 639         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 640         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 641         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 642         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 643         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 644         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 645         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 646         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 647         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 648         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 649         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 650         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 651 };
 652
 653 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 654         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 655         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 656         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 657         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 658         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 659         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 660         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 661         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 662         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 663         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 664         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 665         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 666         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 667         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 668         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 669         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 670 };
 671
 672 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 673 {
 674         state &= 0x7fff;
 675
 676         /*
 677          * This loop is just a simple implementation of a Fibonacci LFSR using
 678          * the x16 + x15 + 1 polynomial.
 679          */
 680         while (count--)
 681                 state = ((state >> 1) |
 682                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 683
 684         return state;
 685 }
 686
 687 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 688 {
 689         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 690         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 691
 692         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 693                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 694
 695         if (ecc) {
 696                 if (mtd->ecc_step_size == 512)
 697                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 698                 else
 699                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 700         }
 701
 702         return seeds[page % mod];
 703 }
 704
 705 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 706                                         int page, bool ecc)
 707 {
 708         struct nand_chip *nand = mtd_to_nand(mtd);
 709         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 710         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 711         u16 state;
 712
 713         if (!(nand->options & NAND_NEED_SCRAMBLING))
 714                 return;
 715
 716         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 717         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 718         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 719         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 720 }
 721
 722 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 723 {
 724         struct nand_chip *nand = mtd_to_nand(mtd);
 725         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 726
 727         if (!(nand->options & NAND_NEED_SCRAMBLING))
 728                 return;
 729
 730         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 731                nfc->regs + NFC_REG_ECC_CTL);
 732 }
 733
 734 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 735 {
 736         struct nand_chip *nand = mtd_to_nand(mtd);
 737         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 738
 739         if (!(nand->options & NAND_NEED_SCRAMBLING))
 740                 return;
 741
 742         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 743                nfc->regs + NFC_REG_ECC_CTL);
 744 }
 745
 746 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 747 {
 748         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 749
 750         bbm[0] ^= state;
 751         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 752 }
 753
 754 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 755                                            const uint8_t *buf, int len,
 756                                            bool ecc, int page)
 757 {
 758         sunxi_nfc_randomizer_config(mtd, page, ecc);
 759         sunxi_nfc_randomizer_enable(mtd);
 760         sunxi_nfc_write_buf(mtd_to_nand(mtd), buf, len);
 761         sunxi_nfc_randomizer_disable(mtd);
 762 }
 763
 764 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 765                                           int len, bool ecc, int page)
 766 {
 767         sunxi_nfc_randomizer_config(mtd, page, ecc);
 768         sunxi_nfc_randomizer_enable(mtd);
 769         sunxi_nfc_read_buf(mtd_to_nand(mtd), buf, len);
 770         sunxi_nfc_randomizer_disable(mtd);
 771 }
 772
 773 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 774 {
 775         struct nand_chip *nand = mtd_to_nand(mtd);
 776         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 777         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 778         u32 ecc_ctl;
 779
 780         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 781         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 782                      NFC_ECC_BLOCK_SIZE_MSK);
 783         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 784                    NFC_ECC_PIPELINE;
 785
 786         if (nand->ecc.size == 512)
 787                 ecc_ctl |= NFC_ECC_BLOCK_512;
 788
 789         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 790 }
 791
 792 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 793 {
 794         struct nand_chip *nand = mtd_to_nand(mtd);
 795         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 796
 797         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 798                nfc->regs + NFC_REG_ECC_CTL);
 799 }
 800
 801 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 802 {
 803         buf[0] = user_data;
 804         buf[1] = user_data >> 8;
 805         buf[2] = user_data >> 16;
 806         buf[3] = user_data >> 24;
 807 }
 808
 809 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 810 {
 811         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 812 }
 813
 814 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 815                                                 int step, bool bbm, int page)
 816 {
 817         struct nand_chip *nand = mtd_to_nand(mtd);
 818         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 819
 820         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 821                                    oob);
 822
 823         /* De-randomize the Bad Block Marker. */
 824         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 825                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 826 }
 827
 828 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 829                                                 const u8 *oob, int step,
 830                                                 bool bbm, int page)
 831 {
 832         struct nand_chip *nand = mtd_to_nand(mtd);
 833         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 834         u8 user_data[4];
 835
 836         /* Randomize the Bad Block Marker. */
 837         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 838                 memcpy(user_data, oob, sizeof(user_data));
 839                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 840                 oob = user_data;
 841         }
 842
 843         writel(sunxi_nfc_buf_to_user_data(oob),
 844                nfc->regs + NFC_REG_USER_DATA(step));
 845 }
 846
 847 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 848                                           unsigned int *max_bitflips, int ret)
 849 {
 850         if (ret < 0) {
 851                 mtd->ecc_stats.failed++;
 852         } else {
 853                 mtd->ecc_stats.corrected += ret;
 854                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 855         }
 856 }
 857
 858 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 859                                     int step, u32 status, bool *erased)
 860 {
 861         struct nand_chip *nand = mtd_to_nand(mtd);
 862         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 863         struct nand_ecc_ctrl *ecc = &nand->ecc;
 864         u32 tmp;
 865
 866         *erased = false;
 867
 868         if (status & NFC_ECC_ERR(step))
 869                 return -EBADMSG;
 870
 871         if (status & NFC_ECC_PAT_FOUND(step)) {
 872                 u8 pattern;
 873
 874                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 875                         pattern = 0x0;
 876                 } else {
 877                         pattern = 0xff;
 878                         *erased = true;
 879                 }
 880
 881                 if (data)
 882                         memset(data, pattern, ecc->size);
 883
 884                 if (oob)
 885                         memset(oob, pattern, ecc->bytes + 4);
 886
 887                 return 0;
 888         }
 889
 890         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 891
 892         return NFC_ECC_ERR_CNT(step, tmp);
 893 }
 894
 895 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 896                                        u8 *data, int data_off,
 897                                        u8 *oob, int oob_off,
 898                                        int *cur_off,
 899                                        unsigned int *max_bitflips,
 900                                        bool bbm, bool oob_required, int page)
 901 {
 902         struct nand_chip *nand = mtd_to_nand(mtd);
 903         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 904         struct nand_ecc_ctrl *ecc = &nand->ecc;
 905         int raw_mode = 0;
 906         bool erased;
 907         int ret;
 908
 909         if (*cur_off != data_off)
 910                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 911
 912         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 913
 914         if (data_off + ecc->size != oob_off)
 915                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 916
 917         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 918         if (ret)
 919                 return ret;
 920
 921         sunxi_nfc_randomizer_enable(mtd);
 922         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 923                nfc->regs + NFC_REG_CMD);
 924
 925         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 926         sunxi_nfc_randomizer_disable(mtd);
 927         if (ret)
 928                 return ret;
 929
 930         *cur_off = oob_off + ecc->bytes + 4;
 931
 932         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 933                                        readl(nfc->regs + NFC_REG_ECC_ST),
 934                                        &erased);
 935         if (erased)
 936                 return 1;
 937
 938         if (ret < 0) {
 939                 /*
 940                  * Re-read the data with the randomizer disabled to identify
 941                  * bitflips in erased pages.
 942                  */
 943                 if (nand->options & NAND_NEED_SCRAMBLING)
 944                         nand_change_read_column_op(nand, data_off, data,
 945                                                    ecc->size, false);
 946                 else
 947                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 948                                       ecc->size);
 949
 950                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 951                                            false);
 952
 953                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 954                                                   oob, ecc->bytes + 4,
 955                                                   NULL, 0, ecc->strength);
 956                 if (ret >= 0)
 957                         raw_mode = 1;
 958         } else {
 959                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 960
 961                 if (oob_required) {
 962                         nand_change_read_column_op(nand, oob_off, NULL, 0,
 963                                                    false);
 964                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
 965                                                       true, page);
 966
 967                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
 968                                                             bbm, page);
 969                 }
 970         }
 971
 972         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
 973
 974         return raw_mode;
 975 }
 976
 977 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
 978                                             u8 *oob, int *cur_off,
 979                                             bool randomize, int page)
 980 {
 981         struct nand_chip *nand = mtd_to_nand(mtd);
 982         struct nand_ecc_ctrl *ecc = &nand->ecc;
 983         int offset = ((ecc->bytes + 4) * ecc->steps);
 984         int len = mtd->oobsize - offset;
 985
 986         if (len <= 0)
 987                 return;
 988
 989         if (!cur_off || *cur_off != offset)
 990                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 991                                            false);
 992
 993         if (!randomize)
 994                 sunxi_nfc_read_buf(nand, oob + offset, len);
 995         else
 996                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
 997                                               false, page);
 998
 999         if (cur_off)
1000                 *cur_off = mtd->oobsize + mtd->writesize;
1001 }
1002
1003 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1004                                             int oob_required, int page,
1005                                             int nchunks)
1006 {
1007         struct nand_chip *nand = mtd_to_nand(mtd);
1008         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1009         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1010         struct nand_ecc_ctrl *ecc = &nand->ecc;
1011         unsigned int max_bitflips = 0;
1012         int ret, i, raw_mode = 0;
1013         struct scatterlist sg;
1014         u32 status;
1015
1016         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1017         if (ret)
1018                 return ret;
1019
1020         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1021                                        DMA_FROM_DEVICE, &sg);
1022         if (ret)
1023                 return ret;
1024
1025         sunxi_nfc_hw_ecc_enable(mtd);
1026         sunxi_nfc_randomizer_config(mtd, page, false);
1027         sunxi_nfc_randomizer_enable(mtd);
1028
1029         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1030                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1031
1032         dma_async_issue_pending(nfc->dmac);
1033
1034         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1035                nfc->regs + NFC_REG_CMD);
1036
1037         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1038         if (ret)
1039                 dmaengine_terminate_all(nfc->dmac);
1040
1041         sunxi_nfc_randomizer_disable(mtd);
1042         sunxi_nfc_hw_ecc_disable(mtd);
1043
1044         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1045
1046         if (ret)
1047                 return ret;
1048
1049         status = readl(nfc->regs + NFC_REG_ECC_ST);
1050
1051         for (i = 0; i < nchunks; i++) {
1052                 int data_off = i * ecc->size;
1053                 int oob_off = i * (ecc->bytes + 4);
1054                 u8 *data = buf + data_off;
1055                 u8 *oob = nand->oob_poi + oob_off;
1056                 bool erased;
1057
1058                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1059                                                oob_required ? oob : NULL,
1060                                                i, status, &erased);
1061
1062                 /* ECC errors are handled in the second loop. */
1063                 if (ret < 0)
1064                         continue;
1065
1066                 if (oob_required && !erased) {
1067                         /* TODO: use DMA to retrieve OOB */
1068                         nand_change_read_column_op(nand,
1069                                                    mtd->writesize + oob_off,
1070                                                    oob, ecc->bytes + 4, false);
1071
1072                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1073                                                             !i, page);
1074                 }
1075
1076                 if (erased)
1077                         raw_mode = 1;
1078
1079                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1080         }
1081
1082         if (status & NFC_ECC_ERR_MSK) {
1083                 for (i = 0; i < nchunks; i++) {
1084                         int data_off = i * ecc->size;
1085                         int oob_off = i * (ecc->bytes + 4);
1086                         u8 *data = buf + data_off;
1087                         u8 *oob = nand->oob_poi + oob_off;
1088
1089                         if (!(status & NFC_ECC_ERR(i)))
1090                                 continue;
1091
1092                         /*
1093                          * Re-read the data with the randomizer disabled to
1094                          * identify bitflips in erased pages.
1095                          * TODO: use DMA to read page in raw mode
1096                          */
1097                         if (randomized)
1098                                 nand_change_read_column_op(nand, data_off,
1099                                                            data, ecc->size,
1100                                                            false);
1101
1102                         /* TODO: use DMA to retrieve OOB */
1103                         nand_change_read_column_op(nand,
1104                                                    mtd->writesize + oob_off,
1105                                                    oob, ecc->bytes + 4, false);
1106
1107                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1108                                                           oob, ecc->bytes + 4,
1109                                                           NULL, 0,
1110                                                           ecc->strength);
1111                         if (ret >= 0)
1112                                 raw_mode = 1;
1113
1114                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1115                 }
1116         }
1117
1118         if (oob_required)
1119                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1120                                                 NULL, !raw_mode,
1121                                                 page);
1122
1123         return max_bitflips;
1124 }
1125
1126 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1127                                         const u8 *data, int data_off,
1128                                         const u8 *oob, int oob_off,
1129                                         int *cur_off, bool bbm,
1130                                         int page)
1131 {
1132         struct nand_chip *nand = mtd_to_nand(mtd);
1133         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1134         struct nand_ecc_ctrl *ecc = &nand->ecc;
1135         int ret;
1136
1137         if (data_off != *cur_off)
1138                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1139
1140         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1141
1142         if (data_off + ecc->size != oob_off)
1143                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1144
1145         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1146         if (ret)
1147                 return ret;
1148
1149         sunxi_nfc_randomizer_enable(mtd);
1150         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1151
1152         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1153                NFC_ACCESS_DIR | NFC_ECC_OP,
1154                nfc->regs + NFC_REG_CMD);
1155
1156         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1157         sunxi_nfc_randomizer_disable(mtd);
1158         if (ret)
1159                 return ret;
1160
1161         *cur_off = oob_off + ecc->bytes + 4;
1162
1163         return 0;
1164 }
1165
1166 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1167                                              u8 *oob, int *cur_off,
1168                                              int page)
1169 {
1170         struct nand_chip *nand = mtd_to_nand(mtd);
1171         struct nand_ecc_ctrl *ecc = &nand->ecc;
1172         int offset = ((ecc->bytes + 4) * ecc->steps);
1173         int len = mtd->oobsize - offset;
1174
1175         if (len <= 0)
1176                 return;
1177
1178         if (!cur_off || *cur_off != offset)
1179                 nand_change_write_column_op(nand, offset + mtd->writesize,
1180                                             NULL, 0, false);
1181
1182         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1183
1184         if (cur_off)
1185                 *cur_off = mtd->oobsize + mtd->writesize;
1186 }
1187
1188 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *chip, uint8_t *buf,
1189                                       int oob_required, int page)
1190 {
1191         struct mtd_info *mtd = nand_to_mtd(chip);
1192         struct nand_ecc_ctrl *ecc = &chip->ecc;
1193         unsigned int max_bitflips = 0;
1194         int ret, i, cur_off = 0;
1195         bool raw_mode = false;
1196
1197         nand_read_page_op(chip, page, 0, NULL, 0);
1198
1199         sunxi_nfc_hw_ecc_enable(mtd);
1200
1201         for (i = 0; i < ecc->steps; i++) {
1202                 int data_off = i * ecc->size;
1203                 int oob_off = i * (ecc->bytes + 4);
1204                 u8 *data = buf + data_off;
1205                 u8 *oob = chip->oob_poi + oob_off;
1206
1207                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1208                                                   oob_off + mtd->writesize,
1209                                                   &cur_off, &max_bitflips,
1210                                                   !i, oob_required, page);
1211                 if (ret < 0)
1212                         return ret;
1213                 else if (ret)
1214                         raw_mode = true;
1215         }
1216
1217         if (oob_required)
1218                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1219                                                 !raw_mode, page);
1220
1221         sunxi_nfc_hw_ecc_disable(mtd);
1222
1223         return max_bitflips;
1224 }
1225
1226 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *chip, u8 *buf,
1227                                           int oob_required, int page)
1228 {
1229         struct mtd_info *mtd = nand_to_mtd(chip);
1230         int ret;
1231
1232         nand_read_page_op(chip, page, 0, NULL, 0);
1233
1234         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1235                                                chip->ecc.steps);
1236         if (ret >= 0)
1237                 return ret;
1238
1239         /* Fallback to PIO mode */
1240         return sunxi_nfc_hw_ecc_read_page(chip, buf, oob_required, page);
1241 }
1242
1243 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *chip,
1244                                          u32 data_offs, u32 readlen,
1245                                          u8 *bufpoi, int page)
1246 {
1247         struct mtd_info *mtd = nand_to_mtd(chip);
1248         struct nand_ecc_ctrl *ecc = &chip->ecc;
1249         int ret, i, cur_off = 0;
1250         unsigned int max_bitflips = 0;
1251
1252         nand_read_page_op(chip, page, 0, NULL, 0);
1253
1254         sunxi_nfc_hw_ecc_enable(mtd);
1255
1256         for (i = data_offs / ecc->size;
1257              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1258                 int data_off = i * ecc->size;
1259                 int oob_off = i * (ecc->bytes + 4);
1260                 u8 *data = bufpoi + data_off;
1261                 u8 *oob = chip->oob_poi + oob_off;
1262
1263                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1264                                                   oob,
1265                                                   oob_off + mtd->writesize,
1266                                                   &cur_off, &max_bitflips, !i,
1267                                                   false, page);
1268                 if (ret < 0)
1269                         return ret;
1270         }
1271
1272         sunxi_nfc_hw_ecc_disable(mtd);
1273
1274         return max_bitflips;
1275 }
1276
1277 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *chip,
1278                                              u32 data_offs, u32 readlen,
1279                                              u8 *buf, int page)
1280 {
1281         struct mtd_info *mtd = nand_to_mtd(chip);
1282         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1283         int ret;
1284
1285         nand_read_page_op(chip, page, 0, NULL, 0);
1286
1287         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1288         if (ret >= 0)
1289                 return ret;
1290
1291         /* Fallback to PIO mode */
1292         return sunxi_nfc_hw_ecc_read_subpage(chip, data_offs, readlen,
1293                                              buf, page);
1294 }
1295
1296 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *chip,
1297                                        const uint8_t *buf, int oob_required,
1298                                        int page)
1299 {
1300         struct mtd_info *mtd = nand_to_mtd(chip);
1301         struct nand_ecc_ctrl *ecc = &chip->ecc;
1302         int ret, i, cur_off = 0;
1303
1304         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1305
1306         sunxi_nfc_hw_ecc_enable(mtd);
1307
1308         for (i = 0; i < ecc->steps; i++) {
1309                 int data_off = i * ecc->size;
1310                 int oob_off = i * (ecc->bytes + 4);
1311                 const u8 *data = buf + data_off;
1312                 const u8 *oob = chip->oob_poi + oob_off;
1313
1314                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1315                                                    oob_off + mtd->writesize,
1316                                                    &cur_off, !i, page);
1317                 if (ret)
1318                         return ret;
1319         }
1320
1321         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1322                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1323                                                  &cur_off, page);
1324
1325         sunxi_nfc_hw_ecc_disable(mtd);
1326
1327         return nand_prog_page_end_op(chip);
1328 }
1329
1330 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *chip,
1331                                           u32 data_offs, u32 data_len,
1332                                           const u8 *buf, int oob_required,
1333                                           int page)
1334 {
1335         struct mtd_info *mtd = nand_to_mtd(chip);
1336         struct nand_ecc_ctrl *ecc = &chip->ecc;
1337         int ret, i, cur_off = 0;
1338
1339         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1340
1341         sunxi_nfc_hw_ecc_enable(mtd);
1342
1343         for (i = data_offs / ecc->size;
1344              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1345                 int data_off = i * ecc->size;
1346                 int oob_off = i * (ecc->bytes + 4);
1347                 const u8 *data = buf + data_off;
1348                 const u8 *oob = chip->oob_poi + oob_off;
1349
1350                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1351                                                    oob_off + mtd->writesize,
1352                                                    &cur_off, !i, page);
1353                 if (ret)
1354                         return ret;
1355         }
1356
1357         sunxi_nfc_hw_ecc_disable(mtd);
1358
1359         return nand_prog_page_end_op(chip);
1360 }
1361
1362 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *chip,
1363                                            const u8 *buf,
1364                                            int oob_required,
1365                                            int page)
1366 {
1367         struct mtd_info *mtd = nand_to_mtd(chip);
1368         struct nand_chip *nand = mtd_to_nand(mtd);
1369         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1370         struct nand_ecc_ctrl *ecc = &nand->ecc;
1371         struct scatterlist sg;
1372         int ret, i;
1373
1374         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1375         if (ret)
1376                 return ret;
1377
1378         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1379                                        DMA_TO_DEVICE, &sg);
1380         if (ret)
1381                 goto pio_fallback;
1382
1383         for (i = 0; i < ecc->steps; i++) {
1384                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1385
1386                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1387         }
1388
1389         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1390
1391         sunxi_nfc_hw_ecc_enable(mtd);
1392         sunxi_nfc_randomizer_config(mtd, page, false);
1393         sunxi_nfc_randomizer_enable(mtd);
1394
1395         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1396                nfc->regs + NFC_REG_RCMD_SET);
1397
1398         dma_async_issue_pending(nfc->dmac);
1399
1400         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1401                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1402                nfc->regs + NFC_REG_CMD);
1403
1404         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1405         if (ret)
1406                 dmaengine_terminate_all(nfc->dmac);
1407
1408         sunxi_nfc_randomizer_disable(mtd);
1409         sunxi_nfc_hw_ecc_disable(mtd);
1410
1411         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1412
1413         if (ret)
1414                 return ret;
1415
1416         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1417                 /* TODO: use DMA to transfer extra OOB bytes ? */
1418                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1419                                                  NULL, page);
1420
1421         return nand_prog_page_end_op(chip);
1422
1423 pio_fallback:
1424         return sunxi_nfc_hw_ecc_write_page(chip, buf, oob_required, page);
1425 }
1426
1427 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *chip, int page)
1428 {
1429         chip->pagebuf = -1;
1430
1431         return chip->ecc.read_page(chip, chip->data_buf, 1, page);
1432 }
1433
1434 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *chip, int page)
1435 {
1436         struct mtd_info *mtd = nand_to_mtd(chip);
1437         int ret;
1438
1439         chip->pagebuf = -1;
1440
1441         memset(chip->data_buf, 0xff, mtd->writesize);
1442         ret = chip->ecc.write_page(chip, chip->data_buf, 1, page);
1443         if (ret)
1444                 return ret;
1445
1446         /* Send command to program the OOB data */
1447         return nand_prog_page_end_op(chip);
1448 }
1449
1450 static const s32 tWB_lut[] = {6, 12, 16, 20};
1451 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1452
1453 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1454                 u32 clk_period)
1455 {
1456         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1457         int i;
1458
1459         for (i = 0; i < lut_size; i++) {
1460                 if (clk_cycles <= lut[i])
1461                         return i;
1462         }
1463
1464         /* Doesn't fit */
1465         return -EINVAL;
1466 }
1467
1468 #define sunxi_nand_lookup_timing(l, p, c) \
1469                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1470
1471 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline,
1472                                         const struct nand_data_interface *conf)
1473 {
1474         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1475         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1476         const struct nand_sdr_timings *timings;
1477         u32 min_clk_period = 0;
1478         s32 tWB, tADL, tWHR, tRHW, tCAD;
1479         long real_clk_rate;
1480
1481         timings = nand_get_sdr_timings(conf);
1482         if (IS_ERR(timings))
1483                 return -ENOTSUPP;
1484
1485         /* T1 <=> tCLS */
1486         if (timings->tCLS_min > min_clk_period)
1487                 min_clk_period = timings->tCLS_min;
1488
1489         /* T2 <=> tCLH */
1490         if (timings->tCLH_min > min_clk_period)
1491                 min_clk_period = timings->tCLH_min;
1492
1493         /* T3 <=> tCS */
1494         if (timings->tCS_min > min_clk_period)
1495                 min_clk_period = timings->tCS_min;
1496
1497         /* T4 <=> tCH */
1498         if (timings->tCH_min > min_clk_period)
1499                 min_clk_period = timings->tCH_min;
1500
1501         /* T5 <=> tWP */
1502         if (timings->tWP_min > min_clk_period)
1503                 min_clk_period = timings->tWP_min;
1504
1505         /* T6 <=> tWH */
1506         if (timings->tWH_min > min_clk_period)
1507                 min_clk_period = timings->tWH_min;
1508
1509         /* T7 <=> tALS */
1510         if (timings->tALS_min > min_clk_period)
1511                 min_clk_period = timings->tALS_min;
1512
1513         /* T8 <=> tDS */
1514         if (timings->tDS_min > min_clk_period)
1515                 min_clk_period = timings->tDS_min;
1516
1517         /* T9 <=> tDH */
1518         if (timings->tDH_min > min_clk_period)
1519                 min_clk_period = timings->tDH_min;
1520
1521         /* T10 <=> tRR */
1522         if (timings->tRR_min > (min_clk_period * 3))
1523                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1524
1525         /* T11 <=> tALH */
1526         if (timings->tALH_min > min_clk_period)
1527                 min_clk_period = timings->tALH_min;
1528
1529         /* T12 <=> tRP */
1530         if (timings->tRP_min > min_clk_period)
1531                 min_clk_period = timings->tRP_min;
1532
1533         /* T13 <=> tREH */
1534         if (timings->tREH_min > min_clk_period)
1535                 min_clk_period = timings->tREH_min;
1536
1537         /* T14 <=> tRC */
1538         if (timings->tRC_min > (min_clk_period * 2))
1539                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1540
1541         /* T15 <=> tWC */
1542         if (timings->tWC_min > (min_clk_period * 2))
1543                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1544
1545         /* T16 - T19 + tCAD */
1546         if (timings->tWB_max > (min_clk_period * 20))
1547                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1548
1549         if (timings->tADL_min > (min_clk_period * 32))
1550                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1551
1552         if (timings->tWHR_min > (min_clk_period * 32))
1553                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1554
1555         if (timings->tRHW_min > (min_clk_period * 20))
1556                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1557
1558         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1559                                         min_clk_period);
1560         if (tWB < 0) {
1561                 dev_err(nfc->dev, "unsupported tWB\n");
1562                 return tWB;
1563         }
1564
1565         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1566         if (tADL > 3) {
1567                 dev_err(nfc->dev, "unsupported tADL\n");
1568                 return -EINVAL;
1569         }
1570
1571         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1572         if (tWHR > 3) {
1573                 dev_err(nfc->dev, "unsupported tWHR\n");
1574                 return -EINVAL;
1575         }
1576
1577         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1578                                         min_clk_period);
1579         if (tRHW < 0) {
1580                 dev_err(nfc->dev, "unsupported tRHW\n");
1581                 return tRHW;
1582         }
1583
1584         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1585                 return 0;
1586
1587         /*
1588          * TODO: according to ONFI specs this value only applies for DDR NAND,
1589          * but Allwinner seems to set this to 0x7. Mimic them for now.
1590          */
1591         tCAD = 0x7;
1592
1593         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1594         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1595
1596         /* Convert min_clk_period from picoseconds to nanoseconds */
1597         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1598
1599         /*
1600          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1601          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1602          * This new formula was verified with a scope and validated by
1603          * Allwinner engineers.
1604          */
1605         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1606         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1607         if (real_clk_rate <= 0) {
1608                 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate);
1609                 return -EINVAL;
1610         }
1611
1612         /*
1613          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1614          * output cycle timings shall be used if the host drives tRC less than
1615          * 30 ns.
1616          */
1617         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1618         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1619                            NFC_TIMING_CTL_EDO : 0;
1620
1621         return 0;
1622 }
1623
1624 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1625                                     struct mtd_oob_region *oobregion)
1626 {
1627         struct nand_chip *nand = mtd_to_nand(mtd);
1628         struct nand_ecc_ctrl *ecc = &nand->ecc;
1629
1630         if (section >= ecc->steps)
1631                 return -ERANGE;
1632
1633         oobregion->offset = section * (ecc->bytes + 4) + 4;
1634         oobregion->length = ecc->bytes;
1635
1636         return 0;
1637 }
1638
1639 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1640                                      struct mtd_oob_region *oobregion)
1641 {
1642         struct nand_chip *nand = mtd_to_nand(mtd);
1643         struct nand_ecc_ctrl *ecc = &nand->ecc;
1644
1645         if (section > ecc->steps)
1646                 return -ERANGE;
1647
1648         /*
1649          * The first 2 bytes are used for BB markers, hence we
1650          * only have 2 bytes available in the first user data
1651          * section.
1652          */
1653         if (!section && ecc->mode == NAND_ECC_HW) {
1654                 oobregion->offset = 2;
1655                 oobregion->length = 2;
1656
1657                 return 0;
1658         }
1659
1660         oobregion->offset = section * (ecc->bytes + 4);
1661
1662         if (section < ecc->steps)
1663                 oobregion->length = 4;
1664         else
1665                 oobregion->offset = mtd->oobsize - oobregion->offset;
1666
1667         return 0;
1668 }
1669
1670 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1671         .ecc = sunxi_nand_ooblayout_ecc,
1672         .free = sunxi_nand_ooblayout_free,
1673 };
1674
1675 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1676 {
1677         kfree(ecc->priv);
1678 }
1679
1680 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1681                                        struct nand_ecc_ctrl *ecc,
1682                                        struct device_node *np)
1683 {
1684         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1685         struct nand_chip *nand = mtd_to_nand(mtd);
1686         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1687         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1688         struct sunxi_nand_hw_ecc *data;
1689         int nsectors;
1690         int ret;
1691         int i;
1692
1693         if (ecc->options & NAND_ECC_MAXIMIZE) {
1694                 int bytes;
1695
1696                 ecc->size = 1024;
1697                 nsectors = mtd->writesize / ecc->size;
1698
1699                 /* Reserve 2 bytes for the BBM */
1700                 bytes = (mtd->oobsize - 2) / nsectors;
1701
1702                 /* 4 non-ECC bytes are added before each ECC bytes section */
1703                 bytes -= 4;
1704
1705                 /* and bytes has to be even. */
1706                 if (bytes % 2)
1707                         bytes--;
1708
1709                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1710
1711                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1712                         if (strengths[i] > ecc->strength)
1713                                 break;
1714                 }
1715
1716                 if (!i)
1717                         ecc->strength = 0;
1718                 else
1719                         ecc->strength = strengths[i - 1];
1720         }
1721
1722         if (ecc->size != 512 && ecc->size != 1024)
1723                 return -EINVAL;
1724
1725         data = kzalloc(sizeof(*data), GFP_KERNEL);
1726         if (!data)
1727                 return -ENOMEM;
1728
1729         /* Prefer 1k ECC chunk over 512 ones */
1730         if (ecc->size == 512 && mtd->writesize > 512) {
1731                 ecc->size = 1024;
1732                 ecc->strength *= 2;
1733         }
1734
1735         /* Add ECC info retrieval from DT */
1736         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1737                 if (ecc->strength <= strengths[i]) {
1738                         /*
1739                          * Update ecc->strength value with the actual strength
1740                          * that will be used by the ECC engine.
1741                          */
1742                         ecc->strength = strengths[i];
1743                         break;
1744                 }
1745         }
1746
1747         if (i >= ARRAY_SIZE(strengths)) {
1748                 dev_err(nfc->dev, "unsupported strength\n");
1749                 ret = -ENOTSUPP;
1750                 goto err;
1751         }
1752
1753         data->mode = i;
1754
1755         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1756         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1757
1758         /* HW ECC always work with even numbers of ECC bytes */
1759         ecc->bytes = ALIGN(ecc->bytes, 2);
1760
1761         nsectors = mtd->writesize / ecc->size;
1762
1763         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1764                 ret = -EINVAL;
1765                 goto err;
1766         }
1767
1768         ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1769         ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1770         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1771         ecc->priv = data;
1772
1773         if (nfc->dmac) {
1774                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1775                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1776                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1777                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1778         } else {
1779                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1780                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1781                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1782         }
1783
1784         /* TODO: support DMA for raw accesses and subpage write */
1785         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1786         ecc->read_oob_raw = nand_read_oob_std;
1787         ecc->write_oob_raw = nand_write_oob_std;
1788
1789         return 0;
1790
1791 err:
1792         kfree(data);
1793
1794         return ret;
1795 }
1796
1797 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1798 {
1799         switch (ecc->mode) {
1800         case NAND_ECC_HW:
1801                 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1802                 break;
1803         case NAND_ECC_NONE:
1804         default:
1805                 break;
1806         }
1807 }
1808
1809 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1810 {
1811         struct mtd_info *mtd = nand_to_mtd(nand);
1812         struct nand_ecc_ctrl *ecc = &nand->ecc;
1813         struct device_node *np = nand_get_flash_node(nand);
1814         int ret;
1815
1816         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1817                 nand->bbt_options |= NAND_BBT_NO_OOB;
1818
1819         if (nand->options & NAND_NEED_SCRAMBLING)
1820                 nand->options |= NAND_NO_SUBPAGE_WRITE;
1821
1822         nand->options |= NAND_SUBPAGE_READ;
1823
1824         if (!ecc->size) {
1825                 ecc->size = nand->ecc_step_ds;
1826                 ecc->strength = nand->ecc_strength_ds;
1827         }
1828
1829         if (!ecc->size || !ecc->strength)
1830                 return -EINVAL;
1831
1832         switch (ecc->mode) {
1833         case NAND_ECC_HW:
1834                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1835                 if (ret)
1836                         return ret;
1837                 break;
1838         case NAND_ECC_NONE:
1839         case NAND_ECC_SOFT:
1840                 break;
1841         default:
1842                 return -EINVAL;
1843         }
1844
1845         return 0;
1846 }
1847
1848 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1849         .attach_chip = sunxi_nand_attach_chip,
1850 };
1851
1852 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1853                                 struct device_node *np)
1854 {
1855         struct sunxi_nand_chip *chip;
1856         struct mtd_info *mtd;
1857         struct nand_chip *nand;
1858         int nsels;
1859         int ret;
1860         int i;
1861         u32 tmp;
1862
1863         if (!of_get_property(np, "reg", &nsels))
1864                 return -EINVAL;
1865
1866         nsels /= sizeof(u32);
1867         if (!nsels) {
1868                 dev_err(dev, "invalid reg property size\n");
1869                 return -EINVAL;
1870         }
1871
1872         chip = devm_kzalloc(dev,
1873                             sizeof(*chip) +
1874                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
1875                             GFP_KERNEL);
1876         if (!chip) {
1877                 dev_err(dev, "could not allocate chip\n");
1878                 return -ENOMEM;
1879         }
1880
1881         chip->nsels = nsels;
1882         chip->selected = -1;
1883
1884         for (i = 0; i < nsels; i++) {
1885                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1886                 if (ret) {
1887                         dev_err(dev, "could not retrieve reg property: %d\n",
1888                                 ret);
1889                         return ret;
1890                 }
1891
1892                 if (tmp > NFC_MAX_CS) {
1893                         dev_err(dev,
1894                                 "invalid reg value: %u (max CS = 7)\n",
1895                                 tmp);
1896                         return -EINVAL;
1897                 }
1898
1899                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1900                         dev_err(dev, "CS %d already assigned\n", tmp);
1901                         return -EINVAL;
1902                 }
1903
1904                 chip->sels[i].cs = tmp;
1905
1906                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1907                     tmp < 2)
1908                         chip->sels[i].rb = tmp;
1909                 else
1910                         chip->sels[i].rb = -1;
1911         }
1912
1913         nand = &chip->nand;
1914         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1915         nand->chip_delay = 200;
1916         nand->controller = &nfc->controller;
1917         nand->controller->ops = &sunxi_nand_controller_ops;
1918
1919         /*
1920          * Set the ECC mode to the default value in case nothing is specified
1921          * in the DT.
1922          */
1923         nand->ecc.mode = NAND_ECC_HW;
1924         nand_set_flash_node(nand, np);
1925         nand->select_chip = sunxi_nfc_select_chip;
1926         nand->legacy.cmd_ctrl = sunxi_nfc_cmd_ctrl;
1927         nand->legacy.read_buf = sunxi_nfc_read_buf;
1928         nand->legacy.write_buf = sunxi_nfc_write_buf;
1929         nand->legacy.read_byte = sunxi_nfc_read_byte;
1930         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
1931
1932         mtd = nand_to_mtd(nand);
1933         mtd->dev.parent = dev;
1934
1935         ret = nand_scan(nand, nsels);
1936         if (ret)
1937                 return ret;
1938
1939         ret = mtd_device_register(mtd, NULL, 0);
1940         if (ret) {
1941                 dev_err(dev, "failed to register mtd device: %d\n", ret);
1942                 nand_release(nand);
1943                 return ret;
1944         }
1945
1946         list_add_tail(&chip->node, &nfc->chips);
1947
1948         return 0;
1949 }
1950
1951 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
1952 {
1953         struct device_node *np = dev->of_node;
1954         struct device_node *nand_np;
1955         int nchips = of_get_child_count(np);
1956         int ret;
1957
1958         if (nchips > 8) {
1959                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
1960                 return -EINVAL;
1961         }
1962
1963         for_each_child_of_node(np, nand_np) {
1964                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
1965                 if (ret) {
1966                         of_node_put(nand_np);
1967                         return ret;
1968                 }
1969         }
1970
1971         return 0;
1972 }
1973
1974 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
1975 {
1976         struct sunxi_nand_chip *chip;
1977
1978         while (!list_empty(&nfc->chips)) {
1979                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
1980                                         node);
1981                 nand_release(&chip->nand);
1982                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
1983                 list_del(&chip->node);
1984         }
1985 }
1986
1987 static int sunxi_nfc_probe(struct platform_device *pdev)
1988 {
1989         struct device *dev = &pdev->dev;
1990         struct resource *r;
1991         struct sunxi_nfc *nfc;
1992         int irq;
1993         int ret;
1994
1995         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1996         if (!nfc)
1997                 return -ENOMEM;
1998
1999         nfc->dev = dev;
2000         nand_controller_init(&nfc->controller);
2001         INIT_LIST_HEAD(&nfc->chips);
2002
2003         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2004         nfc->regs = devm_ioremap_resource(dev, r);
2005         if (IS_ERR(nfc->regs))
2006                 return PTR_ERR(nfc->regs);
2007
2008         irq = platform_get_irq(pdev, 0);
2009         if (irq < 0) {
2010                 dev_err(dev, "failed to retrieve irq\n");
2011                 return irq;
2012         }
2013
2014         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2015         if (IS_ERR(nfc->ahb_clk)) {
2016                 dev_err(dev, "failed to retrieve ahb clk\n");
2017                 return PTR_ERR(nfc->ahb_clk);
2018         }
2019
2020         ret = clk_prepare_enable(nfc->ahb_clk);
2021         if (ret)
2022                 return ret;
2023
2024         nfc->mod_clk = devm_clk_get(dev, "mod");
2025         if (IS_ERR(nfc->mod_clk)) {
2026                 dev_err(dev, "failed to retrieve mod clk\n");
2027                 ret = PTR_ERR(nfc->mod_clk);
2028                 goto out_ahb_clk_unprepare;
2029         }
2030
2031         ret = clk_prepare_enable(nfc->mod_clk);
2032         if (ret)
2033                 goto out_ahb_clk_unprepare;
2034
2035         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2036         if (IS_ERR(nfc->reset)) {
2037                 ret = PTR_ERR(nfc->reset);
2038                 goto out_mod_clk_unprepare;
2039         }
2040
2041         ret = reset_control_deassert(nfc->reset);
2042         if (ret) {
2043                 dev_err(dev, "reset err %d\n", ret);
2044                 goto out_mod_clk_unprepare;
2045         }
2046
2047         ret = sunxi_nfc_rst(nfc);
2048         if (ret)
2049                 goto out_ahb_reset_reassert;
2050
2051         writel(0, nfc->regs + NFC_REG_INT);
2052         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2053                                0, "sunxi-nand", nfc);
2054         if (ret)
2055                 goto out_ahb_reset_reassert;
2056
2057         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2058         if (nfc->dmac) {
2059                 struct dma_slave_config dmac_cfg = { };
2060
2061                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2062                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2063                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2064                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2065                 dmac_cfg.src_maxburst = 4;
2066                 dmac_cfg.dst_maxburst = 4;
2067                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2068         } else {
2069                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2070         }
2071
2072         platform_set_drvdata(pdev, nfc);
2073
2074         ret = sunxi_nand_chips_init(dev, nfc);
2075         if (ret) {
2076                 dev_err(dev, "failed to init nand chips\n");
2077                 goto out_release_dmac;
2078         }
2079
2080         return 0;
2081
2082 out_release_dmac:
2083         if (nfc->dmac)
2084                 dma_release_channel(nfc->dmac);
2085 out_ahb_reset_reassert:
2086         reset_control_assert(nfc->reset);
2087 out_mod_clk_unprepare:
2088         clk_disable_unprepare(nfc->mod_clk);
2089 out_ahb_clk_unprepare:
2090         clk_disable_unprepare(nfc->ahb_clk);
2091
2092         return ret;
2093 }
2094
2095 static int sunxi_nfc_remove(struct platform_device *pdev)
2096 {
2097         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2098
2099         sunxi_nand_chips_cleanup(nfc);
2100
2101         reset_control_assert(nfc->reset);
2102
2103         if (nfc->dmac)
2104                 dma_release_channel(nfc->dmac);
2105         clk_disable_unprepare(nfc->mod_clk);
2106         clk_disable_unprepare(nfc->ahb_clk);
2107
2108         return 0;
2109 }
2110
2111 static const struct of_device_id sunxi_nfc_ids[] = {
2112         { .compatible = "allwinner,sun4i-a10-nand" },
2113         { /* sentinel */ }
2114 };
2115 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2116
2117 static struct platform_driver sunxi_nfc_driver = {
2118         .driver = {
2119                 .name = "sunxi_nand",
2120                 .of_match_table = sunxi_nfc_ids,
2121         },
2122         .probe = sunxi_nfc_probe,
2123         .remove = sunxi_nfc_remove,
2124 };
2125 module_platform_driver(sunxi_nfc_driver);
2126
2127 MODULE_LICENSE("GPL v2");
2128 MODULE_AUTHOR("Boris BREZILLON");
2129 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2130 MODULE_ALIAS("platform:sunxi_nand");