drivers/mtd/nand/raw/sunxi_nand.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   4  *
   5  * Derived from:
   6  *      https://github.com/yuq/sunxi-nfc-mtd
   7  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   8  *
   9  *      https://github.com/hno/Allwinner-Info
  10  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  11  *
  12  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  13  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  14  */
  15
  16 #include <linux/dma-mapping.h>
  17 #include <linux/slab.h>
  18 #include <linux/module.h>
  19 #include <linux/moduleparam.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/of.h>
  22 #include <linux/of_device.h>
  23 #include <linux/mtd/mtd.h>
  24 #include <linux/mtd/rawnand.h>
  25 #include <linux/mtd/partitions.h>
  26 #include <linux/clk.h>
  27 #include <linux/delay.h>
  28 #include <linux/dmaengine.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/iopoll.h>
  31 #include <linux/reset.h>
  32
  33 #define NFC_REG_CTL             0x0000
  34 #define NFC_REG_ST              0x0004
  35 #define NFC_REG_INT             0x0008
  36 #define NFC_REG_TIMING_CTL      0x000C
  37 #define NFC_REG_TIMING_CFG      0x0010
  38 #define NFC_REG_ADDR_LOW        0x0014
  39 #define NFC_REG_ADDR_HIGH       0x0018
  40 #define NFC_REG_SECTOR_NUM      0x001C
  41 #define NFC_REG_CNT             0x0020
  42 #define NFC_REG_CMD             0x0024
  43 #define NFC_REG_RCMD_SET        0x0028
  44 #define NFC_REG_WCMD_SET        0x002C
  45 #define NFC_REG_IO_DATA         0x0030
  46 #define NFC_REG_ECC_CTL         0x0034
  47 #define NFC_REG_ECC_ST          0x0038
  48 #define NFC_REG_DEBUG           0x003C
  49 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  50 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  51 #define NFC_REG_SPARE_AREA      0x00A0
  52 #define NFC_REG_PAT_ID          0x00A4
  53 #define NFC_RAM0_BASE           0x0400
  54 #define NFC_RAM1_BASE           0x0800
  55
  56 /* define bit use in NFC_CTL */
  57 #define NFC_EN                  BIT(0)
  58 #define NFC_RESET               BIT(1)
  59 #define NFC_BUS_WIDTH_MSK       BIT(2)
  60 #define NFC_BUS_WIDTH_8         (0 << 2)
  61 #define NFC_BUS_WIDTH_16        (1 << 2)
  62 #define NFC_RB_SEL_MSK          BIT(3)
  63 #define NFC_RB_SEL(x)           ((x) << 3)
  64 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  65 #define NFC_CE_SEL(x)           ((x) << 24)
  66 #define NFC_CE_CTL              BIT(6)
  67 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  68 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  69 #define NFC_SAM                 BIT(12)
  70 #define NFC_RAM_METHOD          BIT(14)
  71 #define NFC_DEBUG_CTL           BIT(31)
  72
  73 /* define bit use in NFC_ST */
  74 #define NFC_RB_B2R              BIT(0)
  75 #define NFC_CMD_INT_FLAG        BIT(1)
  76 #define NFC_DMA_INT_FLAG        BIT(2)
  77 #define NFC_CMD_FIFO_STATUS     BIT(3)
  78 #define NFC_STA                 BIT(4)
  79 #define NFC_NATCH_INT_FLAG      BIT(5)
  80 #define NFC_RB_STATE(x)         BIT(x + 8)
  81
  82 /* define bit use in NFC_INT */
  83 #define NFC_B2R_INT_ENABLE      BIT(0)
  84 #define NFC_CMD_INT_ENABLE      BIT(1)
  85 #define NFC_DMA_INT_ENABLE      BIT(2)
  86 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  87                                  NFC_CMD_INT_ENABLE | \
  88                                  NFC_DMA_INT_ENABLE)
  89
  90 /* define bit use in NFC_TIMING_CTL */
  91 #define NFC_TIMING_CTL_EDO      BIT(8)
  92
  93 /* define NFC_TIMING_CFG register layout */
  94 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
  95         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
  96         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
  97         (((tCAD) & 0x7) << 8))
  98
  99 /* define bit use in NFC_CMD */
 100 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 101 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 102 #define NFC_CMD(x)              (x)
 103 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 104 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 105 #define NFC_SEND_ADR            BIT(19)
 106 #define NFC_ACCESS_DIR          BIT(20)
 107 #define NFC_DATA_TRANS          BIT(21)
 108 #define NFC_SEND_CMD1           BIT(22)
 109 #define NFC_WAIT_FLAG           BIT(23)
 110 #define NFC_SEND_CMD2           BIT(24)
 111 #define NFC_SEQ                 BIT(25)
 112 #define NFC_DATA_SWAP_METHOD    BIT(26)
 113 #define NFC_ROW_AUTO_INC        BIT(27)
 114 #define NFC_SEND_CMD3           BIT(28)
 115 #define NFC_SEND_CMD4           BIT(29)
 116 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 117 #define NFC_NORMAL_OP           (0 << 30)
 118 #define NFC_ECC_OP              (1 << 30)
 119 #define NFC_PAGE_OP             (2U << 30)
 120
 121 /* define bit use in NFC_RCMD_SET */
 122 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 123 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 124 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 125
 126 /* define bit use in NFC_WCMD_SET */
 127 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 128 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 129 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 130 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 131
 132 /* define bit use in NFC_ECC_CTL */
 133 #define NFC_ECC_EN              BIT(0)
 134 #define NFC_ECC_PIPELINE        BIT(3)
 135 #define NFC_ECC_EXCEPTION       BIT(4)
 136 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 137 #define NFC_ECC_BLOCK_512       BIT(5)
 138 #define NFC_RANDOM_EN           BIT(9)
 139 #define NFC_RANDOM_DIRECTION    BIT(10)
 140 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 141 #define NFC_ECC_MODE(x)         ((x) << 12)
 142 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 143 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 144
 145 /* define bit use in NFC_ECC_ST */
 146 #define NFC_ECC_ERR(x)          BIT(x)
 147 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 148 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 149 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 150
 151 #define NFC_DEFAULT_TIMEOUT_MS  1000
 152
 153 #define NFC_SRAM_SIZE           1024
 154
 155 #define NFC_MAX_CS              7
 156
 157 /**
 158  * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
 159  *
 160  * @cs: the NAND CS id used to communicate with a NAND Chip
 161  * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
 162  */
 163 struct sunxi_nand_chip_sel {
 164         u8 cs;
 165         s8 rb;
 166 };
 167
 168 /**
 169  * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
 170  *
 171  * @mode: the sunxi ECC mode field deduced from ECC requirements
 172  */
 173 struct sunxi_nand_hw_ecc {
 174         int mode;
 175 };
 176
 177 /**
 178  * struct sunxi_nand_chip - stores NAND chip device related information
 179  *
 180  * @node: used to store NAND chips into a list
 181  * @nand: base NAND chip structure
 182  * @clk_rate: clk_rate required for this NAND chip
 183  * @timing_cfg: TIMING_CFG register value for this NAND chip
 184  * @timing_ctl: TIMING_CTL register value for this NAND chip
 185  * @nsels: number of CS lines required by the NAND chip
 186  * @sels: array of CS lines descriptions
 187  */
 188 struct sunxi_nand_chip {
 189         struct list_head node;
 190         struct nand_chip nand;
 191         unsigned long clk_rate;
 192         u32 timing_cfg;
 193         u32 timing_ctl;
 194         int nsels;
 195         struct sunxi_nand_chip_sel sels[0];
 196 };
 197
 198 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 199 {
 200         return container_of(nand, struct sunxi_nand_chip, nand);
 201 }
 202
 203 /**
 204  * struct sunxi_nfc - stores sunxi NAND controller information
 205  *
 206  * @controller: base controller structure
 207  * @dev: parent device (used to print error messages)
 208  * @regs: NAND controller registers
 209  * @ahb_clk: NAND controller AHB clock
 210  * @mod_clk: NAND controller mod clock
 211  * @reset: NAND controller reset line
 212  * @assigned_cs: bitmask describing already assigned CS lines
 213  * @clk_rate: NAND controller current clock rate
 214  * @chips: a list containing all the NAND chips attached to this NAND
 215  *         controller
 216  * @complete: a completion object used to wait for NAND controller events
 217  * @dmac: the DMA channel attached to the NAND controller
 218  */
 219 struct sunxi_nfc {
 220         struct nand_controller controller;
 221         struct device *dev;
 222         void __iomem *regs;
 223         struct clk *ahb_clk;
 224         struct clk *mod_clk;
 225         struct reset_control *reset;
 226         unsigned long assigned_cs;
 227         unsigned long clk_rate;
 228         struct list_head chips;
 229         struct completion complete;
 230         struct dma_chan *dmac;
 231 };
 232
 233 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 234 {
 235         return container_of(ctrl, struct sunxi_nfc, controller);
 236 }
 237
 238 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 239 {
 240         struct sunxi_nfc *nfc = dev_id;
 241         u32 st = readl(nfc->regs + NFC_REG_ST);
 242         u32 ien = readl(nfc->regs + NFC_REG_INT);
 243
 244         if (!(ien & st))
 245                 return IRQ_NONE;
 246
 247         if ((ien & st) == ien)
 248                 complete(&nfc->complete);
 249
 250         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 251         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 252
 253         return IRQ_HANDLED;
 254 }
 255
 256 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 257                                  bool use_polling, unsigned int timeout_ms)
 258 {
 259         int ret;
 260
 261         if (events & ~NFC_INT_MASK)
 262                 return -EINVAL;
 263
 264         if (!timeout_ms)
 265                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 266
 267         if (!use_polling) {
 268                 init_completion(&nfc->complete);
 269
 270                 writel(events, nfc->regs + NFC_REG_INT);
 271
 272                 ret = wait_for_completion_timeout(&nfc->complete,
 273                                                 msecs_to_jiffies(timeout_ms));
 274                 if (!ret)
 275                         ret = -ETIMEDOUT;
 276                 else
 277                         ret = 0;
 278
 279                 writel(0, nfc->regs + NFC_REG_INT);
 280         } else {
 281                 u32 status;
 282
 283                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 284                                          (status & events) == events, 1,
 285                                          timeout_ms * 1000);
 286         }
 287
 288         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 289
 290         if (ret)
 291                 dev_err(nfc->dev, "wait interrupt timedout\n");
 292
 293         return ret;
 294 }
 295
 296 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 297 {
 298         u32 status;
 299         int ret;
 300
 301         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 302                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 303                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 304         if (ret)
 305                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 306
 307         return ret;
 308 }
 309
 310 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 311 {
 312         u32 ctl;
 313         int ret;
 314
 315         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 316         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 317
 318         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 319                                  !(ctl & NFC_RESET), 1,
 320                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 321         if (ret)
 322                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 323
 324         return ret;
 325 }
 326
 327 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
 328                                     int chunksize, int nchunks,
 329                                     enum dma_data_direction ddir,
 330                                     struct scatterlist *sg)
 331 {
 332         struct dma_async_tx_descriptor *dmad;
 333         enum dma_transfer_direction tdir;
 334         dma_cookie_t dmat;
 335         int ret;
 336
 337         if (ddir == DMA_FROM_DEVICE)
 338                 tdir = DMA_DEV_TO_MEM;
 339         else
 340                 tdir = DMA_MEM_TO_DEV;
 341
 342         sg_init_one(sg, buf, nchunks * chunksize);
 343         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 344         if (!ret)
 345                 return -ENOMEM;
 346
 347         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 348         if (!dmad) {
 349                 ret = -EINVAL;
 350                 goto err_unmap_buf;
 351         }
 352
 353         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 354                nfc->regs + NFC_REG_CTL);
 355         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 356         writel(chunksize, nfc->regs + NFC_REG_CNT);
 357         dmat = dmaengine_submit(dmad);
 358
 359         ret = dma_submit_error(dmat);
 360         if (ret)
 361                 goto err_clr_dma_flag;
 362
 363         return 0;
 364
 365 err_clr_dma_flag:
 366         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 367                nfc->regs + NFC_REG_CTL);
 368
 369 err_unmap_buf:
 370         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 371         return ret;
 372 }
 373
 374 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
 375                                      enum dma_data_direction ddir,
 376                                      struct scatterlist *sg)
 377 {
 378         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 379         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 380                nfc->regs + NFC_REG_CTL);
 381 }
 382
 383 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
 384 {
 385         struct mtd_info *mtd = nand_to_mtd(nand);
 386         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 387         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 388         struct sunxi_nand_chip_sel *sel;
 389         u32 ctl;
 390
 391         if (cs > 0 && cs >= sunxi_nand->nsels)
 392                 return;
 393
 394         ctl = readl(nfc->regs + NFC_REG_CTL) &
 395               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 396
 397         sel = &sunxi_nand->sels[cs];
 398         ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
 399         if (sel->rb >= 0)
 400                 ctl |= NFC_RB_SEL(sel->rb);
 401
 402         writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 403
 404         if (nfc->clk_rate != sunxi_nand->clk_rate) {
 405                 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 406                 nfc->clk_rate = sunxi_nand->clk_rate;
 407         }
 408
 409         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 410         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 411         writel(ctl, nfc->regs + NFC_REG_CTL);
 412 }
 413
 414 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
 415 {
 416         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 417         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 418         int ret;
 419         int cnt;
 420         int offs = 0;
 421         u32 tmp;
 422
 423         while (len > offs) {
 424                 bool poll = false;
 425
 426                 cnt = min(len - offs, NFC_SRAM_SIZE);
 427
 428                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 429                 if (ret)
 430                         break;
 431
 432                 writel(cnt, nfc->regs + NFC_REG_CNT);
 433                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 434                 writel(tmp, nfc->regs + NFC_REG_CMD);
 435
 436                 /* Arbitrary limit for polling mode */
 437                 if (cnt < 64)
 438                         poll = true;
 439
 440                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 441                 if (ret)
 442                         break;
 443
 444                 if (buf)
 445                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 446                                       cnt);
 447                 offs += cnt;
 448         }
 449 }
 450
 451 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
 452                                 int len)
 453 {
 454         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 455         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 456         int ret;
 457         int cnt;
 458         int offs = 0;
 459         u32 tmp;
 460
 461         while (len > offs) {
 462                 bool poll = false;
 463
 464                 cnt = min(len - offs, NFC_SRAM_SIZE);
 465
 466                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 467                 if (ret)
 468                         break;
 469
 470                 writel(cnt, nfc->regs + NFC_REG_CNT);
 471                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 472                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 473                       NFC_ACCESS_DIR;
 474                 writel(tmp, nfc->regs + NFC_REG_CMD);
 475
 476                 /* Arbitrary limit for polling mode */
 477                 if (cnt < 64)
 478                         poll = true;
 479
 480                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 481                 if (ret)
 482                         break;
 483
 484                 offs += cnt;
 485         }
 486 }
 487
 488 /* These seed values have been extracted from Allwinner's BSP */
 489 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 490         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 491         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 492         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 493         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 494         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 495         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 496         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 497         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 498         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 499         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 500         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 501         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 502         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 503         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 504         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 505         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 506 };
 507
 508 /*
 509  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 510  * have been generated using
 511  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 512  * the randomizer engine does internally before de/scrambling OOB data.
 513  *
 514  * Those tables are statically defined to avoid calculating randomizer state
 515  * at runtime.
 516  */
 517 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 518         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 519         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 520         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 521         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 522         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 523         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 524         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 525         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 526         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 527         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 528         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 529         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 530         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 531         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 532         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 533         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 534 };
 535
 536 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 537         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 538         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 539         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 540         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 541         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 542         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 543         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 544         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 545         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 546         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 547         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 548         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 549         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 550         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 551         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 552         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 553 };
 554
 555 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 556 {
 557         state &= 0x7fff;
 558
 559         /*
 560          * This loop is just a simple implementation of a Fibonacci LFSR using
 561          * the x16 + x15 + 1 polynomial.
 562          */
 563         while (count--)
 564                 state = ((state >> 1) |
 565                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 566
 567         return state;
 568 }
 569
 570 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
 571                                       bool ecc)
 572 {
 573         struct mtd_info *mtd = nand_to_mtd(nand);
 574         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 575         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 576
 577         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 578                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 579
 580         if (ecc) {
 581                 if (mtd->ecc_step_size == 512)
 582                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 583                 else
 584                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 585         }
 586
 587         return seeds[page % mod];
 588 }
 589
 590 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
 591                                         bool ecc)
 592 {
 593         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 594         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 595         u16 state;
 596
 597         if (!(nand->options & NAND_NEED_SCRAMBLING))
 598                 return;
 599
 600         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 601         state = sunxi_nfc_randomizer_state(nand, page, ecc);
 602         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 603         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 604 }
 605
 606 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
 607 {
 608         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 609
 610         if (!(nand->options & NAND_NEED_SCRAMBLING))
 611                 return;
 612
 613         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 614                nfc->regs + NFC_REG_ECC_CTL);
 615 }
 616
 617 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
 618 {
 619         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 620
 621         if (!(nand->options & NAND_NEED_SCRAMBLING))
 622                 return;
 623
 624         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 625                nfc->regs + NFC_REG_ECC_CTL);
 626 }
 627
 628 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
 629 {
 630         u16 state = sunxi_nfc_randomizer_state(nand, page, true);
 631
 632         bbm[0] ^= state;
 633         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 634 }
 635
 636 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
 637                                            const uint8_t *buf, int len,
 638                                            bool ecc, int page)
 639 {
 640         sunxi_nfc_randomizer_config(nand, page, ecc);
 641         sunxi_nfc_randomizer_enable(nand);
 642         sunxi_nfc_write_buf(nand, buf, len);
 643         sunxi_nfc_randomizer_disable(nand);
 644 }
 645
 646 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
 647                                           int len, bool ecc, int page)
 648 {
 649         sunxi_nfc_randomizer_config(nand, page, ecc);
 650         sunxi_nfc_randomizer_enable(nand);
 651         sunxi_nfc_read_buf(nand, buf, len);
 652         sunxi_nfc_randomizer_disable(nand);
 653 }
 654
 655 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
 656 {
 657         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 658         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 659         u32 ecc_ctl;
 660
 661         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 662         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 663                      NFC_ECC_BLOCK_SIZE_MSK);
 664         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 665                    NFC_ECC_PIPELINE;
 666
 667         if (nand->ecc.size == 512)
 668                 ecc_ctl |= NFC_ECC_BLOCK_512;
 669
 670         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 671 }
 672
 673 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
 674 {
 675         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 676
 677         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 678                nfc->regs + NFC_REG_ECC_CTL);
 679 }
 680
 681 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 682 {
 683         buf[0] = user_data;
 684         buf[1] = user_data >> 8;
 685         buf[2] = user_data >> 16;
 686         buf[3] = user_data >> 24;
 687 }
 688
 689 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 690 {
 691         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 692 }
 693
 694 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
 695                                                 int step, bool bbm, int page)
 696 {
 697         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 698
 699         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 700                                    oob);
 701
 702         /* De-randomize the Bad Block Marker. */
 703         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 704                 sunxi_nfc_randomize_bbm(nand, page, oob);
 705 }
 706
 707 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 708                                                 const u8 *oob, int step,
 709                                                 bool bbm, int page)
 710 {
 711         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 712         u8 user_data[4];
 713
 714         /* Randomize the Bad Block Marker. */
 715         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 716                 memcpy(user_data, oob, sizeof(user_data));
 717                 sunxi_nfc_randomize_bbm(nand, page, user_data);
 718                 oob = user_data;
 719         }
 720
 721         writel(sunxi_nfc_buf_to_user_data(oob),
 722                nfc->regs + NFC_REG_USER_DATA(step));
 723 }
 724
 725 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
 726                                           unsigned int *max_bitflips, int ret)
 727 {
 728         struct mtd_info *mtd = nand_to_mtd(nand);
 729
 730         if (ret < 0) {
 731                 mtd->ecc_stats.failed++;
 732         } else {
 733                 mtd->ecc_stats.corrected += ret;
 734                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 735         }
 736 }
 737
 738 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
 739                                     int step, u32 status, bool *erased)
 740 {
 741         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 742         struct nand_ecc_ctrl *ecc = &nand->ecc;
 743         u32 tmp;
 744
 745         *erased = false;
 746
 747         if (status & NFC_ECC_ERR(step))
 748                 return -EBADMSG;
 749
 750         if (status & NFC_ECC_PAT_FOUND(step)) {
 751                 u8 pattern;
 752
 753                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 754                         pattern = 0x0;
 755                 } else {
 756                         pattern = 0xff;
 757                         *erased = true;
 758                 }
 759
 760                 if (data)
 761                         memset(data, pattern, ecc->size);
 762
 763                 if (oob)
 764                         memset(oob, pattern, ecc->bytes + 4);
 765
 766                 return 0;
 767         }
 768
 769         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 770
 771         return NFC_ECC_ERR_CNT(step, tmp);
 772 }
 773
 774 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
 775                                        u8 *data, int data_off,
 776                                        u8 *oob, int oob_off,
 777                                        int *cur_off,
 778                                        unsigned int *max_bitflips,
 779                                        bool bbm, bool oob_required, int page)
 780 {
 781         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 782         struct nand_ecc_ctrl *ecc = &nand->ecc;
 783         int raw_mode = 0;
 784         bool erased;
 785         int ret;
 786
 787         if (*cur_off != data_off)
 788                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 789
 790         sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
 791
 792         if (data_off + ecc->size != oob_off)
 793                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 794
 795         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 796         if (ret)
 797                 return ret;
 798
 799         sunxi_nfc_randomizer_enable(nand);
 800         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 801                nfc->regs + NFC_REG_CMD);
 802
 803         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 804         sunxi_nfc_randomizer_disable(nand);
 805         if (ret)
 806                 return ret;
 807
 808         *cur_off = oob_off + ecc->bytes + 4;
 809
 810         ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
 811                                        readl(nfc->regs + NFC_REG_ECC_ST),
 812                                        &erased);
 813         if (erased)
 814                 return 1;
 815
 816         if (ret < 0) {
 817                 /*
 818                  * Re-read the data with the randomizer disabled to identify
 819                  * bitflips in erased pages.
 820                  */
 821                 if (nand->options & NAND_NEED_SCRAMBLING)
 822                         nand_change_read_column_op(nand, data_off, data,
 823                                                    ecc->size, false);
 824                 else
 825                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 826                                       ecc->size);
 827
 828                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 829                                            false);
 830
 831                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 832                                                   oob, ecc->bytes + 4,
 833                                                   NULL, 0, ecc->strength);
 834                 if (ret >= 0)
 835                         raw_mode = 1;
 836         } else {
 837                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 838
 839                 if (oob_required) {
 840                         nand_change_read_column_op(nand, oob_off, NULL, 0,
 841                                                    false);
 842                         sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
 843                                                       true, page);
 844
 845                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
 846                                                             bbm, page);
 847                 }
 848         }
 849
 850         sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
 851
 852         return raw_mode;
 853 }
 854
 855 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
 856                                             u8 *oob, int *cur_off,
 857                                             bool randomize, int page)
 858 {
 859         struct mtd_info *mtd = nand_to_mtd(nand);
 860         struct nand_ecc_ctrl *ecc = &nand->ecc;
 861         int offset = ((ecc->bytes + 4) * ecc->steps);
 862         int len = mtd->oobsize - offset;
 863
 864         if (len <= 0)
 865                 return;
 866
 867         if (!cur_off || *cur_off != offset)
 868                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 869                                            false);
 870
 871         if (!randomize)
 872                 sunxi_nfc_read_buf(nand, oob + offset, len);
 873         else
 874                 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
 875                                               false, page);
 876
 877         if (cur_off)
 878                 *cur_off = mtd->oobsize + mtd->writesize;
 879 }
 880
 881 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
 882                                             int oob_required, int page,
 883                                             int nchunks)
 884 {
 885         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
 886         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 887         struct mtd_info *mtd = nand_to_mtd(nand);
 888         struct nand_ecc_ctrl *ecc = &nand->ecc;
 889         unsigned int max_bitflips = 0;
 890         int ret, i, raw_mode = 0;
 891         struct scatterlist sg;
 892         u32 status;
 893
 894         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 895         if (ret)
 896                 return ret;
 897
 898         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
 899                                        DMA_FROM_DEVICE, &sg);
 900         if (ret)
 901                 return ret;
 902
 903         sunxi_nfc_hw_ecc_enable(nand);
 904         sunxi_nfc_randomizer_config(nand, page, false);
 905         sunxi_nfc_randomizer_enable(nand);
 906
 907         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
 908                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
 909
 910         dma_async_issue_pending(nfc->dmac);
 911
 912         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
 913                nfc->regs + NFC_REG_CMD);
 914
 915         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 916         if (ret)
 917                 dmaengine_terminate_all(nfc->dmac);
 918
 919         sunxi_nfc_randomizer_disable(nand);
 920         sunxi_nfc_hw_ecc_disable(nand);
 921
 922         sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
 923
 924         if (ret)
 925                 return ret;
 926
 927         status = readl(nfc->regs + NFC_REG_ECC_ST);
 928
 929         for (i = 0; i < nchunks; i++) {
 930                 int data_off = i * ecc->size;
 931                 int oob_off = i * (ecc->bytes + 4);
 932                 u8 *data = buf + data_off;
 933                 u8 *oob = nand->oob_poi + oob_off;
 934                 bool erased;
 935
 936                 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
 937                                                oob_required ? oob : NULL,
 938                                                i, status, &erased);
 939
 940                 /* ECC errors are handled in the second loop. */
 941                 if (ret < 0)
 942                         continue;
 943
 944                 if (oob_required && !erased) {
 945                         /* TODO: use DMA to retrieve OOB */
 946                         nand_change_read_column_op(nand,
 947                                                    mtd->writesize + oob_off,
 948                                                    oob, ecc->bytes + 4, false);
 949
 950                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
 951                                                             !i, page);
 952                 }
 953
 954                 if (erased)
 955                         raw_mode = 1;
 956
 957                 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
 958         }
 959
 960         if (status & NFC_ECC_ERR_MSK) {
 961                 for (i = 0; i < nchunks; i++) {
 962                         int data_off = i * ecc->size;
 963                         int oob_off = i * (ecc->bytes + 4);
 964                         u8 *data = buf + data_off;
 965                         u8 *oob = nand->oob_poi + oob_off;
 966
 967                         if (!(status & NFC_ECC_ERR(i)))
 968                                 continue;
 969
 970                         /*
 971                          * Re-read the data with the randomizer disabled to
 972                          * identify bitflips in erased pages.
 973                          * TODO: use DMA to read page in raw mode
 974                          */
 975                         if (randomized)
 976                                 nand_change_read_column_op(nand, data_off,
 977                                                            data, ecc->size,
 978                                                            false);
 979
 980                         /* TODO: use DMA to retrieve OOB */
 981                         nand_change_read_column_op(nand,
 982                                                    mtd->writesize + oob_off,
 983                                                    oob, ecc->bytes + 4, false);
 984
 985                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
 986                                                           oob, ecc->bytes + 4,
 987                                                           NULL, 0,
 988                                                           ecc->strength);
 989                         if (ret >= 0)
 990                                 raw_mode = 1;
 991
 992                         sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
 993                 }
 994         }
 995
 996         if (oob_required)
 997                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
 998                                                 NULL, !raw_mode,
 999                                                 page);
1000
1001         return max_bitflips;
1002 }
1003
1004 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1005                                         const u8 *data, int data_off,
1006                                         const u8 *oob, int oob_off,
1007                                         int *cur_off, bool bbm,
1008                                         int page)
1009 {
1010         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1011         struct nand_ecc_ctrl *ecc = &nand->ecc;
1012         int ret;
1013
1014         if (data_off != *cur_off)
1015                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1016
1017         sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1018
1019         if (data_off + ecc->size != oob_off)
1020                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1021
1022         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1023         if (ret)
1024                 return ret;
1025
1026         sunxi_nfc_randomizer_enable(nand);
1027         sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1028
1029         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1030                NFC_ACCESS_DIR | NFC_ECC_OP,
1031                nfc->regs + NFC_REG_CMD);
1032
1033         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1034         sunxi_nfc_randomizer_disable(nand);
1035         if (ret)
1036                 return ret;
1037
1038         *cur_off = oob_off + ecc->bytes + 4;
1039
1040         return 0;
1041 }
1042
1043 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1044                                              u8 *oob, int *cur_off,
1045                                              int page)
1046 {
1047         struct mtd_info *mtd = nand_to_mtd(nand);
1048         struct nand_ecc_ctrl *ecc = &nand->ecc;
1049         int offset = ((ecc->bytes + 4) * ecc->steps);
1050         int len = mtd->oobsize - offset;
1051
1052         if (len <= 0)
1053                 return;
1054
1055         if (!cur_off || *cur_off != offset)
1056                 nand_change_write_column_op(nand, offset + mtd->writesize,
1057                                             NULL, 0, false);
1058
1059         sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1060
1061         if (cur_off)
1062                 *cur_off = mtd->oobsize + mtd->writesize;
1063 }
1064
1065 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1066                                       int oob_required, int page)
1067 {
1068         struct mtd_info *mtd = nand_to_mtd(nand);
1069         struct nand_ecc_ctrl *ecc = &nand->ecc;
1070         unsigned int max_bitflips = 0;
1071         int ret, i, cur_off = 0;
1072         bool raw_mode = false;
1073
1074         sunxi_nfc_select_chip(nand, nand->cur_cs);
1075
1076         nand_read_page_op(nand, page, 0, NULL, 0);
1077
1078         sunxi_nfc_hw_ecc_enable(nand);
1079
1080         for (i = 0; i < ecc->steps; i++) {
1081                 int data_off = i * ecc->size;
1082                 int oob_off = i * (ecc->bytes + 4);
1083                 u8 *data = buf + data_off;
1084                 u8 *oob = nand->oob_poi + oob_off;
1085
1086                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1087                                                   oob_off + mtd->writesize,
1088                                                   &cur_off, &max_bitflips,
1089                                                   !i, oob_required, page);
1090                 if (ret < 0)
1091                         return ret;
1092                 else if (ret)
1093                         raw_mode = true;
1094         }
1095
1096         if (oob_required)
1097                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1098                                                 !raw_mode, page);
1099
1100         sunxi_nfc_hw_ecc_disable(nand);
1101
1102         return max_bitflips;
1103 }
1104
1105 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1106                                           int oob_required, int page)
1107 {
1108         int ret;
1109
1110         sunxi_nfc_select_chip(nand, nand->cur_cs);
1111
1112         nand_read_page_op(nand, page, 0, NULL, 0);
1113
1114         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1115                                                nand->ecc.steps);
1116         if (ret >= 0)
1117                 return ret;
1118
1119         /* Fallback to PIO mode */
1120         return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1121 }
1122
1123 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1124                                          u32 data_offs, u32 readlen,
1125                                          u8 *bufpoi, int page)
1126 {
1127         struct mtd_info *mtd = nand_to_mtd(nand);
1128         struct nand_ecc_ctrl *ecc = &nand->ecc;
1129         int ret, i, cur_off = 0;
1130         unsigned int max_bitflips = 0;
1131
1132         sunxi_nfc_select_chip(nand, nand->cur_cs);
1133
1134         nand_read_page_op(nand, page, 0, NULL, 0);
1135
1136         sunxi_nfc_hw_ecc_enable(nand);
1137
1138         for (i = data_offs / ecc->size;
1139              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1140                 int data_off = i * ecc->size;
1141                 int oob_off = i * (ecc->bytes + 4);
1142                 u8 *data = bufpoi + data_off;
1143                 u8 *oob = nand->oob_poi + oob_off;
1144
1145                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1146                                                   oob,
1147                                                   oob_off + mtd->writesize,
1148                                                   &cur_off, &max_bitflips, !i,
1149                                                   false, page);
1150                 if (ret < 0)
1151                         return ret;
1152         }
1153
1154         sunxi_nfc_hw_ecc_disable(nand);
1155
1156         return max_bitflips;
1157 }
1158
1159 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1160                                              u32 data_offs, u32 readlen,
1161                                              u8 *buf, int page)
1162 {
1163         int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1164         int ret;
1165
1166         sunxi_nfc_select_chip(nand, nand->cur_cs);
1167
1168         nand_read_page_op(nand, page, 0, NULL, 0);
1169
1170         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1171         if (ret >= 0)
1172                 return ret;
1173
1174         /* Fallback to PIO mode */
1175         return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1176                                              buf, page);
1177 }
1178
1179 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1180                                        const uint8_t *buf, int oob_required,
1181                                        int page)
1182 {
1183         struct mtd_info *mtd = nand_to_mtd(nand);
1184         struct nand_ecc_ctrl *ecc = &nand->ecc;
1185         int ret, i, cur_off = 0;
1186
1187         sunxi_nfc_select_chip(nand, nand->cur_cs);
1188
1189         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1190
1191         sunxi_nfc_hw_ecc_enable(nand);
1192
1193         for (i = 0; i < ecc->steps; i++) {
1194                 int data_off = i * ecc->size;
1195                 int oob_off = i * (ecc->bytes + 4);
1196                 const u8 *data = buf + data_off;
1197                 const u8 *oob = nand->oob_poi + oob_off;
1198
1199                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1200                                                    oob_off + mtd->writesize,
1201                                                    &cur_off, !i, page);
1202                 if (ret)
1203                         return ret;
1204         }
1205
1206         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1207                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1208                                                  &cur_off, page);
1209
1210         sunxi_nfc_hw_ecc_disable(nand);
1211
1212         return nand_prog_page_end_op(nand);
1213 }
1214
1215 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1216                                           u32 data_offs, u32 data_len,
1217                                           const u8 *buf, int oob_required,
1218                                           int page)
1219 {
1220         struct mtd_info *mtd = nand_to_mtd(nand);
1221         struct nand_ecc_ctrl *ecc = &nand->ecc;
1222         int ret, i, cur_off = 0;
1223
1224         sunxi_nfc_select_chip(nand, nand->cur_cs);
1225
1226         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1227
1228         sunxi_nfc_hw_ecc_enable(nand);
1229
1230         for (i = data_offs / ecc->size;
1231              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1232                 int data_off = i * ecc->size;
1233                 int oob_off = i * (ecc->bytes + 4);
1234                 const u8 *data = buf + data_off;
1235                 const u8 *oob = nand->oob_poi + oob_off;
1236
1237                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1238                                                    oob_off + mtd->writesize,
1239                                                    &cur_off, !i, page);
1240                 if (ret)
1241                         return ret;
1242         }
1243
1244         sunxi_nfc_hw_ecc_disable(nand);
1245
1246         return nand_prog_page_end_op(nand);
1247 }
1248
1249 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1250                                            const u8 *buf,
1251                                            int oob_required,
1252                                            int page)
1253 {
1254         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1255         struct nand_ecc_ctrl *ecc = &nand->ecc;
1256         struct scatterlist sg;
1257         int ret, i;
1258
1259         sunxi_nfc_select_chip(nand, nand->cur_cs);
1260
1261         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1262         if (ret)
1263                 return ret;
1264
1265         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1266                                        DMA_TO_DEVICE, &sg);
1267         if (ret)
1268                 goto pio_fallback;
1269
1270         for (i = 0; i < ecc->steps; i++) {
1271                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1272
1273                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1274         }
1275
1276         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1277
1278         sunxi_nfc_hw_ecc_enable(nand);
1279         sunxi_nfc_randomizer_config(nand, page, false);
1280         sunxi_nfc_randomizer_enable(nand);
1281
1282         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1283                nfc->regs + NFC_REG_WCMD_SET);
1284
1285         dma_async_issue_pending(nfc->dmac);
1286
1287         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1288                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1289                nfc->regs + NFC_REG_CMD);
1290
1291         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1292         if (ret)
1293                 dmaengine_terminate_all(nfc->dmac);
1294
1295         sunxi_nfc_randomizer_disable(nand);
1296         sunxi_nfc_hw_ecc_disable(nand);
1297
1298         sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1299
1300         if (ret)
1301                 return ret;
1302
1303         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1304                 /* TODO: use DMA to transfer extra OOB bytes ? */
1305                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1306                                                  NULL, page);
1307
1308         return nand_prog_page_end_op(nand);
1309
1310 pio_fallback:
1311         return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1312 }
1313
1314 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1315 {
1316         nand->pagebuf = -1;
1317
1318         return nand->ecc.read_page(nand, nand->data_buf, 1, page);
1319 }
1320
1321 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1322 {
1323         struct mtd_info *mtd = nand_to_mtd(nand);
1324         int ret;
1325
1326         nand->pagebuf = -1;
1327
1328         memset(nand->data_buf, 0xff, mtd->writesize);
1329         ret = nand->ecc.write_page(nand, nand->data_buf, 1, page);
1330         if (ret)
1331                 return ret;
1332
1333         /* Send command to program the OOB data */
1334         return nand_prog_page_end_op(nand);
1335 }
1336
1337 static const s32 tWB_lut[] = {6, 12, 16, 20};
1338 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1339
1340 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1341                 u32 clk_period)
1342 {
1343         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1344         int i;
1345
1346         for (i = 0; i < lut_size; i++) {
1347                 if (clk_cycles <= lut[i])
1348                         return i;
1349         }
1350
1351         /* Doesn't fit */
1352         return -EINVAL;
1353 }
1354
1355 #define sunxi_nand_lookup_timing(l, p, c) \
1356                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1357
1358 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline,
1359                                         const struct nand_data_interface *conf)
1360 {
1361         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1362         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1363         const struct nand_sdr_timings *timings;
1364         u32 min_clk_period = 0;
1365         s32 tWB, tADL, tWHR, tRHW, tCAD;
1366         long real_clk_rate;
1367
1368         timings = nand_get_sdr_timings(conf);
1369         if (IS_ERR(timings))
1370                 return -ENOTSUPP;
1371
1372         /* T1 <=> tCLS */
1373         if (timings->tCLS_min > min_clk_period)
1374                 min_clk_period = timings->tCLS_min;
1375
1376         /* T2 <=> tCLH */
1377         if (timings->tCLH_min > min_clk_period)
1378                 min_clk_period = timings->tCLH_min;
1379
1380         /* T3 <=> tCS */
1381         if (timings->tCS_min > min_clk_period)
1382                 min_clk_period = timings->tCS_min;
1383
1384         /* T4 <=> tCH */
1385         if (timings->tCH_min > min_clk_period)
1386                 min_clk_period = timings->tCH_min;
1387
1388         /* T5 <=> tWP */
1389         if (timings->tWP_min > min_clk_period)
1390                 min_clk_period = timings->tWP_min;
1391
1392         /* T6 <=> tWH */
1393         if (timings->tWH_min > min_clk_period)
1394                 min_clk_period = timings->tWH_min;
1395
1396         /* T7 <=> tALS */
1397         if (timings->tALS_min > min_clk_period)
1398                 min_clk_period = timings->tALS_min;
1399
1400         /* T8 <=> tDS */
1401         if (timings->tDS_min > min_clk_period)
1402                 min_clk_period = timings->tDS_min;
1403
1404         /* T9 <=> tDH */
1405         if (timings->tDH_min > min_clk_period)
1406                 min_clk_period = timings->tDH_min;
1407
1408         /* T10 <=> tRR */
1409         if (timings->tRR_min > (min_clk_period * 3))
1410                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1411
1412         /* T11 <=> tALH */
1413         if (timings->tALH_min > min_clk_period)
1414                 min_clk_period = timings->tALH_min;
1415
1416         /* T12 <=> tRP */
1417         if (timings->tRP_min > min_clk_period)
1418                 min_clk_period = timings->tRP_min;
1419
1420         /* T13 <=> tREH */
1421         if (timings->tREH_min > min_clk_period)
1422                 min_clk_period = timings->tREH_min;
1423
1424         /* T14 <=> tRC */
1425         if (timings->tRC_min > (min_clk_period * 2))
1426                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1427
1428         /* T15 <=> tWC */
1429         if (timings->tWC_min > (min_clk_period * 2))
1430                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1431
1432         /* T16 - T19 + tCAD */
1433         if (timings->tWB_max > (min_clk_period * 20))
1434                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1435
1436         if (timings->tADL_min > (min_clk_period * 32))
1437                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1438
1439         if (timings->tWHR_min > (min_clk_period * 32))
1440                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1441
1442         if (timings->tRHW_min > (min_clk_period * 20))
1443                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1444
1445         /*
1446          * In non-EDO, tREA should be less than tRP to guarantee that the
1447          * controller does not sample the IO lines too early. Unfortunately,
1448          * the sunxi NAND controller does not allow us to have different
1449          * values for tRP and tREH (tRP = tREH = tRW / 2).
1450          *
1451          * We have 2 options to overcome this limitation:
1452          *
1453          * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1454          * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1455          */
1456         if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1457                 min_clk_period = timings->tREA_max;
1458
1459         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1460                                         min_clk_period);
1461         if (tWB < 0) {
1462                 dev_err(nfc->dev, "unsupported tWB\n");
1463                 return tWB;
1464         }
1465
1466         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1467         if (tADL > 3) {
1468                 dev_err(nfc->dev, "unsupported tADL\n");
1469                 return -EINVAL;
1470         }
1471
1472         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1473         if (tWHR > 3) {
1474                 dev_err(nfc->dev, "unsupported tWHR\n");
1475                 return -EINVAL;
1476         }
1477
1478         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1479                                         min_clk_period);
1480         if (tRHW < 0) {
1481                 dev_err(nfc->dev, "unsupported tRHW\n");
1482                 return tRHW;
1483         }
1484
1485         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1486                 return 0;
1487
1488         /*
1489          * TODO: according to ONFI specs this value only applies for DDR NAND,
1490          * but Allwinner seems to set this to 0x7. Mimic them for now.
1491          */
1492         tCAD = 0x7;
1493
1494         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1495         sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1496
1497         /* Convert min_clk_period from picoseconds to nanoseconds */
1498         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1499
1500         /*
1501          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1502          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1503          * This new formula was verified with a scope and validated by
1504          * Allwinner engineers.
1505          */
1506         sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1507         real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1508         if (real_clk_rate <= 0) {
1509                 dev_err(nfc->dev, "Unable to round clk %lu\n",
1510                         sunxi_nand->clk_rate);
1511                 return -EINVAL;
1512         }
1513
1514         sunxi_nand->timing_ctl = 0;
1515
1516         /*
1517          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1518          * output cycle timings shall be used if the host drives tRC less than
1519          * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1520          */
1521         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1522         if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1523                 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1524
1525         return 0;
1526 }
1527
1528 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1529                                     struct mtd_oob_region *oobregion)
1530 {
1531         struct nand_chip *nand = mtd_to_nand(mtd);
1532         struct nand_ecc_ctrl *ecc = &nand->ecc;
1533
1534         if (section >= ecc->steps)
1535                 return -ERANGE;
1536
1537         oobregion->offset = section * (ecc->bytes + 4) + 4;
1538         oobregion->length = ecc->bytes;
1539
1540         return 0;
1541 }
1542
1543 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1544                                      struct mtd_oob_region *oobregion)
1545 {
1546         struct nand_chip *nand = mtd_to_nand(mtd);
1547         struct nand_ecc_ctrl *ecc = &nand->ecc;
1548
1549         if (section > ecc->steps)
1550                 return -ERANGE;
1551
1552         /*
1553          * The first 2 bytes are used for BB markers, hence we
1554          * only have 2 bytes available in the first user data
1555          * section.
1556          */
1557         if (!section && ecc->mode == NAND_ECC_HW) {
1558                 oobregion->offset = 2;
1559                 oobregion->length = 2;
1560
1561                 return 0;
1562         }
1563
1564         oobregion->offset = section * (ecc->bytes + 4);
1565
1566         if (section < ecc->steps)
1567                 oobregion->length = 4;
1568         else
1569                 oobregion->offset = mtd->oobsize - oobregion->offset;
1570
1571         return 0;
1572 }
1573
1574 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1575         .ecc = sunxi_nand_ooblayout_ecc,
1576         .free = sunxi_nand_ooblayout_free,
1577 };
1578
1579 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1580 {
1581         kfree(ecc->priv);
1582 }
1583
1584 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1585                                        struct nand_ecc_ctrl *ecc,
1586                                        struct device_node *np)
1587 {
1588         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1589         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1590         struct mtd_info *mtd = nand_to_mtd(nand);
1591         struct sunxi_nand_hw_ecc *data;
1592         int nsectors;
1593         int ret;
1594         int i;
1595
1596         if (ecc->options & NAND_ECC_MAXIMIZE) {
1597                 int bytes;
1598
1599                 ecc->size = 1024;
1600                 nsectors = mtd->writesize / ecc->size;
1601
1602                 /* Reserve 2 bytes for the BBM */
1603                 bytes = (mtd->oobsize - 2) / nsectors;
1604
1605                 /* 4 non-ECC bytes are added before each ECC bytes section */
1606                 bytes -= 4;
1607
1608                 /* and bytes has to be even. */
1609                 if (bytes % 2)
1610                         bytes--;
1611
1612                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1613
1614                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1615                         if (strengths[i] > ecc->strength)
1616                                 break;
1617                 }
1618
1619                 if (!i)
1620                         ecc->strength = 0;
1621                 else
1622                         ecc->strength = strengths[i - 1];
1623         }
1624
1625         if (ecc->size != 512 && ecc->size != 1024)
1626                 return -EINVAL;
1627
1628         data = kzalloc(sizeof(*data), GFP_KERNEL);
1629         if (!data)
1630                 return -ENOMEM;
1631
1632         /* Prefer 1k ECC chunk over 512 ones */
1633         if (ecc->size == 512 && mtd->writesize > 512) {
1634                 ecc->size = 1024;
1635                 ecc->strength *= 2;
1636         }
1637
1638         /* Add ECC info retrieval from DT */
1639         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1640                 if (ecc->strength <= strengths[i]) {
1641                         /*
1642                          * Update ecc->strength value with the actual strength
1643                          * that will be used by the ECC engine.
1644                          */
1645                         ecc->strength = strengths[i];
1646                         break;
1647                 }
1648         }
1649
1650         if (i >= ARRAY_SIZE(strengths)) {
1651                 dev_err(nfc->dev, "unsupported strength\n");
1652                 ret = -ENOTSUPP;
1653                 goto err;
1654         }
1655
1656         data->mode = i;
1657
1658         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1659         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1660
1661         /* HW ECC always work with even numbers of ECC bytes */
1662         ecc->bytes = ALIGN(ecc->bytes, 2);
1663
1664         nsectors = mtd->writesize / ecc->size;
1665
1666         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1667                 ret = -EINVAL;
1668                 goto err;
1669         }
1670
1671         ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1672         ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1673         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1674         ecc->priv = data;
1675
1676         if (nfc->dmac) {
1677                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1678                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1679                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1680                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1681         } else {
1682                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1683                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1684                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1685         }
1686
1687         /* TODO: support DMA for raw accesses and subpage write */
1688         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1689         ecc->read_oob_raw = nand_read_oob_std;
1690         ecc->write_oob_raw = nand_write_oob_std;
1691
1692         return 0;
1693
1694 err:
1695         kfree(data);
1696
1697         return ret;
1698 }
1699
1700 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1701 {
1702         switch (ecc->mode) {
1703         case NAND_ECC_HW:
1704                 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1705                 break;
1706         case NAND_ECC_NONE:
1707         default:
1708                 break;
1709         }
1710 }
1711
1712 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1713 {
1714         struct nand_ecc_ctrl *ecc = &nand->ecc;
1715         struct device_node *np = nand_get_flash_node(nand);
1716         int ret;
1717
1718         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1719                 nand->bbt_options |= NAND_BBT_NO_OOB;
1720
1721         if (nand->options & NAND_NEED_SCRAMBLING)
1722                 nand->options |= NAND_NO_SUBPAGE_WRITE;
1723
1724         nand->options |= NAND_SUBPAGE_READ;
1725
1726         if (!ecc->size) {
1727                 ecc->size = nand->ecc_step_ds;
1728                 ecc->strength = nand->ecc_strength_ds;
1729         }
1730
1731         if (!ecc->size || !ecc->strength)
1732                 return -EINVAL;
1733
1734         switch (ecc->mode) {
1735         case NAND_ECC_HW:
1736                 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1737                 if (ret)
1738                         return ret;
1739                 break;
1740         case NAND_ECC_NONE:
1741         case NAND_ECC_SOFT:
1742                 break;
1743         default:
1744                 return -EINVAL;
1745         }
1746
1747         return 0;
1748 }
1749
1750 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1751                                 const struct nand_subop *subop)
1752 {
1753         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1754         u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1755         unsigned int i, j, remaining, start;
1756         void *inbuf = NULL;
1757         int ret;
1758
1759         for (i = 0; i < subop->ninstrs; i++) {
1760                 const struct nand_op_instr *instr = &subop->instrs[i];
1761
1762                 switch (instr->type) {
1763                 case NAND_OP_CMD_INSTR:
1764                         if (cmd & NFC_SEND_CMD1) {
1765                                 if (WARN_ON(cmd & NFC_SEND_CMD2))
1766                                         return -EINVAL;
1767
1768                                 cmd |= NFC_SEND_CMD2;
1769                                 extcmd |= instr->ctx.cmd.opcode;
1770                         } else {
1771                                 cmd |= NFC_SEND_CMD1 |
1772                                        NFC_CMD(instr->ctx.cmd.opcode);
1773                         }
1774                         break;
1775
1776                 case NAND_OP_ADDR_INSTR:
1777                         remaining = nand_subop_get_num_addr_cyc(subop, i);
1778                         start = nand_subop_get_addr_start_off(subop, i);
1779                         for (j = 0; j < 8 && j + start < remaining; j++) {
1780                                 u32 addr = instr->ctx.addr.addrs[j + start];
1781
1782                                 addrs[j / 4] |= addr << (j % 4) * 8;
1783                         }
1784
1785                         if (j)
1786                                 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1787
1788                         break;
1789
1790                 case NAND_OP_DATA_IN_INSTR:
1791                 case NAND_OP_DATA_OUT_INSTR:
1792                         start = nand_subop_get_data_start_off(subop, i);
1793                         remaining = nand_subop_get_data_len(subop, i);
1794                         cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1795                         cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1796
1797                         if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1798                                 cmd |= NFC_ACCESS_DIR;
1799                                 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1800                                             instr->ctx.data.buf.out + start,
1801                                             cnt);
1802                         } else {
1803                                 inbuf = instr->ctx.data.buf.in + start;
1804                         }
1805
1806                         break;
1807
1808                 case NAND_OP_WAITRDY_INSTR:
1809                         cmd |= NFC_WAIT_FLAG;
1810                         break;
1811                 }
1812         }
1813
1814         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1815         if (ret)
1816                 return ret;
1817
1818         if (cmd & NFC_SEND_ADR) {
1819                 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1820                 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1821         }
1822
1823         if (cmd & NFC_SEND_CMD2)
1824                 writel(extcmd,
1825                        nfc->regs +
1826                        (cmd & NFC_ACCESS_DIR ?
1827                         NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1828
1829         if (cmd & NFC_DATA_TRANS)
1830                 writel(cnt, nfc->regs + NFC_REG_CNT);
1831
1832         writel(cmd, nfc->regs + NFC_REG_CMD);
1833
1834         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1835                                     !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1836                                     0);
1837         if (ret)
1838                 return ret;
1839
1840         if (inbuf)
1841                 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1842
1843         return 0;
1844 }
1845
1846 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1847                                   const struct nand_subop *subop)
1848 {
1849         return nand_soft_waitrdy(nand,
1850                                  subop->instrs[0].ctx.waitrdy.timeout_ms);
1851 }
1852
1853 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1854         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1855                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1856                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1857                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1858                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1859                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1860         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1861                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1862                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1863                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1864                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1865                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1866 );
1867
1868 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1869         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1870                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1871                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1872                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1873                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1874         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1875                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1876                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1877                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1878                                NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1879         NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1880                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1881 );
1882
1883 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1884                              const struct nand_operation *op, bool check_only)
1885 {
1886         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1887         const struct nand_op_parser *parser;
1888
1889         sunxi_nfc_select_chip(nand, op->cs);
1890
1891         if (sunxi_nand->sels[op->cs].rb >= 0)
1892                 parser = &sunxi_nfc_op_parser;
1893         else
1894                 parser = &sunxi_nfc_norb_op_parser;
1895
1896         return nand_op_parser_exec_op(nand, parser, op, check_only);
1897 }
1898
1899 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1900         .attach_chip = sunxi_nand_attach_chip,
1901         .setup_data_interface = sunxi_nfc_setup_data_interface,
1902         .exec_op = sunxi_nfc_exec_op,
1903 };
1904
1905 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1906                                 struct device_node *np)
1907 {
1908         struct sunxi_nand_chip *sunxi_nand;
1909         struct mtd_info *mtd;
1910         struct nand_chip *nand;
1911         int nsels;
1912         int ret;
1913         int i;
1914         u32 tmp;
1915
1916         if (!of_get_property(np, "reg", &nsels))
1917                 return -EINVAL;
1918
1919         nsels /= sizeof(u32);
1920         if (!nsels) {
1921                 dev_err(dev, "invalid reg property size\n");
1922                 return -EINVAL;
1923         }
1924
1925         sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1926                                   GFP_KERNEL);
1927         if (!sunxi_nand) {
1928                 dev_err(dev, "could not allocate chip\n");
1929                 return -ENOMEM;
1930         }
1931
1932         sunxi_nand->nsels = nsels;
1933
1934         for (i = 0; i < nsels; i++) {
1935                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1936                 if (ret) {
1937                         dev_err(dev, "could not retrieve reg property: %d\n",
1938                                 ret);
1939                         return ret;
1940                 }
1941
1942                 if (tmp > NFC_MAX_CS) {
1943                         dev_err(dev,
1944                                 "invalid reg value: %u (max CS = 7)\n",
1945                                 tmp);
1946                         return -EINVAL;
1947                 }
1948
1949                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1950                         dev_err(dev, "CS %d already assigned\n", tmp);
1951                         return -EINVAL;
1952                 }
1953
1954                 sunxi_nand->sels[i].cs = tmp;
1955
1956                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1957                     tmp < 2)
1958                         sunxi_nand->sels[i].rb = tmp;
1959                 else
1960                         sunxi_nand->sels[i].rb = -1;
1961         }
1962
1963         nand = &sunxi_nand->nand;
1964         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1965         nand->controller = &nfc->controller;
1966         nand->controller->ops = &sunxi_nand_controller_ops;
1967
1968         /*
1969          * Set the ECC mode to the default value in case nothing is specified
1970          * in the DT.
1971          */
1972         nand->ecc.mode = NAND_ECC_HW;
1973         nand_set_flash_node(nand, np);
1974
1975         mtd = nand_to_mtd(nand);
1976         mtd->dev.parent = dev;
1977
1978         ret = nand_scan(nand, nsels);
1979         if (ret)
1980                 return ret;
1981
1982         ret = mtd_device_register(mtd, NULL, 0);
1983         if (ret) {
1984                 dev_err(dev, "failed to register mtd device: %d\n", ret);
1985                 nand_release(nand);
1986                 return ret;
1987         }
1988
1989         list_add_tail(&sunxi_nand->node, &nfc->chips);
1990
1991         return 0;
1992 }
1993
1994 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
1995 {
1996         struct device_node *np = dev->of_node;
1997         struct device_node *nand_np;
1998         int nchips = of_get_child_count(np);
1999         int ret;
2000
2001         if (nchips > 8) {
2002                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2003                 return -EINVAL;
2004         }
2005
2006         for_each_child_of_node(np, nand_np) {
2007                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2008                 if (ret) {
2009                         of_node_put(nand_np);
2010                         return ret;
2011                 }
2012         }
2013
2014         return 0;
2015 }
2016
2017 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2018 {
2019         struct sunxi_nand_chip *sunxi_nand;
2020
2021         while (!list_empty(&nfc->chips)) {
2022                 sunxi_nand = list_first_entry(&nfc->chips,
2023                                               struct sunxi_nand_chip,
2024                                               node);
2025                 nand_release(&sunxi_nand->nand);
2026                 sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc);
2027                 list_del(&sunxi_nand->node);
2028         }
2029 }
2030
2031 static int sunxi_nfc_probe(struct platform_device *pdev)
2032 {
2033         struct device *dev = &pdev->dev;
2034         struct resource *r;
2035         struct sunxi_nfc *nfc;
2036         int irq;
2037         int ret;
2038
2039         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2040         if (!nfc)
2041                 return -ENOMEM;
2042
2043         nfc->dev = dev;
2044         nand_controller_init(&nfc->controller);
2045         INIT_LIST_HEAD(&nfc->chips);
2046
2047         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2048         nfc->regs = devm_ioremap_resource(dev, r);
2049         if (IS_ERR(nfc->regs))
2050                 return PTR_ERR(nfc->regs);
2051
2052         irq = platform_get_irq(pdev, 0);
2053         if (irq < 0) {
2054                 dev_err(dev, "failed to retrieve irq\n");
2055                 return irq;
2056         }
2057
2058         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2059         if (IS_ERR(nfc->ahb_clk)) {
2060                 dev_err(dev, "failed to retrieve ahb clk\n");
2061                 return PTR_ERR(nfc->ahb_clk);
2062         }
2063
2064         ret = clk_prepare_enable(nfc->ahb_clk);
2065         if (ret)
2066                 return ret;
2067
2068         nfc->mod_clk = devm_clk_get(dev, "mod");
2069         if (IS_ERR(nfc->mod_clk)) {
2070                 dev_err(dev, "failed to retrieve mod clk\n");
2071                 ret = PTR_ERR(nfc->mod_clk);
2072                 goto out_ahb_clk_unprepare;
2073         }
2074
2075         ret = clk_prepare_enable(nfc->mod_clk);
2076         if (ret)
2077                 goto out_ahb_clk_unprepare;
2078
2079         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2080         if (IS_ERR(nfc->reset)) {
2081                 ret = PTR_ERR(nfc->reset);
2082                 goto out_mod_clk_unprepare;
2083         }
2084
2085         ret = reset_control_deassert(nfc->reset);
2086         if (ret) {
2087                 dev_err(dev, "reset err %d\n", ret);
2088                 goto out_mod_clk_unprepare;
2089         }
2090
2091         ret = sunxi_nfc_rst(nfc);
2092         if (ret)
2093                 goto out_ahb_reset_reassert;
2094
2095         writel(0, nfc->regs + NFC_REG_INT);
2096         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2097                                0, "sunxi-nand", nfc);
2098         if (ret)
2099                 goto out_ahb_reset_reassert;
2100
2101         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2102         if (nfc->dmac) {
2103                 struct dma_slave_config dmac_cfg = { };
2104
2105                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2106                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2107                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2108                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2109                 dmac_cfg.src_maxburst = 4;
2110                 dmac_cfg.dst_maxburst = 4;
2111                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2112         } else {
2113                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2114         }
2115
2116         platform_set_drvdata(pdev, nfc);
2117
2118         ret = sunxi_nand_chips_init(dev, nfc);
2119         if (ret) {
2120                 dev_err(dev, "failed to init nand chips\n");
2121                 goto out_release_dmac;
2122         }
2123
2124         return 0;
2125
2126 out_release_dmac:
2127         if (nfc->dmac)
2128                 dma_release_channel(nfc->dmac);
2129 out_ahb_reset_reassert:
2130         reset_control_assert(nfc->reset);
2131 out_mod_clk_unprepare:
2132         clk_disable_unprepare(nfc->mod_clk);
2133 out_ahb_clk_unprepare:
2134         clk_disable_unprepare(nfc->ahb_clk);
2135
2136         return ret;
2137 }
2138
2139 static int sunxi_nfc_remove(struct platform_device *pdev)
2140 {
2141         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2142
2143         sunxi_nand_chips_cleanup(nfc);
2144
2145         reset_control_assert(nfc->reset);
2146
2147         if (nfc->dmac)
2148                 dma_release_channel(nfc->dmac);
2149         clk_disable_unprepare(nfc->mod_clk);
2150         clk_disable_unprepare(nfc->ahb_clk);
2151
2152         return 0;
2153 }
2154
2155 static const struct of_device_id sunxi_nfc_ids[] = {
2156         { .compatible = "allwinner,sun4i-a10-nand" },
2157         { /* sentinel */ }
2158 };
2159 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2160
2161 static struct platform_driver sunxi_nfc_driver = {
2162         .driver = {
2163                 .name = "sunxi_nand",
2164                 .of_match_table = sunxi_nfc_ids,
2165         },
2166         .probe = sunxi_nfc_probe,
2167         .remove = sunxi_nfc_remove,
2168 };
2169 module_platform_driver(sunxi_nfc_driver);
2170
2171 MODULE_LICENSE("GPL");
2172 MODULE_AUTHOR("Boris BREZILLON");
2173 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2174 MODULE_ALIAS("platform:sunxi_nand");