drivers/mtd/nand/raw/meson_nand.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
   2 /*
   3  * Amlogic Meson Nand Flash Controller Driver
   4  *
   5  * Copyright (c) 2018 Amlogic, inc.
   6  * Author: Liang Yang <liang.yang@amlogic.com>
   7  */
   8
   9 #include <linux/platform_device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/clk.h>
  13 #include <linux/clk-provider.h>
  14 #include <linux/mtd/rawnand.h>
  15 #include <linux/mtd/mtd.h>
  16 #include <linux/mfd/syscon.h>
  17 #include <linux/regmap.h>
  18 #include <linux/slab.h>
  19 #include <linux/module.h>
  20 #include <linux/iopoll.h>
  21 #include <linux/of.h>
  22 #include <linux/sched/task_stack.h>
  23
  24 #define NFC_REG_CMD             0x00
  25 #define NFC_CMD_IDLE            (0xc << 14)
  26 #define NFC_CMD_CLE             (0x5 << 14)
  27 #define NFC_CMD_ALE             (0x6 << 14)
  28 #define NFC_CMD_ADL             ((0 << 16) | (3 << 20))
  29 #define NFC_CMD_ADH             ((1 << 16) | (3 << 20))
  30 #define NFC_CMD_AIL             ((2 << 16) | (3 << 20))
  31 #define NFC_CMD_AIH             ((3 << 16) | (3 << 20))
  32 #define NFC_CMD_SEED            ((8 << 16) | (3 << 20))
  33 #define NFC_CMD_M2N             ((0 << 17) | (2 << 20))
  34 #define NFC_CMD_N2M             ((1 << 17) | (2 << 20))
  35 #define NFC_CMD_RB              BIT(20)
  36 #define NFC_CMD_SCRAMBLER_ENABLE        BIT(19)
  37 #define NFC_CMD_SCRAMBLER_DISABLE       0
  38 #define NFC_CMD_SHORTMODE_DISABLE       0
  39 #define NFC_CMD_RB_INT          BIT(14)
  40 #define NFC_CMD_RB_INT_NO_PIN   ((0xb << 10) | BIT(18) | BIT(16))
  41
  42 #define NFC_CMD_GET_SIZE(x)     (((x) >> 22) & GENMASK(4, 0))
  43
  44 #define NFC_REG_CFG             0x04
  45 #define NFC_REG_DADR            0x08
  46 #define NFC_REG_IADR            0x0c
  47 #define NFC_REG_BUF             0x10
  48 #define NFC_REG_INFO            0x14
  49 #define NFC_REG_DC              0x18
  50 #define NFC_REG_ADR             0x1c
  51 #define NFC_REG_DL              0x20
  52 #define NFC_REG_DH              0x24
  53 #define NFC_REG_CADR            0x28
  54 #define NFC_REG_SADR            0x2c
  55 #define NFC_REG_PINS            0x30
  56 #define NFC_REG_VER             0x38
  57
  58 #define NFC_RB_IRQ_EN           BIT(21)
  59
  60 #define CLK_DIV_SHIFT           0
  61 #define CLK_DIV_WIDTH           6
  62
  63 #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)       \
  64         (                                                               \
  65                 (cmd_dir)                       |                       \
  66                 ((ran) << 19)                   |                       \
  67                 ((bch) << 14)                   |                       \
  68                 ((short_mode) << 13)            |                       \
  69                 (((page_size) & 0x7f) << 6)     |                       \
  70                 ((pages) & 0x3f)                                        \
  71         )
  72
  73 #define GENCMDDADDRL(adl, addr)         ((adl) | ((addr) & 0xffff))
  74 #define GENCMDDADDRH(adh, addr)         ((adh) | (((addr) >> 16) & 0xffff))
  75 #define GENCMDIADDRL(ail, addr)         ((ail) | ((addr) & 0xffff))
  76 #define GENCMDIADDRH(aih, addr)         ((aih) | (((addr) >> 16) & 0xffff))
  77
  78 #define DMA_DIR(dir)            ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
  79 #define DMA_ADDR_ALIGN          8
  80
  81 #define ECC_CHECK_RETURN_FF     (-1)
  82
  83 #define NAND_CE0                (0xe << 10)
  84 #define NAND_CE1                (0xd << 10)
  85
  86 #define DMA_BUSY_TIMEOUT        0x100000
  87 #define CMD_FIFO_EMPTY_TIMEOUT  1000
  88
  89 #define MAX_CE_NUM              2
  90
  91 /* eMMC clock register, misc control */
  92 #define CLK_SELECT_NAND         BIT(31)
  93
  94 #define NFC_CLK_CYCLE           6
  95
  96 /* nand flash controller delay 3 ns */
  97 #define NFC_DEFAULT_DELAY       3000
  98
  99 #define ROW_ADDER(page, index)  (((page) >> (8 * (index))) & 0xff)
 100 #define MAX_CYCLE_ADDRS         5
 101 #define DIRREAD                 1
 102 #define DIRWRITE                0
 103
 104 #define ECC_PARITY_BCH8_512B    14
 105 #define ECC_COMPLETE            BIT(31)
 106 #define ECC_ERR_CNT(x)          (((x) >> 24) & GENMASK(5, 0))
 107 #define ECC_ZERO_CNT(x)         (((x) >> 16) & GENMASK(5, 0))
 108 #define ECC_UNCORRECTABLE       0x3f
 109
 110 #define PER_INFO_BYTE           8
 111
 112 #define NFC_CMD_RAW_LEN GENMASK(13, 0)
 113
 114 #define NFC_COLUMN_ADDR_0       0
 115 #define NFC_COLUMN_ADDR_1       0
 116
 117 struct meson_nfc_nand_chip {
 118         struct list_head node;
 119         struct nand_chip nand;
 120         unsigned long clk_rate;
 121         unsigned long level1_divider;
 122         u32 bus_timing;
 123         u32 twb;
 124         u32 tadl;
 125         u32 tbers_max;
 126
 127         u32 bch_mode;
 128         u8 *data_buf;
 129         __le64 *info_buf;
 130         u32 nsels;
 131         u8 sels[];
 132 };
 133
 134 struct meson_nand_ecc {
 135         u32 bch;
 136         u32 strength;
 137         u32 size;
 138 };
 139
 140 struct meson_nfc_data {
 141         const struct nand_ecc_caps *ecc_caps;
 142 };
 143
 144 struct meson_nfc_param {
 145         u32 chip_select;
 146         u32 rb_select;
 147 };
 148
 149 struct nand_rw_cmd {
 150         u32 cmd0;
 151         u32 addrs[MAX_CYCLE_ADDRS];
 152         u32 cmd1;
 153 };
 154
 155 struct nand_timing {
 156         u32 twb;
 157         u32 tadl;
 158         u32 tbers_max;
 159 };
 160
 161 struct meson_nfc {
 162         struct nand_controller controller;
 163         struct clk *core_clk;
 164         struct clk *device_clk;
 165         struct clk *nand_clk;
 166         struct clk_divider nand_divider;
 167
 168         unsigned long clk_rate;
 169         u32 bus_timing;
 170
 171         struct device *dev;
 172         void __iomem *reg_base;
 173         void __iomem *reg_clk;
 174         struct completion completion;
 175         struct list_head chips;
 176         const struct meson_nfc_data *data;
 177         struct meson_nfc_param param;
 178         struct nand_timing timing;
 179         union {
 180                 int cmd[32];
 181                 struct nand_rw_cmd rw;
 182         } cmdfifo;
 183
 184         dma_addr_t daddr;
 185         dma_addr_t iaddr;
 186         u32 info_bytes;
 187
 188         unsigned long assigned_cs;
 189         bool no_rb_pin;
 190 };
 191
 192 enum {
 193         NFC_ECC_BCH8_512        = 1,
 194         NFC_ECC_BCH8_1K,
 195         NFC_ECC_BCH24_1K,
 196         NFC_ECC_BCH30_1K,
 197         NFC_ECC_BCH40_1K,
 198         NFC_ECC_BCH50_1K,
 199         NFC_ECC_BCH60_1K,
 200 };
 201
 202 #define MESON_ECC_DATA(b, s, sz)        { .bch = (b), .strength = (s), .size = (sz) }
 203
 204 static struct meson_nand_ecc meson_ecc[] = {
 205         MESON_ECC_DATA(NFC_ECC_BCH8_512, 8,  512),
 206         MESON_ECC_DATA(NFC_ECC_BCH8_1K,  8,  1024),
 207         MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24, 1024),
 208         MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30, 1024),
 209         MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40, 1024),
 210         MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50, 1024),
 211         MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60, 1024),
 212 };
 213
 214 static int meson_nand_calc_ecc_bytes(int step_size, int strength)
 215 {
 216         int ecc_bytes;
 217
 218         if (step_size == 512 && strength == 8)
 219                 return ECC_PARITY_BCH8_512B;
 220
 221         ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
 222         ecc_bytes = ALIGN(ecc_bytes, 2);
 223
 224         return ecc_bytes;
 225 }
 226
 227 NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
 228                      meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
 229
 230 static const int axg_stepinfo_strengths[] = { 8 };
 231
 232 static const struct nand_ecc_step_info axg_stepinfo[] = {
 233         {
 234                 .stepsize = 1024,
 235                 .strengths = axg_stepinfo_strengths,
 236                 .nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
 237         },
 238         {
 239                 .stepsize = 512,
 240                 .strengths = axg_stepinfo_strengths,
 241                 .nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
 242         },
 243 };
 244
 245 static const struct nand_ecc_caps meson_axg_ecc_caps = {
 246         .stepinfos = axg_stepinfo,
 247         .nstepinfos = ARRAY_SIZE(axg_stepinfo),
 248         .calc_ecc_bytes = meson_nand_calc_ecc_bytes,
 249 };
 250
 251 static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
 252 {
 253         return container_of(nand, struct meson_nfc_nand_chip, nand);
 254 }
 255
 256 static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
 257 {
 258         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 259         struct meson_nfc *nfc = nand_get_controller_data(nand);
 260         int ret, value;
 261
 262         if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
 263                 return;
 264
 265         nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
 266         nfc->param.rb_select = nfc->param.chip_select;
 267         nfc->timing.twb = meson_chip->twb;
 268         nfc->timing.tadl = meson_chip->tadl;
 269         nfc->timing.tbers_max = meson_chip->tbers_max;
 270
 271         if (nfc->clk_rate != meson_chip->clk_rate) {
 272                 ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
 273                 if (ret) {
 274                         dev_err(nfc->dev, "failed to set clock rate\n");
 275                         return;
 276                 }
 277                 nfc->clk_rate = meson_chip->clk_rate;
 278         }
 279         if (nfc->bus_timing != meson_chip->bus_timing) {
 280                 value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
 281                 writel(value, nfc->reg_base + NFC_REG_CFG);
 282                 writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
 283                 nfc->bus_timing =  meson_chip->bus_timing;
 284         }
 285 }
 286
 287 static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
 288 {
 289         writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
 290                nfc->reg_base + NFC_REG_CMD);
 291 }
 292
 293 static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
 294 {
 295         writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
 296                nfc->reg_base + NFC_REG_CMD);
 297 }
 298
 299 static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
 300                                  int scrambler)
 301 {
 302         struct mtd_info *mtd = nand_to_mtd(nand);
 303         struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
 304         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 305         u32 bch = meson_chip->bch_mode, cmd;
 306         int len = mtd->writesize, pagesize, pages;
 307
 308         pagesize = nand->ecc.size;
 309
 310         if (raw) {
 311                 len = mtd->writesize + mtd->oobsize;
 312                 cmd = len | scrambler | DMA_DIR(dir);
 313                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 314                 return;
 315         }
 316
 317         pages = len / nand->ecc.size;
 318
 319         cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
 320                        NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
 321
 322         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 323 }
 324
 325 static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
 326 {
 327         /*
 328          * Insert two commands to make sure all valid commands are finished.
 329          *
 330          * The Nand flash controller is designed as two stages pipleline -
 331          *  a) fetch and b) excute.
 332          * There might be cases when the driver see command queue is empty,
 333          * but the Nand flash controller still has two commands buffered,
 334          * one is fetched into NFC request queue (ready to run), and another
 335          * is actively executing. So pushing 2 "IDLE" commands guarantees that
 336          * the pipeline is emptied.
 337          */
 338         meson_nfc_cmd_idle(nfc, 0);
 339         meson_nfc_cmd_idle(nfc, 0);
 340 }
 341
 342 static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
 343                                      unsigned int timeout_ms)
 344 {
 345         u32 cmd_size = 0;
 346         int ret;
 347
 348         /* wait cmd fifo is empty */
 349         ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
 350                                          !NFC_CMD_GET_SIZE(cmd_size),
 351                                          10, timeout_ms * 1000);
 352         if (ret)
 353                 dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
 354
 355         return ret;
 356 }
 357
 358 static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
 359 {
 360         meson_nfc_drain_cmd(nfc);
 361
 362         return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
 363 }
 364
 365 static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
 366 {
 367         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 368         int len;
 369
 370         len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
 371
 372         return meson_chip->data_buf + len;
 373 }
 374
 375 static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
 376 {
 377         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 378         int len, temp;
 379
 380         temp = nand->ecc.size + nand->ecc.bytes;
 381         len = (temp + 2) * i;
 382
 383         return meson_chip->data_buf + len;
 384 }
 385
 386 static void meson_nfc_get_data_oob(struct nand_chip *nand,
 387                                    u8 *buf, u8 *oobbuf)
 388 {
 389         int i, oob_len = 0;
 390         u8 *dsrc, *osrc;
 391
 392         oob_len = nand->ecc.bytes + 2;
 393         for (i = 0; i < nand->ecc.steps; i++) {
 394                 if (buf) {
 395                         dsrc = meson_nfc_data_ptr(nand, i);
 396                         memcpy(buf, dsrc, nand->ecc.size);
 397                         buf += nand->ecc.size;
 398                 }
 399                 osrc = meson_nfc_oob_ptr(nand, i);
 400                 memcpy(oobbuf, osrc, oob_len);
 401                 oobbuf += oob_len;
 402         }
 403 }
 404
 405 static void meson_nfc_set_data_oob(struct nand_chip *nand,
 406                                    const u8 *buf, u8 *oobbuf)
 407 {
 408         int i, oob_len = 0;
 409         u8 *dsrc, *osrc;
 410
 411         oob_len = nand->ecc.bytes + 2;
 412         for (i = 0; i < nand->ecc.steps; i++) {
 413                 if (buf) {
 414                         dsrc = meson_nfc_data_ptr(nand, i);
 415                         memcpy(dsrc, buf, nand->ecc.size);
 416                         buf += nand->ecc.size;
 417                 }
 418                 osrc = meson_nfc_oob_ptr(nand, i);
 419                 memcpy(osrc, oobbuf, oob_len);
 420                 oobbuf += oob_len;
 421         }
 422 }
 423
 424 static int meson_nfc_wait_no_rb_pin(struct nand_chip *nand, int timeout_ms,
 425                                     bool need_cmd_read0)
 426 {
 427         struct meson_nfc *nfc = nand_get_controller_data(nand);
 428         u32 cmd, cfg;
 429
 430         meson_nfc_cmd_idle(nfc, nfc->timing.twb);
 431         meson_nfc_drain_cmd(nfc);
 432         meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 433
 434         cfg = readl(nfc->reg_base + NFC_REG_CFG);
 435         cfg |= NFC_RB_IRQ_EN;
 436         writel(cfg, nfc->reg_base + NFC_REG_CFG);
 437
 438         reinit_completion(&nfc->completion);
 439         nand_status_op(nand, NULL);
 440
 441         /* use the max erase time as the maximum clock for waiting R/B */
 442         cmd = NFC_CMD_RB | NFC_CMD_RB_INT_NO_PIN | nfc->timing.tbers_max;
 443         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 444
 445         if (!wait_for_completion_timeout(&nfc->completion,
 446                                          msecs_to_jiffies(timeout_ms)))
 447                 return -ETIMEDOUT;
 448
 449         if (need_cmd_read0)
 450                 nand_exit_status_op(nand);
 451
 452         return 0;
 453 }
 454
 455 static int meson_nfc_wait_rb_pin(struct meson_nfc *nfc, int timeout_ms)
 456 {
 457         u32 cmd, cfg;
 458         int ret = 0;
 459
 460         meson_nfc_cmd_idle(nfc, nfc->timing.twb);
 461         meson_nfc_drain_cmd(nfc);
 462         meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 463
 464         cfg = readl(nfc->reg_base + NFC_REG_CFG);
 465         cfg |= NFC_RB_IRQ_EN;
 466         writel(cfg, nfc->reg_base + NFC_REG_CFG);
 467
 468         reinit_completion(&nfc->completion);
 469
 470         /* use the max erase time as the maximum clock for waiting R/B */
 471         cmd = NFC_CMD_RB | NFC_CMD_RB_INT
 472                 | nfc->param.chip_select | nfc->timing.tbers_max;
 473         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 474
 475         ret = wait_for_completion_timeout(&nfc->completion,
 476                                           msecs_to_jiffies(timeout_ms));
 477         if (ret == 0)
 478                 ret = -1;
 479
 480         return ret;
 481 }
 482
 483 static int meson_nfc_queue_rb(struct nand_chip *nand, int timeout_ms,
 484                               bool need_cmd_read0)
 485 {
 486         struct meson_nfc *nfc = nand_get_controller_data(nand);
 487
 488         if (nfc->no_rb_pin) {
 489                 /* This mode is used when there is no wired R/B pin.
 490                  * It works like 'nand_soft_waitrdy()', but instead of
 491                  * polling NAND_CMD_STATUS bit in the software loop,
 492                  * it will wait for interrupt - controllers checks IO
 493                  * bus and when it detects NAND_CMD_STATUS on it, it
 494                  * raises interrupt. After interrupt, NAND_CMD_READ0 is
 495                  * sent as terminator of the ready waiting procedure if
 496                  * needed (for all cases except page programming - this
 497                  * is reason of 'need_cmd_read0' flag).
 498                  */
 499                 return meson_nfc_wait_no_rb_pin(nand, timeout_ms,
 500                                                 need_cmd_read0);
 501         } else {
 502                 return meson_nfc_wait_rb_pin(nfc, timeout_ms);
 503         }
 504 }
 505
 506 static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
 507 {
 508         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 509         __le64 *info;
 510         int i, count;
 511
 512         for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 513                 info = &meson_chip->info_buf[i];
 514                 *info |= oob_buf[count];
 515                 *info |= oob_buf[count + 1] << 8;
 516         }
 517 }
 518
 519 static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
 520 {
 521         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 522         __le64 *info;
 523         int i, count;
 524
 525         for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 526                 info = &meson_chip->info_buf[i];
 527                 oob_buf[count] = *info;
 528                 oob_buf[count + 1] = *info >> 8;
 529         }
 530 }
 531
 532 static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
 533                                  u64 *correct_bitmap)
 534 {
 535         struct mtd_info *mtd = nand_to_mtd(nand);
 536         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 537         __le64 *info;
 538         int ret = 0, i;
 539
 540         for (i = 0; i < nand->ecc.steps; i++) {
 541                 info = &meson_chip->info_buf[i];
 542                 if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
 543                         mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
 544                         *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
 545                         *correct_bitmap |= BIT_ULL(i);
 546                         continue;
 547                 }
 548                 if ((nand->options & NAND_NEED_SCRAMBLING) &&
 549                     ECC_ZERO_CNT(*info) < nand->ecc.strength) {
 550                         mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
 551                         *bitflips = max_t(u32, *bitflips,
 552                                           ECC_ZERO_CNT(*info));
 553                         ret = ECC_CHECK_RETURN_FF;
 554                 } else {
 555                         ret = -EBADMSG;
 556                 }
 557         }
 558         return ret;
 559 }
 560
 561 static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
 562                                       int datalen, void *infobuf, int infolen,
 563                                       enum dma_data_direction dir)
 564 {
 565         struct meson_nfc *nfc = nand_get_controller_data(nand);
 566         u32 cmd;
 567         int ret = 0;
 568
 569         nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
 570         ret = dma_mapping_error(nfc->dev, nfc->daddr);
 571         if (ret) {
 572                 dev_err(nfc->dev, "DMA mapping error\n");
 573                 return ret;
 574         }
 575         cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
 576         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 577
 578         cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
 579         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 580
 581         if (infobuf) {
 582                 nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
 583                 ret = dma_mapping_error(nfc->dev, nfc->iaddr);
 584                 if (ret) {
 585                         dev_err(nfc->dev, "DMA mapping error\n");
 586                         dma_unmap_single(nfc->dev,
 587                                          nfc->daddr, datalen, dir);
 588                         return ret;
 589                 }
 590                 nfc->info_bytes = infolen;
 591                 cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
 592                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 593
 594                 cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
 595                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 596         }
 597
 598         return ret;
 599 }
 600
 601 static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
 602                                          int datalen, int infolen,
 603                                          enum dma_data_direction dir)
 604 {
 605         struct meson_nfc *nfc = nand_get_controller_data(nand);
 606
 607         dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
 608         if (infolen) {
 609                 dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
 610                 nfc->info_bytes = 0;
 611         }
 612 }
 613
 614 static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
 615 {
 616         struct meson_nfc *nfc = nand_get_controller_data(nand);
 617         int ret = 0;
 618         u32 cmd;
 619         u8 *info;
 620
 621         info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
 622         if (!info)
 623                 return -ENOMEM;
 624
 625         ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
 626                                          PER_INFO_BYTE, DMA_FROM_DEVICE);
 627         if (ret)
 628                 goto out;
 629
 630         cmd = NFC_CMD_N2M | len;
 631         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 632
 633         meson_nfc_drain_cmd(nfc);
 634         meson_nfc_wait_cmd_finish(nfc, 1000);
 635         meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
 636
 637 out:
 638         kfree(info);
 639
 640         return ret;
 641 }
 642
 643 static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
 644 {
 645         struct meson_nfc *nfc = nand_get_controller_data(nand);
 646         int ret = 0;
 647         u32 cmd;
 648
 649         ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
 650                                          0, DMA_TO_DEVICE);
 651         if (ret)
 652                 return ret;
 653
 654         cmd = NFC_CMD_M2N | len;
 655         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 656
 657         meson_nfc_drain_cmd(nfc);
 658         meson_nfc_wait_cmd_finish(nfc, 1000);
 659         meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
 660
 661         return ret;
 662 }
 663
 664 static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
 665                                                 int page, bool in)
 666 {
 667         const struct nand_sdr_timings *sdr =
 668                 nand_get_sdr_timings(nand_get_interface_config(nand));
 669         struct mtd_info *mtd = nand_to_mtd(nand);
 670         struct meson_nfc *nfc = nand_get_controller_data(nand);
 671         u32 *addrs = nfc->cmdfifo.rw.addrs;
 672         u32 cs = nfc->param.chip_select;
 673         u32 cmd0, cmd_num, row_start;
 674         int i;
 675
 676         cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
 677
 678         cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
 679         nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
 680
 681         addrs[0] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_0;
 682         if (mtd->writesize <= 512) {
 683                 cmd_num--;
 684                 row_start = 1;
 685         } else {
 686                 addrs[1] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_1;
 687                 row_start = 2;
 688         }
 689
 690         addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
 691         addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
 692
 693         if (nand->options & NAND_ROW_ADDR_3)
 694                 addrs[row_start + 2] =
 695                         cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
 696         else
 697                 cmd_num--;
 698
 699         /* subtract cmd1 */
 700         cmd_num--;
 701
 702         for (i = 0; i < cmd_num; i++)
 703                 writel_relaxed(nfc->cmdfifo.cmd[i],
 704                                nfc->reg_base + NFC_REG_CMD);
 705
 706         if (in) {
 707                 nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
 708                 writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
 709                 meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tR_max), true);
 710         } else {
 711                 meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
 712         }
 713
 714         return 0;
 715 }
 716
 717 static int meson_nfc_write_page_sub(struct nand_chip *nand,
 718                                     int page, int raw)
 719 {
 720         const struct nand_sdr_timings *sdr =
 721                 nand_get_sdr_timings(nand_get_interface_config(nand));
 722         struct mtd_info *mtd = nand_to_mtd(nand);
 723         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 724         struct meson_nfc *nfc = nand_get_controller_data(nand);
 725         int data_len, info_len;
 726         u32 cmd;
 727         int ret;
 728
 729         meson_nfc_select_chip(nand, nand->cur_cs);
 730
 731         data_len =  mtd->writesize + mtd->oobsize;
 732         info_len = nand->ecc.steps * PER_INFO_BYTE;
 733
 734         ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
 735         if (ret)
 736                 return ret;
 737
 738         ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 739                                          data_len, meson_chip->info_buf,
 740                                          info_len, DMA_TO_DEVICE);
 741         if (ret)
 742                 return ret;
 743
 744         if (nand->options & NAND_NEED_SCRAMBLING) {
 745                 meson_nfc_cmd_seed(nfc, page);
 746                 meson_nfc_cmd_access(nand, raw, DIRWRITE,
 747                                      NFC_CMD_SCRAMBLER_ENABLE);
 748         } else {
 749                 meson_nfc_cmd_access(nand, raw, DIRWRITE,
 750                                      NFC_CMD_SCRAMBLER_DISABLE);
 751         }
 752
 753         cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
 754         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 755         meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tPROG_max), false);
 756
 757         meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
 758
 759         return ret;
 760 }
 761
 762 static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
 763                                     int oob_required, int page)
 764 {
 765         u8 *oob_buf = nand->oob_poi;
 766
 767         meson_nfc_set_data_oob(nand, buf, oob_buf);
 768
 769         return meson_nfc_write_page_sub(nand, page, 1);
 770 }
 771
 772 static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
 773                                       const u8 *buf, int oob_required, int page)
 774 {
 775         struct mtd_info *mtd = nand_to_mtd(nand);
 776         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 777         u8 *oob_buf = nand->oob_poi;
 778
 779         memcpy(meson_chip->data_buf, buf, mtd->writesize);
 780         memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
 781         meson_nfc_set_user_byte(nand, oob_buf);
 782
 783         return meson_nfc_write_page_sub(nand, page, 0);
 784 }
 785
 786 static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
 787                                             struct nand_chip *nand, int raw)
 788 {
 789         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 790         __le64 *info;
 791         u32 neccpages;
 792         int ret;
 793
 794         neccpages = raw ? 1 : nand->ecc.steps;
 795         info = &meson_chip->info_buf[neccpages - 1];
 796         do {
 797                 usleep_range(10, 15);
 798                 /* info is updated by nfc dma engine*/
 799                 smp_rmb();
 800                 dma_sync_single_for_cpu(nfc->dev, nfc->iaddr, nfc->info_bytes,
 801                                         DMA_FROM_DEVICE);
 802                 ret = *info & ECC_COMPLETE;
 803         } while (!ret);
 804 }
 805
 806 static int meson_nfc_read_page_sub(struct nand_chip *nand,
 807                                    int page, int raw)
 808 {
 809         struct mtd_info *mtd = nand_to_mtd(nand);
 810         struct meson_nfc *nfc = nand_get_controller_data(nand);
 811         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 812         int data_len, info_len;
 813         int ret;
 814
 815         meson_nfc_select_chip(nand, nand->cur_cs);
 816
 817         data_len =  mtd->writesize + mtd->oobsize;
 818         info_len = nand->ecc.steps * PER_INFO_BYTE;
 819
 820         ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
 821         if (ret)
 822                 return ret;
 823
 824         ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 825                                          data_len, meson_chip->info_buf,
 826                                          info_len, DMA_FROM_DEVICE);
 827         if (ret)
 828                 return ret;
 829
 830         if (nand->options & NAND_NEED_SCRAMBLING) {
 831                 meson_nfc_cmd_seed(nfc, page);
 832                 meson_nfc_cmd_access(nand, raw, DIRREAD,
 833                                      NFC_CMD_SCRAMBLER_ENABLE);
 834         } else {
 835                 meson_nfc_cmd_access(nand, raw, DIRREAD,
 836                                      NFC_CMD_SCRAMBLER_DISABLE);
 837         }
 838
 839         ret = meson_nfc_wait_dma_finish(nfc);
 840         meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
 841
 842         meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
 843
 844         return ret;
 845 }
 846
 847 static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
 848                                    int oob_required, int page)
 849 {
 850         u8 *oob_buf = nand->oob_poi;
 851         int ret;
 852
 853         ret = meson_nfc_read_page_sub(nand, page, 1);
 854         if (ret)
 855                 return ret;
 856
 857         meson_nfc_get_data_oob(nand, buf, oob_buf);
 858
 859         return 0;
 860 }
 861
 862 static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
 863                                      int oob_required, int page)
 864 {
 865         struct mtd_info *mtd = nand_to_mtd(nand);
 866         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 867         struct nand_ecc_ctrl *ecc = &nand->ecc;
 868         u64 correct_bitmap = 0;
 869         u32 bitflips = 0;
 870         u8 *oob_buf = nand->oob_poi;
 871         int ret, i;
 872
 873         ret = meson_nfc_read_page_sub(nand, page, 0);
 874         if (ret)
 875                 return ret;
 876
 877         meson_nfc_get_user_byte(nand, oob_buf);
 878         ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
 879         if (ret == ECC_CHECK_RETURN_FF) {
 880                 if (buf)
 881                         memset(buf, 0xff, mtd->writesize);
 882                 memset(oob_buf, 0xff, mtd->oobsize);
 883         } else if (ret < 0) {
 884                 if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
 885                         mtd->ecc_stats.failed++;
 886                         return bitflips;
 887                 }
 888                 ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
 889                 if (ret)
 890                         return ret;
 891
 892                 for (i = 0; i < nand->ecc.steps ; i++) {
 893                         u8 *data = buf + i * ecc->size;
 894                         u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
 895
 896                         if (correct_bitmap & BIT_ULL(i))
 897                                 continue;
 898                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
 899                                                           oob, ecc->bytes + 2,
 900                                                           NULL, 0,
 901                                                           ecc->strength);
 902                         if (ret < 0) {
 903                                 mtd->ecc_stats.failed++;
 904                         } else {
 905                                 mtd->ecc_stats.corrected += ret;
 906                                 bitflips =  max_t(u32, bitflips, ret);
 907                         }
 908                 }
 909         } else if (buf && buf != meson_chip->data_buf) {
 910                 memcpy(buf, meson_chip->data_buf, mtd->writesize);
 911         }
 912
 913         return bitflips;
 914 }
 915
 916 static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
 917 {
 918         return meson_nfc_read_page_raw(nand, NULL, 1, page);
 919 }
 920
 921 static int meson_nfc_read_oob(struct nand_chip *nand, int page)
 922 {
 923         return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
 924 }
 925
 926 static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
 927 {
 928         if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
 929                 return false;
 930
 931         if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
 932                 return true;
 933         return false;
 934 }
 935
 936 static void *
 937 meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
 938 {
 939         if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
 940                 return NULL;
 941
 942         if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
 943                 return instr->ctx.data.buf.in;
 944
 945         return kzalloc(instr->ctx.data.len, GFP_KERNEL);
 946 }
 947
 948 static void
 949 meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
 950                                      void *buf)
 951 {
 952         if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
 953             WARN_ON(!buf))
 954                 return;
 955
 956         if (buf == instr->ctx.data.buf.in)
 957                 return;
 958
 959         memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
 960         kfree(buf);
 961 }
 962
 963 static void *
 964 meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
 965 {
 966         if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
 967                 return NULL;
 968
 969         if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
 970                 return (void *)instr->ctx.data.buf.out;
 971
 972         return kmemdup(instr->ctx.data.buf.out,
 973                        instr->ctx.data.len, GFP_KERNEL);
 974 }
 975
 976 static void
 977 meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
 978                                       const void *buf)
 979 {
 980         if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
 981             WARN_ON(!buf))
 982                 return;
 983
 984         if (buf != instr->ctx.data.buf.out)
 985                 kfree(buf);
 986 }
 987
 988 static int meson_nfc_check_op(struct nand_chip *chip,
 989                               const struct nand_operation *op)
 990 {
 991         int op_id;
 992
 993         for (op_id = 0; op_id < op->ninstrs; op_id++) {
 994                 const struct nand_op_instr *instr;
 995
 996                 instr = &op->instrs[op_id];
 997
 998                 switch (instr->type) {
 999                 case NAND_OP_DATA_IN_INSTR:
1000                 case NAND_OP_DATA_OUT_INSTR:
1001                         if (instr->ctx.data.len > NFC_CMD_RAW_LEN)
1002                                 return -ENOTSUPP;
1003
1004                         break;
1005                 default:
1006                         break;
1007                 }
1008         }
1009
1010         return 0;
1011 }
1012
1013 static int meson_nfc_exec_op(struct nand_chip *nand,
1014                              const struct nand_operation *op, bool check_only)
1015 {
1016         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1017         struct meson_nfc *nfc = nand_get_controller_data(nand);
1018         const struct nand_op_instr *instr = NULL;
1019         void *buf;
1020         u32 op_id, delay_idle, cmd;
1021         int err;
1022         int i;
1023
1024         err = meson_nfc_check_op(nand, op);
1025         if (err)
1026                 return err;
1027
1028         if (check_only)
1029                 return 0;
1030
1031         meson_nfc_select_chip(nand, op->cs);
1032         for (op_id = 0; op_id < op->ninstrs; op_id++) {
1033                 instr = &op->instrs[op_id];
1034                 delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
1035                                           meson_chip->level1_divider *
1036                                           NFC_CLK_CYCLE);
1037                 switch (instr->type) {
1038                 case NAND_OP_CMD_INSTR:
1039                         cmd = nfc->param.chip_select | NFC_CMD_CLE;
1040                         cmd |= instr->ctx.cmd.opcode & 0xff;
1041                         writel(cmd, nfc->reg_base + NFC_REG_CMD);
1042                         meson_nfc_cmd_idle(nfc, delay_idle);
1043                         break;
1044
1045                 case NAND_OP_ADDR_INSTR:
1046                         for (i = 0; i < instr->ctx.addr.naddrs; i++) {
1047                                 cmd = nfc->param.chip_select | NFC_CMD_ALE;
1048                                 cmd |= instr->ctx.addr.addrs[i] & 0xff;
1049                                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
1050                         }
1051                         meson_nfc_cmd_idle(nfc, delay_idle);
1052                         break;
1053
1054                 case NAND_OP_DATA_IN_INSTR:
1055                         buf = meson_nand_op_get_dma_safe_input_buf(instr);
1056                         if (!buf)
1057                                 return -ENOMEM;
1058                         meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
1059                         meson_nand_op_put_dma_safe_input_buf(instr, buf);
1060                         break;
1061
1062                 case NAND_OP_DATA_OUT_INSTR:
1063                         buf = meson_nand_op_get_dma_safe_output_buf(instr);
1064                         if (!buf)
1065                                 return -ENOMEM;
1066                         meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
1067                         meson_nand_op_put_dma_safe_output_buf(instr, buf);
1068                         break;
1069
1070                 case NAND_OP_WAITRDY_INSTR:
1071                         meson_nfc_queue_rb(nand, instr->ctx.waitrdy.timeout_ms,
1072                                            true);
1073                         if (instr->delay_ns)
1074                                 meson_nfc_cmd_idle(nfc, delay_idle);
1075                         break;
1076                 }
1077         }
1078         meson_nfc_wait_cmd_finish(nfc, 1000);
1079         return 0;
1080 }
1081
1082 static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
1083                                struct mtd_oob_region *oobregion)
1084 {
1085         struct nand_chip *nand = mtd_to_nand(mtd);
1086
1087         if (section >= nand->ecc.steps)
1088                 return -ERANGE;
1089
1090         oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
1091         oobregion->length = nand->ecc.bytes;
1092
1093         return 0;
1094 }
1095
1096 static int meson_ooblayout_free(struct mtd_info *mtd, int section,
1097                                 struct mtd_oob_region *oobregion)
1098 {
1099         struct nand_chip *nand = mtd_to_nand(mtd);
1100
1101         if (section >= nand->ecc.steps)
1102                 return -ERANGE;
1103
1104         oobregion->offset = section * (2 + nand->ecc.bytes);
1105         oobregion->length = 2;
1106
1107         return 0;
1108 }
1109
1110 static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
1111         .ecc = meson_ooblayout_ecc,
1112         .free = meson_ooblayout_free,
1113 };
1114
1115 static int meson_nfc_clk_init(struct meson_nfc *nfc)
1116 {
1117         struct clk_parent_data nfc_divider_parent_data[1] = {0};
1118         struct clk_init_data init = {0};
1119         int ret;
1120
1121         /* request core clock */
1122         nfc->core_clk = devm_clk_get(nfc->dev, "core");
1123         if (IS_ERR(nfc->core_clk)) {
1124                 dev_err(nfc->dev, "failed to get core clock\n");
1125                 return PTR_ERR(nfc->core_clk);
1126         }
1127
1128         nfc->device_clk = devm_clk_get(nfc->dev, "device");
1129         if (IS_ERR(nfc->device_clk)) {
1130                 dev_err(nfc->dev, "failed to get device clock\n");
1131                 return PTR_ERR(nfc->device_clk);
1132         }
1133
1134         init.name = devm_kasprintf(nfc->dev,
1135                                    GFP_KERNEL, "%s#div",
1136                                    dev_name(nfc->dev));
1137         init.ops = &clk_divider_ops;
1138         nfc_divider_parent_data[0].fw_name = "device";
1139         init.parent_data = nfc_divider_parent_data;
1140         init.num_parents = 1;
1141         nfc->nand_divider.reg = nfc->reg_clk;
1142         nfc->nand_divider.shift = CLK_DIV_SHIFT;
1143         nfc->nand_divider.width = CLK_DIV_WIDTH;
1144         nfc->nand_divider.hw.init = &init;
1145         nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1146                                   CLK_DIVIDER_ROUND_CLOSEST |
1147                                   CLK_DIVIDER_ALLOW_ZERO;
1148
1149         nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1150         if (IS_ERR(nfc->nand_clk))
1151                 return PTR_ERR(nfc->nand_clk);
1152
1153         /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1154         writel(CLK_SELECT_NAND | readl(nfc->reg_clk),
1155                nfc->reg_clk);
1156
1157         ret = clk_prepare_enable(nfc->core_clk);
1158         if (ret) {
1159                 dev_err(nfc->dev, "failed to enable core clock\n");
1160                 return ret;
1161         }
1162
1163         ret = clk_prepare_enable(nfc->device_clk);
1164         if (ret) {
1165                 dev_err(nfc->dev, "failed to enable device clock\n");
1166                 goto err_device_clk;
1167         }
1168
1169         ret = clk_prepare_enable(nfc->nand_clk);
1170         if (ret) {
1171                 dev_err(nfc->dev, "pre enable NFC divider fail\n");
1172                 goto err_nand_clk;
1173         }
1174
1175         ret = clk_set_rate(nfc->nand_clk, 24000000);
1176         if (ret)
1177                 goto err_disable_clk;
1178
1179         return 0;
1180
1181 err_disable_clk:
1182         clk_disable_unprepare(nfc->nand_clk);
1183 err_nand_clk:
1184         clk_disable_unprepare(nfc->device_clk);
1185 err_device_clk:
1186         clk_disable_unprepare(nfc->core_clk);
1187         return ret;
1188 }
1189
1190 static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1191 {
1192         clk_disable_unprepare(nfc->nand_clk);
1193         clk_disable_unprepare(nfc->device_clk);
1194         clk_disable_unprepare(nfc->core_clk);
1195 }
1196
1197 static void meson_nfc_free_buffer(struct nand_chip *nand)
1198 {
1199         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1200
1201         kfree(meson_chip->info_buf);
1202         kfree(meson_chip->data_buf);
1203 }
1204
1205 static int meson_chip_buffer_init(struct nand_chip *nand)
1206 {
1207         struct mtd_info *mtd = nand_to_mtd(nand);
1208         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1209         u32 page_bytes, info_bytes, nsectors;
1210
1211         nsectors = mtd->writesize / nand->ecc.size;
1212
1213         page_bytes =  mtd->writesize + mtd->oobsize;
1214         info_bytes = nsectors * PER_INFO_BYTE;
1215
1216         meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1217         if (!meson_chip->data_buf)
1218                 return -ENOMEM;
1219
1220         meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1221         if (!meson_chip->info_buf) {
1222                 kfree(meson_chip->data_buf);
1223                 return -ENOMEM;
1224         }
1225
1226         return 0;
1227 }
1228
1229 static
1230 int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1231                               const struct nand_interface_config *conf)
1232 {
1233         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1234         const struct nand_sdr_timings *timings;
1235         u32 div, bt_min, bt_max, tbers_clocks;
1236
1237         timings = nand_get_sdr_timings(conf);
1238         if (IS_ERR(timings))
1239                 return -ENOTSUPP;
1240
1241         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1242                 return 0;
1243
1244         div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1245         bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1246         bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1247                   timings->tRC_min / 2) / div;
1248
1249         meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1250                                        div * NFC_CLK_CYCLE);
1251         meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1252                                         div * NFC_CLK_CYCLE);
1253         tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1254                                         div * NFC_CLK_CYCLE);
1255         meson_chip->tbers_max = ilog2(tbers_clocks);
1256         if (!is_power_of_2(tbers_clocks))
1257                 meson_chip->tbers_max++;
1258
1259         bt_min = DIV_ROUND_UP(bt_min, 1000);
1260         bt_max = DIV_ROUND_UP(bt_max, 1000);
1261
1262         if (bt_max < bt_min)
1263                 return -EINVAL;
1264
1265         meson_chip->level1_divider = div;
1266         meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1267         meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1268
1269         return 0;
1270 }
1271
1272 static int meson_nand_bch_mode(struct nand_chip *nand)
1273 {
1274         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1275         int i;
1276
1277         if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1278                 return -EINVAL;
1279
1280         for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1281                 if (meson_ecc[i].strength == nand->ecc.strength &&
1282                     meson_ecc[i].size == nand->ecc.size) {
1283                         meson_chip->bch_mode = meson_ecc[i].bch;
1284                         return 0;
1285                 }
1286         }
1287
1288         return -EINVAL;
1289 }
1290
1291 static void meson_nand_detach_chip(struct nand_chip *nand)
1292 {
1293         meson_nfc_free_buffer(nand);
1294 }
1295
1296 static int meson_nand_attach_chip(struct nand_chip *nand)
1297 {
1298         struct meson_nfc *nfc = nand_get_controller_data(nand);
1299         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1300         struct mtd_info *mtd = nand_to_mtd(nand);
1301         int raw_writesize;
1302         int ret;
1303
1304         if (!mtd->name) {
1305                 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1306                                            "%s:nand%d",
1307                                            dev_name(nfc->dev),
1308                                            meson_chip->sels[0]);
1309                 if (!mtd->name)
1310                         return -ENOMEM;
1311         }
1312
1313         raw_writesize = mtd->writesize + mtd->oobsize;
1314         if (raw_writesize > NFC_CMD_RAW_LEN) {
1315                 dev_err(nfc->dev, "too big write size in raw mode: %d > %ld\n",
1316                         raw_writesize, NFC_CMD_RAW_LEN);
1317                 return -EINVAL;
1318         }
1319
1320         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1321                 nand->bbt_options |= NAND_BBT_NO_OOB;
1322
1323         nand->options |= NAND_NO_SUBPAGE_WRITE;
1324
1325         ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1326                                    mtd->oobsize - 2);
1327         if (ret) {
1328                 dev_err(nfc->dev, "failed to ECC init\n");
1329                 return -EINVAL;
1330         }
1331
1332         mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1333
1334         ret = meson_nand_bch_mode(nand);
1335         if (ret)
1336                 return -EINVAL;
1337
1338         nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1339         nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1340         nand->ecc.write_page = meson_nfc_write_page_hwecc;
1341         nand->ecc.write_oob_raw = nand_write_oob_std;
1342         nand->ecc.write_oob = nand_write_oob_std;
1343
1344         nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1345         nand->ecc.read_page = meson_nfc_read_page_hwecc;
1346         nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1347         nand->ecc.read_oob = meson_nfc_read_oob;
1348
1349         if (nand->options & NAND_BUSWIDTH_16) {
1350                 dev_err(nfc->dev, "16bits bus width not supported");
1351                 return -EINVAL;
1352         }
1353         ret = meson_chip_buffer_init(nand);
1354         if (ret)
1355                 return -ENOMEM;
1356
1357         return ret;
1358 }
1359
1360 static const struct nand_controller_ops meson_nand_controller_ops = {
1361         .attach_chip = meson_nand_attach_chip,
1362         .detach_chip = meson_nand_detach_chip,
1363         .setup_interface = meson_nfc_setup_interface,
1364         .exec_op = meson_nfc_exec_op,
1365 };
1366
1367 static int
1368 meson_nfc_nand_chip_init(struct device *dev,
1369                          struct meson_nfc *nfc, struct device_node *np)
1370 {
1371         struct meson_nfc_nand_chip *meson_chip;
1372         struct nand_chip *nand;
1373         struct mtd_info *mtd;
1374         int ret, i;
1375         u32 tmp, nsels;
1376         u32 nand_rb_val = 0;
1377
1378         nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1379         if (!nsels || nsels > MAX_CE_NUM) {
1380                 dev_err(dev, "invalid register property size\n");
1381                 return -EINVAL;
1382         }
1383
1384         meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1385                                   GFP_KERNEL);
1386         if (!meson_chip)
1387                 return -ENOMEM;
1388
1389         meson_chip->nsels = nsels;
1390
1391         for (i = 0; i < nsels; i++) {
1392                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1393                 if (ret) {
1394                         dev_err(dev, "could not retrieve register property: %d\n",
1395                                 ret);
1396                         return ret;
1397                 }
1398
1399                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1400                         dev_err(dev, "CS %d already assigned\n", tmp);
1401                         return -EINVAL;
1402                 }
1403         }
1404
1405         nand = &meson_chip->nand;
1406         nand->controller = &nfc->controller;
1407         nand->controller->ops = &meson_nand_controller_ops;
1408         nand_set_flash_node(nand, np);
1409         nand_set_controller_data(nand, nfc);
1410
1411         nand->options |= NAND_USES_DMA;
1412         mtd = nand_to_mtd(nand);
1413         mtd->owner = THIS_MODULE;
1414         mtd->dev.parent = dev;
1415
1416         ret = of_property_read_u32(np, "nand-rb", &nand_rb_val);
1417         if (ret == -EINVAL)
1418                 nfc->no_rb_pin = true;
1419         else if (ret)
1420                 return ret;
1421
1422         if (nand_rb_val)
1423                 return -EINVAL;
1424
1425         ret = nand_scan(nand, nsels);
1426         if (ret)
1427                 return ret;
1428
1429         ret = mtd_device_register(mtd, NULL, 0);
1430         if (ret) {
1431                 dev_err(dev, "failed to register MTD device: %d\n", ret);
1432                 nand_cleanup(nand);
1433                 return ret;
1434         }
1435
1436         list_add_tail(&meson_chip->node, &nfc->chips);
1437
1438         return 0;
1439 }
1440
1441 static void meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1442 {
1443         struct meson_nfc_nand_chip *meson_chip;
1444         struct mtd_info *mtd;
1445
1446         while (!list_empty(&nfc->chips)) {
1447                 meson_chip = list_first_entry(&nfc->chips,
1448                                               struct meson_nfc_nand_chip, node);
1449                 mtd = nand_to_mtd(&meson_chip->nand);
1450                 WARN_ON(mtd_device_unregister(mtd));
1451
1452                 nand_cleanup(&meson_chip->nand);
1453                 list_del(&meson_chip->node);
1454         }
1455 }
1456
1457 static int meson_nfc_nand_chips_init(struct device *dev,
1458                                      struct meson_nfc *nfc)
1459 {
1460         struct device_node *np = dev->of_node;
1461         struct device_node *nand_np;
1462         int ret;
1463
1464         for_each_child_of_node(np, nand_np) {
1465                 ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1466                 if (ret) {
1467                         meson_nfc_nand_chip_cleanup(nfc);
1468                         of_node_put(nand_np);
1469                         return ret;
1470                 }
1471         }
1472
1473         return 0;
1474 }
1475
1476 static irqreturn_t meson_nfc_irq(int irq, void *id)
1477 {
1478         struct meson_nfc *nfc = id;
1479         u32 cfg;
1480
1481         cfg = readl(nfc->reg_base + NFC_REG_CFG);
1482         if (!(cfg & NFC_RB_IRQ_EN))
1483                 return IRQ_NONE;
1484
1485         cfg &= ~(NFC_RB_IRQ_EN);
1486         writel(cfg, nfc->reg_base + NFC_REG_CFG);
1487
1488         complete(&nfc->completion);
1489         return IRQ_HANDLED;
1490 }
1491
1492 static const struct meson_nfc_data meson_gxl_data = {
1493         .ecc_caps = &meson_gxl_ecc_caps,
1494 };
1495
1496 static const struct meson_nfc_data meson_axg_data = {
1497         .ecc_caps = &meson_axg_ecc_caps,
1498 };
1499
1500 static const struct of_device_id meson_nfc_id_table[] = {
1501         {
1502                 .compatible = "amlogic,meson-gxl-nfc",
1503                 .data = &meson_gxl_data,
1504         }, {
1505                 .compatible = "amlogic,meson-axg-nfc",
1506                 .data = &meson_axg_data,
1507         },
1508         {}
1509 };
1510 MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1511
1512 static int meson_nfc_probe(struct platform_device *pdev)
1513 {
1514         struct device *dev = &pdev->dev;
1515         struct meson_nfc *nfc;
1516         int ret, irq;
1517
1518         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1519         if (!nfc)
1520                 return -ENOMEM;
1521
1522         nfc->data = of_device_get_match_data(&pdev->dev);
1523         if (!nfc->data)
1524                 return -ENODEV;
1525
1526         nand_controller_init(&nfc->controller);
1527         INIT_LIST_HEAD(&nfc->chips);
1528         init_completion(&nfc->completion);
1529
1530         nfc->dev = dev;
1531
1532         nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1533         if (IS_ERR(nfc->reg_base))
1534                 return PTR_ERR(nfc->reg_base);
1535
1536         nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1537         if (IS_ERR(nfc->reg_clk))
1538                 return PTR_ERR(nfc->reg_clk);
1539
1540         irq = platform_get_irq(pdev, 0);
1541         if (irq < 0)
1542                 return -EINVAL;
1543
1544         ret = meson_nfc_clk_init(nfc);
1545         if (ret) {
1546                 dev_err(dev, "failed to initialize NAND clock\n");
1547                 return ret;
1548         }
1549
1550         writel(0, nfc->reg_base + NFC_REG_CFG);
1551         ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1552         if (ret) {
1553                 dev_err(dev, "failed to request NFC IRQ\n");
1554                 ret = -EINVAL;
1555                 goto err_clk;
1556         }
1557
1558         ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1559         if (ret) {
1560                 dev_err(dev, "failed to set DMA mask\n");
1561                 goto err_clk;
1562         }
1563
1564         platform_set_drvdata(pdev, nfc);
1565
1566         ret = meson_nfc_nand_chips_init(dev, nfc);
1567         if (ret) {
1568                 dev_err(dev, "failed to init NAND chips\n");
1569                 goto err_clk;
1570         }
1571
1572         return 0;
1573 err_clk:
1574         meson_nfc_disable_clk(nfc);
1575         return ret;
1576 }
1577
1578 static void meson_nfc_remove(struct platform_device *pdev)
1579 {
1580         struct meson_nfc *nfc = platform_get_drvdata(pdev);
1581
1582         meson_nfc_nand_chip_cleanup(nfc);
1583
1584         meson_nfc_disable_clk(nfc);
1585 }
1586
1587 static struct platform_driver meson_nfc_driver = {
1588         .probe  = meson_nfc_probe,
1589         .remove_new = meson_nfc_remove,
1590         .driver = {
1591                 .name  = "meson-nand",
1592                 .of_match_table = meson_nfc_id_table,
1593         },
1594 };
1595 module_platform_driver(meson_nfc_driver);
1596
1597 MODULE_LICENSE("Dual MIT/GPL");
1598 MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1599 MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");