drivers/mtd/nand/raw/qcom_nandc.c

   1 /*
   2  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
   3  *
   4  * This software is licensed under the terms of the GNU General Public
   5  * License version 2, as published by the Free Software Foundation, and
   6  * may be copied, distributed, and modified under those terms.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  */
  13
  14 #include <linux/clk.h>
  15 #include <linux/slab.h>
  16 #include <linux/bitops.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmaengine.h>
  19 #include <linux/module.h>
  20 #include <linux/mtd/rawnand.h>
  21 #include <linux/mtd/partitions.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/delay.h>
  25 #include <linux/dma/qcom_bam_dma.h>
  26
  27 /* NANDc reg offsets */
  28 #define NAND_FLASH_CMD                  0x00
  29 #define NAND_ADDR0                      0x04
  30 #define NAND_ADDR1                      0x08
  31 #define NAND_FLASH_CHIP_SELECT          0x0c
  32 #define NAND_EXEC_CMD                   0x10
  33 #define NAND_FLASH_STATUS               0x14
  34 #define NAND_BUFFER_STATUS              0x18
  35 #define NAND_DEV0_CFG0                  0x20
  36 #define NAND_DEV0_CFG1                  0x24
  37 #define NAND_DEV0_ECC_CFG               0x28
  38 #define NAND_DEV1_ECC_CFG               0x2c
  39 #define NAND_DEV1_CFG0                  0x30
  40 #define NAND_DEV1_CFG1                  0x34
  41 #define NAND_READ_ID                    0x40
  42 #define NAND_READ_STATUS                0x44
  43 #define NAND_DEV_CMD0                   0xa0
  44 #define NAND_DEV_CMD1                   0xa4
  45 #define NAND_DEV_CMD2                   0xa8
  46 #define NAND_DEV_CMD_VLD                0xac
  47 #define SFLASHC_BURST_CFG               0xe0
  48 #define NAND_ERASED_CW_DETECT_CFG       0xe8
  49 #define NAND_ERASED_CW_DETECT_STATUS    0xec
  50 #define NAND_EBI2_ECC_BUF_CFG           0xf0
  51 #define FLASH_BUF_ACC                   0x100
  52
  53 #define NAND_CTRL                       0xf00
  54 #define NAND_VERSION                    0xf08
  55 #define NAND_READ_LOCATION_0            0xf20
  56 #define NAND_READ_LOCATION_1            0xf24
  57 #define NAND_READ_LOCATION_2            0xf28
  58 #define NAND_READ_LOCATION_3            0xf2c
  59
  60 /* dummy register offsets, used by write_reg_dma */
  61 #define NAND_DEV_CMD1_RESTORE           0xdead
  62 #define NAND_DEV_CMD_VLD_RESTORE        0xbeef
  63
  64 /* NAND_FLASH_CMD bits */
  65 #define PAGE_ACC                        BIT(4)
  66 #define LAST_PAGE                       BIT(5)
  67
  68 /* NAND_FLASH_CHIP_SELECT bits */
  69 #define NAND_DEV_SEL                    0
  70 #define DM_EN                           BIT(2)
  71
  72 /* NAND_FLASH_STATUS bits */
  73 #define FS_OP_ERR                       BIT(4)
  74 #define FS_READY_BSY_N                  BIT(5)
  75 #define FS_MPU_ERR                      BIT(8)
  76 #define FS_DEVICE_STS_ERR               BIT(16)
  77 #define FS_DEVICE_WP                    BIT(23)
  78
  79 /* NAND_BUFFER_STATUS bits */
  80 #define BS_UNCORRECTABLE_BIT            BIT(8)
  81 #define BS_CORRECTABLE_ERR_MSK          0x1f
  82
  83 /* NAND_DEVn_CFG0 bits */
  84 #define DISABLE_STATUS_AFTER_WRITE      4
  85 #define CW_PER_PAGE                     6
  86 #define UD_SIZE_BYTES                   9
  87 #define ECC_PARITY_SIZE_BYTES_RS        19
  88 #define SPARE_SIZE_BYTES                23
  89 #define NUM_ADDR_CYCLES                 27
  90 #define STATUS_BFR_READ                 30
  91 #define SET_RD_MODE_AFTER_STATUS        31
  92
  93 /* NAND_DEVn_CFG0 bits */
  94 #define DEV0_CFG1_ECC_DISABLE           0
  95 #define WIDE_FLASH                      1
  96 #define NAND_RECOVERY_CYCLES            2
  97 #define CS_ACTIVE_BSY                   5
  98 #define BAD_BLOCK_BYTE_NUM              6
  99 #define BAD_BLOCK_IN_SPARE_AREA         16
 100 #define WR_RD_BSY_GAP                   17
 101 #define ENABLE_BCH_ECC                  27
 102
 103 /* NAND_DEV0_ECC_CFG bits */
 104 #define ECC_CFG_ECC_DISABLE             0
 105 #define ECC_SW_RESET                    1
 106 #define ECC_MODE                        4
 107 #define ECC_PARITY_SIZE_BYTES_BCH       8
 108 #define ECC_NUM_DATA_BYTES              16
 109 #define ECC_FORCE_CLK_OPEN              30
 110
 111 /* NAND_DEV_CMD1 bits */
 112 #define READ_ADDR                       0
 113
 114 /* NAND_DEV_CMD_VLD bits */
 115 #define READ_START_VLD                  BIT(0)
 116 #define READ_STOP_VLD                   BIT(1)
 117 #define WRITE_START_VLD                 BIT(2)
 118 #define ERASE_START_VLD                 BIT(3)
 119 #define SEQ_READ_START_VLD              BIT(4)
 120
 121 /* NAND_EBI2_ECC_BUF_CFG bits */
 122 #define NUM_STEPS                       0
 123
 124 /* NAND_ERASED_CW_DETECT_CFG bits */
 125 #define ERASED_CW_ECC_MASK              1
 126 #define AUTO_DETECT_RES                 0
 127 #define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
 128 #define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
 129 #define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
 130 #define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
 131 #define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
 132
 133 /* NAND_ERASED_CW_DETECT_STATUS bits */
 134 #define PAGE_ALL_ERASED                 BIT(7)
 135 #define CODEWORD_ALL_ERASED             BIT(6)
 136 #define PAGE_ERASED                     BIT(5)
 137 #define CODEWORD_ERASED                 BIT(4)
 138 #define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
 139 #define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 140
 141 /* NAND_READ_LOCATION_n bits */
 142 #define READ_LOCATION_OFFSET            0
 143 #define READ_LOCATION_SIZE              16
 144 #define READ_LOCATION_LAST              31
 145
 146 /* Version Mask */
 147 #define NAND_VERSION_MAJOR_MASK         0xf0000000
 148 #define NAND_VERSION_MAJOR_SHIFT        28
 149 #define NAND_VERSION_MINOR_MASK         0x0fff0000
 150 #define NAND_VERSION_MINOR_SHIFT        16
 151
 152 /* NAND OP_CMDs */
 153 #define OP_PAGE_READ                    0x2
 154 #define OP_PAGE_READ_WITH_ECC           0x3
 155 #define OP_PAGE_READ_WITH_ECC_SPARE     0x4
 156 #define OP_PROGRAM_PAGE                 0x6
 157 #define OP_PAGE_PROGRAM_WITH_ECC        0x7
 158 #define OP_PROGRAM_PAGE_SPARE           0x9
 159 #define OP_BLOCK_ERASE                  0xa
 160 #define OP_FETCH_ID                     0xb
 161 #define OP_RESET_DEVICE                 0xd
 162
 163 /* Default Value for NAND_DEV_CMD_VLD */
 164 #define NAND_DEV_CMD_VLD_VAL            (READ_START_VLD | WRITE_START_VLD | \
 165                                          ERASE_START_VLD | SEQ_READ_START_VLD)
 166
 167 /* NAND_CTRL bits */
 168 #define BAM_MODE_EN                     BIT(0)
 169
 170 /*
 171  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 172  * the driver calls the chunks 'step' or 'codeword' interchangeably
 173  */
 174 #define NANDC_STEP_SIZE                 512
 175
 176 /*
 177  * the largest page size we support is 8K, this will have 16 steps/codewords
 178  * of 512 bytes each
 179  */
 180 #define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
 181
 182 /* we read at most 3 registers per codeword scan */
 183 #define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
 184
 185 /* ECC modes supported by the controller */
 186 #define ECC_NONE        BIT(0)
 187 #define ECC_RS_4BIT     BIT(1)
 188 #define ECC_BCH_4BIT    BIT(2)
 189 #define ECC_BCH_8BIT    BIT(3)
 190
 191 #define nandc_set_read_loc(nandc, reg, offset, size, is_last)   \
 192 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg,                  \
 193               ((offset) << READ_LOCATION_OFFSET) |              \
 194               ((size) << READ_LOCATION_SIZE) |                  \
 195               ((is_last) << READ_LOCATION_LAST))
 196
 197 /*
 198  * Returns the actual register address for all NAND_DEV_ registers
 199  * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
 200  */
 201 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
 202
 203 /* Returns the NAND register physical address */
 204 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
 205
 206 /* Returns the dma address for reg read buffer */
 207 #define reg_buf_dma_addr(chip, vaddr) \
 208         ((chip)->reg_read_dma + \
 209         ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
 210
 211 #define QPIC_PER_CW_CMD_ELEMENTS        32
 212 #define QPIC_PER_CW_CMD_SGL             32
 213 #define QPIC_PER_CW_DATA_SGL            8
 214
 215 #define QPIC_NAND_COMPLETION_TIMEOUT    msecs_to_jiffies(2000)
 216
 217 /*
 218  * Flags used in DMA descriptor preparation helper functions
 219  * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
 220  */
 221 /* Don't set the EOT in current tx BAM sgl */
 222 #define NAND_BAM_NO_EOT                 BIT(0)
 223 /* Set the NWD flag in current BAM sgl */
 224 #define NAND_BAM_NWD                    BIT(1)
 225 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
 226 #define NAND_BAM_NEXT_SGL               BIT(2)
 227 /*
 228  * Erased codeword status is being used two times in single transfer so this
 229  * flag will determine the current value of erased codeword status register
 230  */
 231 #define NAND_ERASED_CW_SET              BIT(4)
 232
 233 /*
 234  * This data type corresponds to the BAM transaction which will be used for all
 235  * NAND transfers.
 236  * @bam_ce - the array of BAM command elements
 237  * @cmd_sgl - sgl for NAND BAM command pipe
 238  * @data_sgl - sgl for NAND BAM consumer/producer pipe
 239  * @bam_ce_pos - the index in bam_ce which is available for next sgl
 240  * @bam_ce_start - the index in bam_ce which marks the start position ce
 241  *                 for current sgl. It will be used for size calculation
 242  *                 for current sgl
 243  * @cmd_sgl_pos - current index in command sgl.
 244  * @cmd_sgl_start - start index in command sgl.
 245  * @tx_sgl_pos - current index in data sgl for tx.
 246  * @tx_sgl_start - start index in data sgl for tx.
 247  * @rx_sgl_pos - current index in data sgl for rx.
 248  * @rx_sgl_start - start index in data sgl for rx.
 249  * @wait_second_completion - wait for second DMA desc completion before making
 250  *                           the NAND transfer completion.
 251  * @txn_done - completion for NAND transfer.
 252  * @last_data_desc - last DMA desc in data channel (tx/rx).
 253  * @last_cmd_desc - last DMA desc in command channel.
 254  */
 255 struct bam_transaction {
 256         struct bam_cmd_element *bam_ce;
 257         struct scatterlist *cmd_sgl;
 258         struct scatterlist *data_sgl;
 259         u32 bam_ce_pos;
 260         u32 bam_ce_start;
 261         u32 cmd_sgl_pos;
 262         u32 cmd_sgl_start;
 263         u32 tx_sgl_pos;
 264         u32 tx_sgl_start;
 265         u32 rx_sgl_pos;
 266         u32 rx_sgl_start;
 267         bool wait_second_completion;
 268         struct completion txn_done;
 269         struct dma_async_tx_descriptor *last_data_desc;
 270         struct dma_async_tx_descriptor *last_cmd_desc;
 271 };
 272
 273 /*
 274  * This data type corresponds to the nand dma descriptor
 275  * @list - list for desc_info
 276  * @dir - DMA transfer direction
 277  * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
 278  *            ADM
 279  * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
 280  * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
 281  * @dma_desc - low level DMA engine descriptor
 282  */
 283 struct desc_info {
 284         struct list_head node;
 285
 286         enum dma_data_direction dir;
 287         union {
 288                 struct scatterlist adm_sgl;
 289                 struct {
 290                         struct scatterlist *bam_sgl;
 291                         int sgl_cnt;
 292                 };
 293         };
 294         struct dma_async_tx_descriptor *dma_desc;
 295 };
 296
 297 /*
 298  * holds the current register values that we want to write. acts as a contiguous
 299  * chunk of memory which we use to write the controller registers through DMA.
 300  */
 301 struct nandc_regs {
 302         __le32 cmd;
 303         __le32 addr0;
 304         __le32 addr1;
 305         __le32 chip_sel;
 306         __le32 exec;
 307
 308         __le32 cfg0;
 309         __le32 cfg1;
 310         __le32 ecc_bch_cfg;
 311
 312         __le32 clrflashstatus;
 313         __le32 clrreadstatus;
 314
 315         __le32 cmd1;
 316         __le32 vld;
 317
 318         __le32 orig_cmd1;
 319         __le32 orig_vld;
 320
 321         __le32 ecc_buf_cfg;
 322         __le32 read_location0;
 323         __le32 read_location1;
 324         __le32 read_location2;
 325         __le32 read_location3;
 326
 327         __le32 erased_cw_detect_cfg_clr;
 328         __le32 erased_cw_detect_cfg_set;
 329 };
 330
 331 /*
 332  * NAND controller data struct
 333  *
 334  * @controller:                 base controller structure
 335  * @host_list:                  list containing all the chips attached to the
 336  *                              controller
 337  * @dev:                        parent device
 338  * @base:                       MMIO base
 339  * @base_phys:                  physical base address of controller registers
 340  * @base_dma:                   dma base address of controller registers
 341  * @core_clk:                   controller clock
 342  * @aon_clk:                    another controller clock
 343  *
 344  * @chan:                       dma channel
 345  * @cmd_crci:                   ADM DMA CRCI for command flow control
 346  * @data_crci:                  ADM DMA CRCI for data flow control
 347  * @desc_list:                  DMA descriptor list (list of desc_infos)
 348  *
 349  * @data_buffer:                our local DMA buffer for page read/writes,
 350  *                              used when we can't use the buffer provided
 351  *                              by upper layers directly
 352  * @buf_size/count/start:       markers for chip->legacy.read_buf/write_buf
 353  *                              functions
 354  * @reg_read_buf:               local buffer for reading back registers via DMA
 355  * @reg_read_dma:               contains dma address for register read buffer
 356  * @reg_read_pos:               marker for data read in reg_read_buf
 357  *
 358  * @regs:                       a contiguous chunk of memory for DMA register
 359  *                              writes. contains the register values to be
 360  *                              written to controller
 361  * @cmd1/vld:                   some fixed controller register values
 362  * @props:                      properties of current NAND controller,
 363  *                              initialized via DT match data
 364  * @max_cwperpage:              maximum QPIC codewords required. calculated
 365  *                              from all connected NAND devices pagesize
 366  */
 367 struct qcom_nand_controller {
 368         struct nand_controller controller;
 369         struct list_head host_list;
 370
 371         struct device *dev;
 372
 373         void __iomem *base;
 374         phys_addr_t base_phys;
 375         dma_addr_t base_dma;
 376
 377         struct clk *core_clk;
 378         struct clk *aon_clk;
 379
 380         union {
 381                 /* will be used only by QPIC for BAM DMA */
 382                 struct {
 383                         struct dma_chan *tx_chan;
 384                         struct dma_chan *rx_chan;
 385                         struct dma_chan *cmd_chan;
 386                 };
 387
 388                 /* will be used only by EBI2 for ADM DMA */
 389                 struct {
 390                         struct dma_chan *chan;
 391                         unsigned int cmd_crci;
 392                         unsigned int data_crci;
 393                 };
 394         };
 395
 396         struct list_head desc_list;
 397         struct bam_transaction *bam_txn;
 398
 399         u8              *data_buffer;
 400         int             buf_size;
 401         int             buf_count;
 402         int             buf_start;
 403         unsigned int    max_cwperpage;
 404
 405         __le32 *reg_read_buf;
 406         dma_addr_t reg_read_dma;
 407         int reg_read_pos;
 408
 409         struct nandc_regs *regs;
 410
 411         u32 cmd1, vld;
 412         const struct qcom_nandc_props *props;
 413 };
 414
 415 /*
 416  * NAND chip structure
 417  *
 418  * @chip:                       base NAND chip structure
 419  * @node:                       list node to add itself to host_list in
 420  *                              qcom_nand_controller
 421  *
 422  * @cs:                         chip select value for this chip
 423  * @cw_size:                    the number of bytes in a single step/codeword
 424  *                              of a page, consisting of all data, ecc, spare
 425  *                              and reserved bytes
 426  * @cw_data:                    the number of bytes within a codeword protected
 427  *                              by ECC
 428  * @use_ecc:                    request the controller to use ECC for the
 429  *                              upcoming read/write
 430  * @bch_enabled:                flag to tell whether BCH ECC mode is used
 431  * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 432  *                              chip
 433  * @status:                     value to be returned if NAND_CMD_STATUS command
 434  *                              is executed
 435  * @last_command:               keeps track of last command on this chip. used
 436  *                              for reading correct status
 437  *
 438  * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 439  *                              ecc/non-ecc mode for the current nand flash
 440  *                              device
 441  */
 442 struct qcom_nand_host {
 443         struct nand_chip chip;
 444         struct list_head node;
 445
 446         int cs;
 447         int cw_size;
 448         int cw_data;
 449         bool use_ecc;
 450         bool bch_enabled;
 451         int ecc_bytes_hw;
 452         int spare_bytes;
 453         int bbm_size;
 454         u8 status;
 455         int last_command;
 456
 457         u32 cfg0, cfg1;
 458         u32 cfg0_raw, cfg1_raw;
 459         u32 ecc_buf_cfg;
 460         u32 ecc_bch_cfg;
 461         u32 clrflashstatus;
 462         u32 clrreadstatus;
 463 };
 464
 465 /*
 466  * This data type corresponds to the NAND controller properties which varies
 467  * among different NAND controllers.
 468  * @ecc_modes - ecc mode for NAND
 469  * @is_bam - whether NAND controller is using BAM
 470  * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
 471  */
 472 struct qcom_nandc_props {
 473         u32 ecc_modes;
 474         bool is_bam;
 475         u32 dev_cmd_reg_start;
 476 };
 477
 478 /* Frees the BAM transaction memory */
 479 static void free_bam_transaction(struct qcom_nand_controller *nandc)
 480 {
 481         struct bam_transaction *bam_txn = nandc->bam_txn;
 482
 483         devm_kfree(nandc->dev, bam_txn);
 484 }
 485
 486 /* Allocates and Initializes the BAM transaction */
 487 static struct bam_transaction *
 488 alloc_bam_transaction(struct qcom_nand_controller *nandc)
 489 {
 490         struct bam_transaction *bam_txn;
 491         size_t bam_txn_size;
 492         unsigned int num_cw = nandc->max_cwperpage;
 493         void *bam_txn_buf;
 494
 495         bam_txn_size =
 496                 sizeof(*bam_txn) + num_cw *
 497                 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
 498                 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
 499                 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
 500
 501         bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
 502         if (!bam_txn_buf)
 503                 return NULL;
 504
 505         bam_txn = bam_txn_buf;
 506         bam_txn_buf += sizeof(*bam_txn);
 507
 508         bam_txn->bam_ce = bam_txn_buf;
 509         bam_txn_buf +=
 510                 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
 511
 512         bam_txn->cmd_sgl = bam_txn_buf;
 513         bam_txn_buf +=
 514                 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
 515
 516         bam_txn->data_sgl = bam_txn_buf;
 517
 518         init_completion(&bam_txn->txn_done);
 519
 520         return bam_txn;
 521 }
 522
 523 /* Clears the BAM transaction indexes */
 524 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
 525 {
 526         struct bam_transaction *bam_txn = nandc->bam_txn;
 527
 528         if (!nandc->props->is_bam)
 529                 return;
 530
 531         bam_txn->bam_ce_pos = 0;
 532         bam_txn->bam_ce_start = 0;
 533         bam_txn->cmd_sgl_pos = 0;
 534         bam_txn->cmd_sgl_start = 0;
 535         bam_txn->tx_sgl_pos = 0;
 536         bam_txn->tx_sgl_start = 0;
 537         bam_txn->rx_sgl_pos = 0;
 538         bam_txn->rx_sgl_start = 0;
 539         bam_txn->last_data_desc = NULL;
 540         bam_txn->wait_second_completion = false;
 541
 542         sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
 543                       QPIC_PER_CW_CMD_SGL);
 544         sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
 545                       QPIC_PER_CW_DATA_SGL);
 546
 547         reinit_completion(&bam_txn->txn_done);
 548 }
 549
 550 /* Callback for DMA descriptor completion */
 551 static void qpic_bam_dma_done(void *data)
 552 {
 553         struct bam_transaction *bam_txn = data;
 554
 555         /*
 556          * In case of data transfer with NAND, 2 callbacks will be generated.
 557          * One for command channel and another one for data channel.
 558          * If current transaction has data descriptors
 559          * (i.e. wait_second_completion is true), then set this to false
 560          * and wait for second DMA descriptor completion.
 561          */
 562         if (bam_txn->wait_second_completion)
 563                 bam_txn->wait_second_completion = false;
 564         else
 565                 complete(&bam_txn->txn_done);
 566 }
 567
 568 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 569 {
 570         return container_of(chip, struct qcom_nand_host, chip);
 571 }
 572
 573 static inline struct qcom_nand_controller *
 574 get_qcom_nand_controller(struct nand_chip *chip)
 575 {
 576         return container_of(chip->controller, struct qcom_nand_controller,
 577                             controller);
 578 }
 579
 580 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
 581 {
 582         return ioread32(nandc->base + offset);
 583 }
 584
 585 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 586                                u32 val)
 587 {
 588         iowrite32(val, nandc->base + offset);
 589 }
 590
 591 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
 592                                           bool is_cpu)
 593 {
 594         if (!nandc->props->is_bam)
 595                 return;
 596
 597         if (is_cpu)
 598                 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
 599                                         MAX_REG_RD *
 600                                         sizeof(*nandc->reg_read_buf),
 601                                         DMA_FROM_DEVICE);
 602         else
 603                 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
 604                                            MAX_REG_RD *
 605                                            sizeof(*nandc->reg_read_buf),
 606                                            DMA_FROM_DEVICE);
 607 }
 608
 609 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 610 {
 611         switch (offset) {
 612         case NAND_FLASH_CMD:
 613                 return &regs->cmd;
 614         case NAND_ADDR0:
 615                 return &regs->addr0;
 616         case NAND_ADDR1:
 617                 return &regs->addr1;
 618         case NAND_FLASH_CHIP_SELECT:
 619                 return &regs->chip_sel;
 620         case NAND_EXEC_CMD:
 621                 return &regs->exec;
 622         case NAND_FLASH_STATUS:
 623                 return &regs->clrflashstatus;
 624         case NAND_DEV0_CFG0:
 625                 return &regs->cfg0;
 626         case NAND_DEV0_CFG1:
 627                 return &regs->cfg1;
 628         case NAND_DEV0_ECC_CFG:
 629                 return &regs->ecc_bch_cfg;
 630         case NAND_READ_STATUS:
 631                 return &regs->clrreadstatus;
 632         case NAND_DEV_CMD1:
 633                 return &regs->cmd1;
 634         case NAND_DEV_CMD1_RESTORE:
 635                 return &regs->orig_cmd1;
 636         case NAND_DEV_CMD_VLD:
 637                 return &regs->vld;
 638         case NAND_DEV_CMD_VLD_RESTORE:
 639                 return &regs->orig_vld;
 640         case NAND_EBI2_ECC_BUF_CFG:
 641                 return &regs->ecc_buf_cfg;
 642         case NAND_READ_LOCATION_0:
 643                 return &regs->read_location0;
 644         case NAND_READ_LOCATION_1:
 645                 return &regs->read_location1;
 646         case NAND_READ_LOCATION_2:
 647                 return &regs->read_location2;
 648         case NAND_READ_LOCATION_3:
 649                 return &regs->read_location3;
 650         default:
 651                 return NULL;
 652         }
 653 }
 654
 655 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
 656                           u32 val)
 657 {
 658         struct nandc_regs *regs = nandc->regs;
 659         __le32 *reg;
 660
 661         reg = offset_to_nandc_reg(regs, offset);
 662
 663         if (reg)
 664                 *reg = cpu_to_le32(val);
 665 }
 666
 667 /* helper to configure address register values */
 668 static void set_address(struct qcom_nand_host *host, u16 column, int page)
 669 {
 670         struct nand_chip *chip = &host->chip;
 671         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 672
 673         if (chip->options & NAND_BUSWIDTH_16)
 674                 column >>= 1;
 675
 676         nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
 677         nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
 678 }
 679
 680 /*
 681  * update_rw_regs:      set up read/write register values, these will be
 682  *                      written to the NAND controller registers via DMA
 683  *
 684  * @num_cw:             number of steps for the read/write operation
 685  * @read:               read or write operation
 686  */
 687 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
 688 {
 689         struct nand_chip *chip = &host->chip;
 690         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 691         u32 cmd, cfg0, cfg1, ecc_bch_cfg;
 692
 693         if (read) {
 694                 if (host->use_ecc)
 695                         cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
 696                 else
 697                         cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
 698         } else {
 699                 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
 700         }
 701
 702         if (host->use_ecc) {
 703                 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
 704                                 (num_cw - 1) << CW_PER_PAGE;
 705
 706                 cfg1 = host->cfg1;
 707                 ecc_bch_cfg = host->ecc_bch_cfg;
 708         } else {
 709                 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
 710                                 (num_cw - 1) << CW_PER_PAGE;
 711
 712                 cfg1 = host->cfg1_raw;
 713                 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
 714         }
 715
 716         nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
 717         nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
 718         nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
 719         nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
 720         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
 721         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 722         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 723         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 724
 725         if (read)
 726                 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
 727                                    host->cw_data : host->cw_size, 1);
 728 }
 729
 730 /*
 731  * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
 732  * for BAM. This descriptor will be added in the NAND DMA descriptor queue
 733  * which will be submitted to DMA engine.
 734  */
 735 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
 736                                   struct dma_chan *chan,
 737                                   unsigned long flags)
 738 {
 739         struct desc_info *desc;
 740         struct scatterlist *sgl;
 741         unsigned int sgl_cnt;
 742         int ret;
 743         struct bam_transaction *bam_txn = nandc->bam_txn;
 744         enum dma_transfer_direction dir_eng;
 745         struct dma_async_tx_descriptor *dma_desc;
 746
 747         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 748         if (!desc)
 749                 return -ENOMEM;
 750
 751         if (chan == nandc->cmd_chan) {
 752                 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
 753                 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
 754                 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
 755                 dir_eng = DMA_MEM_TO_DEV;
 756                 desc->dir = DMA_TO_DEVICE;
 757         } else if (chan == nandc->tx_chan) {
 758                 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
 759                 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
 760                 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
 761                 dir_eng = DMA_MEM_TO_DEV;
 762                 desc->dir = DMA_TO_DEVICE;
 763         } else {
 764                 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
 765                 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
 766                 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
 767                 dir_eng = DMA_DEV_TO_MEM;
 768                 desc->dir = DMA_FROM_DEVICE;
 769         }
 770
 771         sg_mark_end(sgl + sgl_cnt - 1);
 772         ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 773         if (ret == 0) {
 774                 dev_err(nandc->dev, "failure in mapping desc\n");
 775                 kfree(desc);
 776                 return -ENOMEM;
 777         }
 778
 779         desc->sgl_cnt = sgl_cnt;
 780         desc->bam_sgl = sgl;
 781
 782         dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
 783                                            flags);
 784
 785         if (!dma_desc) {
 786                 dev_err(nandc->dev, "failure in prep desc\n");
 787                 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 788                 kfree(desc);
 789                 return -EINVAL;
 790         }
 791
 792         desc->dma_desc = dma_desc;
 793
 794         /* update last data/command descriptor */
 795         if (chan == nandc->cmd_chan)
 796                 bam_txn->last_cmd_desc = dma_desc;
 797         else
 798                 bam_txn->last_data_desc = dma_desc;
 799
 800         list_add_tail(&desc->node, &nandc->desc_list);
 801
 802         return 0;
 803 }
 804
 805 /*
 806  * Prepares the command descriptor for BAM DMA which will be used for NAND
 807  * register reads and writes. The command descriptor requires the command
 808  * to be formed in command element type so this function uses the command
 809  * element from bam transaction ce array and fills the same with required
 810  * data. A single SGL can contain multiple command elements so
 811  * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
 812  * after the current command element.
 813  */
 814 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
 815                                  int reg_off, const void *vaddr,
 816                                  int size, unsigned int flags)
 817 {
 818         int bam_ce_size;
 819         int i, ret;
 820         struct bam_cmd_element *bam_ce_buffer;
 821         struct bam_transaction *bam_txn = nandc->bam_txn;
 822
 823         bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
 824
 825         /* fill the command desc */
 826         for (i = 0; i < size; i++) {
 827                 if (read)
 828                         bam_prep_ce(&bam_ce_buffer[i],
 829                                     nandc_reg_phys(nandc, reg_off + 4 * i),
 830                                     BAM_READ_COMMAND,
 831                                     reg_buf_dma_addr(nandc,
 832                                                      (__le32 *)vaddr + i));
 833                 else
 834                         bam_prep_ce_le32(&bam_ce_buffer[i],
 835                                          nandc_reg_phys(nandc, reg_off + 4 * i),
 836                                          BAM_WRITE_COMMAND,
 837                                          *((__le32 *)vaddr + i));
 838         }
 839
 840         bam_txn->bam_ce_pos += size;
 841
 842         /* use the separate sgl after this command */
 843         if (flags & NAND_BAM_NEXT_SGL) {
 844                 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
 845                 bam_ce_size = (bam_txn->bam_ce_pos -
 846                                 bam_txn->bam_ce_start) *
 847                                 sizeof(struct bam_cmd_element);
 848                 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
 849                            bam_ce_buffer, bam_ce_size);
 850                 bam_txn->cmd_sgl_pos++;
 851                 bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
 852
 853                 if (flags & NAND_BAM_NWD) {
 854                         ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
 855                                                      DMA_PREP_FENCE |
 856                                                      DMA_PREP_CMD);
 857                         if (ret)
 858                                 return ret;
 859                 }
 860         }
 861
 862         return 0;
 863 }
 864
 865 /*
 866  * Prepares the data descriptor for BAM DMA which will be used for NAND
 867  * data reads and writes.
 868  */
 869 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
 870                                   const void *vaddr,
 871                                   int size, unsigned int flags)
 872 {
 873         int ret;
 874         struct bam_transaction *bam_txn = nandc->bam_txn;
 875
 876         if (read) {
 877                 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
 878                            vaddr, size);
 879                 bam_txn->rx_sgl_pos++;
 880         } else {
 881                 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
 882                            vaddr, size);
 883                 bam_txn->tx_sgl_pos++;
 884
 885                 /*
 886                  * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
 887                  * is not set, form the DMA descriptor
 888                  */
 889                 if (!(flags & NAND_BAM_NO_EOT)) {
 890                         ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
 891                                                      DMA_PREP_INTERRUPT);
 892                         if (ret)
 893                                 return ret;
 894                 }
 895         }
 896
 897         return 0;
 898 }
 899
 900 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
 901                              int reg_off, const void *vaddr, int size,
 902                              bool flow_control)
 903 {
 904         struct desc_info *desc;
 905         struct dma_async_tx_descriptor *dma_desc;
 906         struct scatterlist *sgl;
 907         struct dma_slave_config slave_conf;
 908         enum dma_transfer_direction dir_eng;
 909         int ret;
 910
 911         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 912         if (!desc)
 913                 return -ENOMEM;
 914
 915         sgl = &desc->adm_sgl;
 916
 917         sg_init_one(sgl, vaddr, size);
 918
 919         if (read) {
 920                 dir_eng = DMA_DEV_TO_MEM;
 921                 desc->dir = DMA_FROM_DEVICE;
 922         } else {
 923                 dir_eng = DMA_MEM_TO_DEV;
 924                 desc->dir = DMA_TO_DEVICE;
 925         }
 926
 927         ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
 928         if (ret == 0) {
 929                 ret = -ENOMEM;
 930                 goto err;
 931         }
 932
 933         memset(&slave_conf, 0x00, sizeof(slave_conf));
 934
 935         slave_conf.device_fc = flow_control;
 936         if (read) {
 937                 slave_conf.src_maxburst = 16;
 938                 slave_conf.src_addr = nandc->base_dma + reg_off;
 939                 slave_conf.slave_id = nandc->data_crci;
 940         } else {
 941                 slave_conf.dst_maxburst = 16;
 942                 slave_conf.dst_addr = nandc->base_dma + reg_off;
 943                 slave_conf.slave_id = nandc->cmd_crci;
 944         }
 945
 946         ret = dmaengine_slave_config(nandc->chan, &slave_conf);
 947         if (ret) {
 948                 dev_err(nandc->dev, "failed to configure dma channel\n");
 949                 goto err;
 950         }
 951
 952         dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
 953         if (!dma_desc) {
 954                 dev_err(nandc->dev, "failed to prepare desc\n");
 955                 ret = -EINVAL;
 956                 goto err;
 957         }
 958
 959         desc->dma_desc = dma_desc;
 960
 961         list_add_tail(&desc->node, &nandc->desc_list);
 962
 963         return 0;
 964 err:
 965         kfree(desc);
 966
 967         return ret;
 968 }
 969
 970 /*
 971  * read_reg_dma:        prepares a descriptor to read a given number of
 972  *                      contiguous registers to the reg_read_buf pointer
 973  *
 974  * @first:              offset of the first register in the contiguous block
 975  * @num_regs:           number of registers to read
 976  * @flags:              flags to control DMA descriptor preparation
 977  */
 978 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
 979                         int num_regs, unsigned int flags)
 980 {
 981         bool flow_control = false;
 982         void *vaddr;
 983
 984         vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
 985         nandc->reg_read_pos += num_regs;
 986
 987         if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
 988                 first = dev_cmd_reg_addr(nandc, first);
 989
 990         if (nandc->props->is_bam)
 991                 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
 992                                              num_regs, flags);
 993
 994         if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
 995                 flow_control = true;
 996
 997         return prep_adm_dma_desc(nandc, true, first, vaddr,
 998                                  num_regs * sizeof(u32), flow_control);
 999 }
1000
1001 /*
1002  * write_reg_dma:       prepares a descriptor to write a given number of
1003  *                      contiguous registers
1004  *
1005  * @first:              offset of the first register in the contiguous block
1006  * @num_regs:           number of registers to write
1007  * @flags:              flags to control DMA descriptor preparation
1008  */
1009 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
1010                          int num_regs, unsigned int flags)
1011 {
1012         bool flow_control = false;
1013         struct nandc_regs *regs = nandc->regs;
1014         void *vaddr;
1015
1016         vaddr = offset_to_nandc_reg(regs, first);
1017
1018         if (first == NAND_ERASED_CW_DETECT_CFG) {
1019                 if (flags & NAND_ERASED_CW_SET)
1020                         vaddr = &regs->erased_cw_detect_cfg_set;
1021                 else
1022                         vaddr = &regs->erased_cw_detect_cfg_clr;
1023         }
1024
1025         if (first == NAND_EXEC_CMD)
1026                 flags |= NAND_BAM_NWD;
1027
1028         if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
1029                 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
1030
1031         if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
1032                 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
1033
1034         if (nandc->props->is_bam)
1035                 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
1036                                              num_regs, flags);
1037
1038         if (first == NAND_FLASH_CMD)
1039                 flow_control = true;
1040
1041         return prep_adm_dma_desc(nandc, false, first, vaddr,
1042                                  num_regs * sizeof(u32), flow_control);
1043 }
1044
1045 /*
1046  * read_data_dma:       prepares a DMA descriptor to transfer data from the
1047  *                      controller's internal buffer to the buffer 'vaddr'
1048  *
1049  * @reg_off:            offset within the controller's data buffer
1050  * @vaddr:              virtual address of the buffer we want to write to
1051  * @size:               DMA transaction size in bytes
1052  * @flags:              flags to control DMA descriptor preparation
1053  */
1054 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1055                          const u8 *vaddr, int size, unsigned int flags)
1056 {
1057         if (nandc->props->is_bam)
1058                 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1059
1060         return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1061 }
1062
1063 /*
1064  * write_data_dma:      prepares a DMA descriptor to transfer data from
1065  *                      'vaddr' to the controller's internal buffer
1066  *
1067  * @reg_off:            offset within the controller's data buffer
1068  * @vaddr:              virtual address of the buffer we want to read from
1069  * @size:               DMA transaction size in bytes
1070  * @flags:              flags to control DMA descriptor preparation
1071  */
1072 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1073                           const u8 *vaddr, int size, unsigned int flags)
1074 {
1075         if (nandc->props->is_bam)
1076                 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1077
1078         return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1079 }
1080
1081 /*
1082  * Helper to prepare DMA descriptors for configuring registers
1083  * before reading a NAND page.
1084  */
1085 static void config_nand_page_read(struct qcom_nand_controller *nandc)
1086 {
1087         write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1088         write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1089         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1090         write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1091         write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1092                       NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1093 }
1094
1095 /*
1096  * Helper to prepare DMA descriptors for configuring registers
1097  * before reading each codeword in NAND page.
1098  */
1099 static void
1100 config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc)
1101 {
1102         if (nandc->props->is_bam)
1103                 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
1104                               NAND_BAM_NEXT_SGL);
1105
1106         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1107         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1108
1109         if (use_ecc) {
1110                 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1111                 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1112                              NAND_BAM_NEXT_SGL);
1113         } else {
1114                 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1115         }
1116 }
1117
1118 /*
1119  * Helper to prepare dma descriptors to configure registers needed for reading a
1120  * single codeword in page
1121  */
1122 static void
1123 config_nand_single_cw_page_read(struct qcom_nand_controller *nandc,
1124                                 bool use_ecc)
1125 {
1126         config_nand_page_read(nandc);
1127         config_nand_cw_read(nandc, use_ecc);
1128 }
1129
1130 /*
1131  * Helper to prepare DMA descriptors used to configure registers needed for
1132  * before writing a NAND page.
1133  */
1134 static void config_nand_page_write(struct qcom_nand_controller *nandc)
1135 {
1136         write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1137         write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1138         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1139                       NAND_BAM_NEXT_SGL);
1140 }
1141
1142 /*
1143  * Helper to prepare DMA descriptors for configuring registers
1144  * before writing each codeword in NAND page.
1145  */
1146 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1147 {
1148         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1149         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1150
1151         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1152
1153         write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1154         write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1155 }
1156
1157 /*
1158  * the following functions are used within chip->legacy.cmdfunc() to
1159  * perform different NAND_CMD_* commands
1160  */
1161
1162 /* sets up descriptors for NAND_CMD_PARAM */
1163 static int nandc_param(struct qcom_nand_host *host)
1164 {
1165         struct nand_chip *chip = &host->chip;
1166         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1167
1168         /*
1169          * NAND_CMD_PARAM is called before we know much about the FLASH chip
1170          * in use. we configure the controller to perform a raw read of 512
1171          * bytes to read onfi params
1172          */
1173         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
1174         nandc_set_reg(nandc, NAND_ADDR0, 0);
1175         nandc_set_reg(nandc, NAND_ADDR1, 0);
1176         nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1177                                         | 512 << UD_SIZE_BYTES
1178                                         | 5 << NUM_ADDR_CYCLES
1179                                         | 0 << SPARE_SIZE_BYTES);
1180         nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1181                                         | 0 << CS_ACTIVE_BSY
1182                                         | 17 << BAD_BLOCK_BYTE_NUM
1183                                         | 1 << BAD_BLOCK_IN_SPARE_AREA
1184                                         | 2 << WR_RD_BSY_GAP
1185                                         | 0 << WIDE_FLASH
1186                                         | 1 << DEV0_CFG1_ECC_DISABLE);
1187         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1188
1189         /* configure CMD1 and VLD for ONFI param probing */
1190         nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1191                       (nandc->vld & ~READ_START_VLD));
1192         nandc_set_reg(nandc, NAND_DEV_CMD1,
1193                       (nandc->cmd1 & ~(0xFF << READ_ADDR))
1194                       | NAND_CMD_PARAM << READ_ADDR);
1195
1196         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1197
1198         nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1199         nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1200         nandc_set_read_loc(nandc, 0, 0, 512, 1);
1201
1202         write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1203         write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1204
1205         nandc->buf_count = 512;
1206         memset(nandc->data_buffer, 0xff, nandc->buf_count);
1207
1208         config_nand_single_cw_page_read(nandc, false);
1209
1210         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1211                       nandc->buf_count, 0);
1212
1213         /* restore CMD1 and VLD regs */
1214         write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1215         write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1216
1217         return 0;
1218 }
1219
1220 /* sets up descriptors for NAND_CMD_ERASE1 */
1221 static int erase_block(struct qcom_nand_host *host, int page_addr)
1222 {
1223         struct nand_chip *chip = &host->chip;
1224         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1225
1226         nandc_set_reg(nandc, NAND_FLASH_CMD,
1227                       OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1228         nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1229         nandc_set_reg(nandc, NAND_ADDR1, 0);
1230         nandc_set_reg(nandc, NAND_DEV0_CFG0,
1231                       host->cfg0_raw & ~(7 << CW_PER_PAGE));
1232         nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1233         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1234         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1235         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1236
1237         write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1238         write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1239         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1240
1241         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1242
1243         write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1244         write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1245
1246         return 0;
1247 }
1248
1249 /* sets up descriptors for NAND_CMD_READID */
1250 static int read_id(struct qcom_nand_host *host, int column)
1251 {
1252         struct nand_chip *chip = &host->chip;
1253         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1254
1255         if (column == -1)
1256                 return 0;
1257
1258         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
1259         nandc_set_reg(nandc, NAND_ADDR0, column);
1260         nandc_set_reg(nandc, NAND_ADDR1, 0);
1261         nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1262                       nandc->props->is_bam ? 0 : DM_EN);
1263         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1264
1265         write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1266         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1267
1268         read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1269
1270         return 0;
1271 }
1272
1273 /* sets up descriptors for NAND_CMD_RESET */
1274 static int reset(struct qcom_nand_host *host)
1275 {
1276         struct nand_chip *chip = &host->chip;
1277         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1278
1279         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
1280         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1281
1282         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1283         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1284
1285         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1286
1287         return 0;
1288 }
1289
1290 /* helpers to submit/free our list of dma descriptors */
1291 static int submit_descs(struct qcom_nand_controller *nandc)
1292 {
1293         struct desc_info *desc;
1294         dma_cookie_t cookie = 0;
1295         struct bam_transaction *bam_txn = nandc->bam_txn;
1296         int r;
1297
1298         if (nandc->props->is_bam) {
1299                 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1300                         r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1301                         if (r)
1302                                 return r;
1303                 }
1304
1305                 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1306                         r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1307                                                    DMA_PREP_INTERRUPT);
1308                         if (r)
1309                                 return r;
1310                 }
1311
1312                 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1313                         r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1314                                                    DMA_PREP_CMD);
1315                         if (r)
1316                                 return r;
1317                 }
1318         }
1319
1320         list_for_each_entry(desc, &nandc->desc_list, node)
1321                 cookie = dmaengine_submit(desc->dma_desc);
1322
1323         if (nandc->props->is_bam) {
1324                 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done;
1325                 bam_txn->last_cmd_desc->callback_param = bam_txn;
1326                 if (bam_txn->last_data_desc) {
1327                         bam_txn->last_data_desc->callback = qpic_bam_dma_done;
1328                         bam_txn->last_data_desc->callback_param = bam_txn;
1329                         bam_txn->wait_second_completion = true;
1330                 }
1331
1332                 dma_async_issue_pending(nandc->tx_chan);
1333                 dma_async_issue_pending(nandc->rx_chan);
1334                 dma_async_issue_pending(nandc->cmd_chan);
1335
1336                 if (!wait_for_completion_timeout(&bam_txn->txn_done,
1337                                                  QPIC_NAND_COMPLETION_TIMEOUT))
1338                         return -ETIMEDOUT;
1339         } else {
1340                 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1341                         return -ETIMEDOUT;
1342         }
1343
1344         return 0;
1345 }
1346
1347 static void free_descs(struct qcom_nand_controller *nandc)
1348 {
1349         struct desc_info *desc, *n;
1350
1351         list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1352                 list_del(&desc->node);
1353
1354                 if (nandc->props->is_bam)
1355                         dma_unmap_sg(nandc->dev, desc->bam_sgl,
1356                                      desc->sgl_cnt, desc->dir);
1357                 else
1358                         dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1359                                      desc->dir);
1360
1361                 kfree(desc);
1362         }
1363 }
1364
1365 /* reset the register read buffer for next NAND operation */
1366 static void clear_read_regs(struct qcom_nand_controller *nandc)
1367 {
1368         nandc->reg_read_pos = 0;
1369         nandc_read_buffer_sync(nandc, false);
1370 }
1371
1372 static void pre_command(struct qcom_nand_host *host, int command)
1373 {
1374         struct nand_chip *chip = &host->chip;
1375         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1376
1377         nandc->buf_count = 0;
1378         nandc->buf_start = 0;
1379         host->use_ecc = false;
1380         host->last_command = command;
1381
1382         clear_read_regs(nandc);
1383
1384         if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1385             command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1386                 clear_bam_transaction(nandc);
1387 }
1388
1389 /*
1390  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1391  * privately maintained status byte, this status byte can be read after
1392  * NAND_CMD_STATUS is called
1393  */
1394 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1395 {
1396         struct nand_chip *chip = &host->chip;
1397         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1398         struct nand_ecc_ctrl *ecc = &chip->ecc;
1399         int num_cw;
1400         int i;
1401
1402         num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1403         nandc_read_buffer_sync(nandc, true);
1404
1405         for (i = 0; i < num_cw; i++) {
1406                 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1407
1408                 if (flash_status & FS_MPU_ERR)
1409                         host->status &= ~NAND_STATUS_WP;
1410
1411                 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1412                                                  (flash_status &
1413                                                   FS_DEVICE_STS_ERR)))
1414                         host->status |= NAND_STATUS_FAIL;
1415         }
1416 }
1417
1418 static void post_command(struct qcom_nand_host *host, int command)
1419 {
1420         struct nand_chip *chip = &host->chip;
1421         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1422
1423         switch (command) {
1424         case NAND_CMD_READID:
1425                 nandc_read_buffer_sync(nandc, true);
1426                 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1427                        nandc->buf_count);
1428                 break;
1429         case NAND_CMD_PAGEPROG:
1430         case NAND_CMD_ERASE1:
1431                 parse_erase_write_errors(host, command);
1432                 break;
1433         default:
1434                 break;
1435         }
1436 }
1437
1438 /*
1439  * Implements chip->legacy.cmdfunc. It's  only used for a limited set of
1440  * commands. The rest of the commands wouldn't be called by upper layers.
1441  * For example, NAND_CMD_READOOB would never be called because we have our own
1442  * versions of read_oob ops for nand_ecc_ctrl.
1443  */
1444 static void qcom_nandc_command(struct nand_chip *chip, unsigned int command,
1445                                int column, int page_addr)
1446 {
1447         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1448         struct nand_ecc_ctrl *ecc = &chip->ecc;
1449         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1450         bool wait = false;
1451         int ret = 0;
1452
1453         pre_command(host, command);
1454
1455         switch (command) {
1456         case NAND_CMD_RESET:
1457                 ret = reset(host);
1458                 wait = true;
1459                 break;
1460
1461         case NAND_CMD_READID:
1462                 nandc->buf_count = 4;
1463                 ret = read_id(host, column);
1464                 wait = true;
1465                 break;
1466
1467         case NAND_CMD_PARAM:
1468                 ret = nandc_param(host);
1469                 wait = true;
1470                 break;
1471
1472         case NAND_CMD_ERASE1:
1473                 ret = erase_block(host, page_addr);
1474                 wait = true;
1475                 break;
1476
1477         case NAND_CMD_READ0:
1478                 /* we read the entire page for now */
1479                 WARN_ON(column != 0);
1480
1481                 host->use_ecc = true;
1482                 set_address(host, 0, page_addr);
1483                 update_rw_regs(host, ecc->steps, true);
1484                 break;
1485
1486         case NAND_CMD_SEQIN:
1487                 WARN_ON(column != 0);
1488                 set_address(host, 0, page_addr);
1489                 break;
1490
1491         case NAND_CMD_PAGEPROG:
1492         case NAND_CMD_STATUS:
1493         case NAND_CMD_NONE:
1494         default:
1495                 break;
1496         }
1497
1498         if (ret) {
1499                 dev_err(nandc->dev, "failure executing command %d\n",
1500                         command);
1501                 free_descs(nandc);
1502                 return;
1503         }
1504
1505         if (wait) {
1506                 ret = submit_descs(nandc);
1507                 if (ret)
1508                         dev_err(nandc->dev,
1509                                 "failure submitting descs for command %d\n",
1510                                 command);
1511         }
1512
1513         free_descs(nandc);
1514
1515         post_command(host, command);
1516 }
1517
1518 /*
1519  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1520  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1521  *
1522  * when using RS ECC, the HW reports the same erros when reading an erased CW,
1523  * but it notifies that it is an erased CW by placing special characters at
1524  * certain offsets in the buffer.
1525  *
1526  * verify if the page is erased or not, and fix up the page for RS ECC by
1527  * replacing the special characters with 0xff.
1528  */
1529 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1530 {
1531         u8 empty1, empty2;
1532
1533         /*
1534          * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1535          * is erased by looking for 0x54s at offsets 3 and 175 from the
1536          * beginning of each codeword
1537          */
1538
1539         empty1 = data_buf[3];
1540         empty2 = data_buf[175];
1541
1542         /*
1543          * if the erased codework markers, if they exist override them with
1544          * 0xffs
1545          */
1546         if ((empty1 == 0x54 && empty2 == 0xff) ||
1547             (empty1 == 0xff && empty2 == 0x54)) {
1548                 data_buf[3] = 0xff;
1549                 data_buf[175] = 0xff;
1550         }
1551
1552         /*
1553          * check if the entire chunk contains 0xffs or not. if it doesn't, then
1554          * restore the original values at the special offsets
1555          */
1556         if (memchr_inv(data_buf, 0xff, data_len)) {
1557                 data_buf[3] = empty1;
1558                 data_buf[175] = empty2;
1559
1560                 return false;
1561         }
1562
1563         return true;
1564 }
1565
1566 struct read_stats {
1567         __le32 flash;
1568         __le32 buffer;
1569         __le32 erased_cw;
1570 };
1571
1572 /* reads back FLASH_STATUS register set by the controller */
1573 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
1574 {
1575         struct nand_chip *chip = &host->chip;
1576         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1577         int i;
1578
1579         for (i = 0; i < cw_cnt; i++) {
1580                 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
1581
1582                 if (flash & (FS_OP_ERR | FS_MPU_ERR))
1583                         return -EIO;
1584         }
1585
1586         return 0;
1587 }
1588
1589 /* performs raw read for one codeword */
1590 static int
1591 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
1592                        u8 *data_buf, u8 *oob_buf, int page, int cw)
1593 {
1594         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1595         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1596         struct nand_ecc_ctrl *ecc = &chip->ecc;
1597         int data_size1, data_size2, oob_size1, oob_size2;
1598         int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
1599
1600         nand_read_page_op(chip, page, 0, NULL, 0);
1601         host->use_ecc = false;
1602
1603         clear_bam_transaction(nandc);
1604         set_address(host, host->cw_size * cw, page);
1605         update_rw_regs(host, 1, true);
1606         config_nand_page_read(nandc);
1607
1608         data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1609         oob_size1 = host->bbm_size;
1610
1611         if (cw == (ecc->steps - 1)) {
1612                 data_size2 = ecc->size - data_size1 -
1613                              ((ecc->steps - 1) * 4);
1614                 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
1615                             host->spare_bytes;
1616         } else {
1617                 data_size2 = host->cw_data - data_size1;
1618                 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1619         }
1620
1621         if (nandc->props->is_bam) {
1622                 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1623                 read_loc += data_size1;
1624
1625                 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1626                 read_loc += oob_size1;
1627
1628                 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1629                 read_loc += data_size2;
1630
1631                 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1632         }
1633
1634         config_nand_cw_read(nandc, false);
1635
1636         read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1637         reg_off += data_size1;
1638
1639         read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1640         reg_off += oob_size1;
1641
1642         read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0);
1643         reg_off += data_size2;
1644
1645         read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0);
1646
1647         ret = submit_descs(nandc);
1648         free_descs(nandc);
1649         if (ret) {
1650                 dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
1651                 return ret;
1652         }
1653
1654         return check_flash_errors(host, 1);
1655 }
1656
1657 /*
1658  * Bitflips can happen in erased codewords also so this function counts the
1659  * number of 0 in each CW for which ECC engine returns the uncorrectable
1660  * error. The page will be assumed as erased if this count is less than or
1661  * equal to the ecc->strength for each CW.
1662  *
1663  * 1. Both DATA and OOB need to be checked for number of 0. The
1664  *    top-level API can be called with only data buf or OOB buf so use
1665  *    chip->data_buf if data buf is null and chip->oob_poi if oob buf
1666  *    is null for copying the raw bytes.
1667  * 2. Perform raw read for all the CW which has uncorrectable errors.
1668  * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1669  *    The BBM and spare bytes bit flip won’t affect the ECC so don’t check
1670  *    the number of bitflips in this area.
1671  */
1672 static int
1673 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
1674                       u8 *oob_buf, unsigned long uncorrectable_cws,
1675                       int page, unsigned int max_bitflips)
1676 {
1677         struct nand_chip *chip = &host->chip;
1678         struct mtd_info *mtd = nand_to_mtd(chip);
1679         struct nand_ecc_ctrl *ecc = &chip->ecc;
1680         u8 *cw_data_buf, *cw_oob_buf;
1681         int cw, data_size, oob_size, ret = 0;
1682
1683         if (!data_buf) {
1684                 data_buf = chip->data_buf;
1685                 chip->pagebuf = -1;
1686         }
1687
1688         if (!oob_buf) {
1689                 oob_buf = chip->oob_poi;
1690                 chip->pagebuf = -1;
1691         }
1692
1693         for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
1694                 if (cw == (ecc->steps - 1)) {
1695                         data_size = ecc->size - ((ecc->steps - 1) * 4);
1696                         oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
1697                 } else {
1698                         data_size = host->cw_data;
1699                         oob_size = host->ecc_bytes_hw;
1700                 }
1701
1702                 /* determine starting buffer address for current CW */
1703                 cw_data_buf = data_buf + (cw * host->cw_data);
1704                 cw_oob_buf = oob_buf + (cw * ecc->bytes);
1705
1706                 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf,
1707                                              cw_oob_buf, page, cw);
1708                 if (ret)
1709                         return ret;
1710
1711                 /*
1712                  * make sure it isn't an erased page reported
1713                  * as not-erased by HW because of a few bitflips
1714                  */
1715                 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size,
1716                                                   cw_oob_buf + host->bbm_size,
1717                                                   oob_size, NULL,
1718                                                   0, ecc->strength);
1719                 if (ret < 0) {
1720                         mtd->ecc_stats.failed++;
1721                 } else {
1722                         mtd->ecc_stats.corrected += ret;
1723                         max_bitflips = max_t(unsigned int, max_bitflips, ret);
1724                 }
1725         }
1726
1727         return max_bitflips;
1728 }
1729
1730 /*
1731  * reads back status registers set by the controller to notify page read
1732  * errors. this is equivalent to what 'ecc->correct()' would do.
1733  */
1734 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1735                              u8 *oob_buf, int page)
1736 {
1737         struct nand_chip *chip = &host->chip;
1738         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1739         struct mtd_info *mtd = nand_to_mtd(chip);
1740         struct nand_ecc_ctrl *ecc = &chip->ecc;
1741         unsigned int max_bitflips = 0, uncorrectable_cws = 0;
1742         struct read_stats *buf;
1743         bool flash_op_err = false, erased;
1744         int i;
1745         u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1746
1747         buf = (struct read_stats *)nandc->reg_read_buf;
1748         nandc_read_buffer_sync(nandc, true);
1749
1750         for (i = 0; i < ecc->steps; i++, buf++) {
1751                 u32 flash, buffer, erased_cw;
1752                 int data_len, oob_len;
1753
1754                 if (i == (ecc->steps - 1)) {
1755                         data_len = ecc->size - ((ecc->steps - 1) << 2);
1756                         oob_len = ecc->steps << 2;
1757                 } else {
1758                         data_len = host->cw_data;
1759                         oob_len = 0;
1760                 }
1761
1762                 flash = le32_to_cpu(buf->flash);
1763                 buffer = le32_to_cpu(buf->buffer);
1764                 erased_cw = le32_to_cpu(buf->erased_cw);
1765
1766                 /*
1767                  * Check ECC failure for each codeword. ECC failure can
1768                  * happen in either of the following conditions
1769                  * 1. If number of bitflips are greater than ECC engine
1770                  *    capability.
1771                  * 2. If this codeword contains all 0xff for which erased
1772                  *    codeword detection check will be done.
1773                  */
1774                 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
1775                         /*
1776                          * For BCH ECC, ignore erased codeword errors, if
1777                          * ERASED_CW bits are set.
1778                          */
1779                         if (host->bch_enabled) {
1780                                 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1781                                          true : false;
1782                         /*
1783                          * For RS ECC, HW reports the erased CW by placing
1784                          * special characters at certain offsets in the buffer.
1785                          * These special characters will be valid only if
1786                          * complete page is read i.e. data_buf is not NULL.
1787                          */
1788                         } else if (data_buf) {
1789                                 erased = erased_chunk_check_and_fixup(data_buf,
1790                                                                       data_len);
1791                         } else {
1792                                 erased = false;
1793                         }
1794
1795                         if (!erased)
1796                                 uncorrectable_cws |= BIT(i);
1797                 /*
1798                  * Check if MPU or any other operational error (timeout,
1799                  * device failure, etc.) happened for this codeword and
1800                  * make flash_op_err true. If flash_op_err is set, then
1801                  * EIO will be returned for page read.
1802                  */
1803                 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1804                         flash_op_err = true;
1805                 /*
1806                  * No ECC or operational errors happened. Check the number of
1807                  * bits corrected and update the ecc_stats.corrected.
1808                  */
1809                 } else {
1810                         unsigned int stat;
1811
1812                         stat = buffer & BS_CORRECTABLE_ERR_MSK;
1813                         mtd->ecc_stats.corrected += stat;
1814                         max_bitflips = max(max_bitflips, stat);
1815                 }
1816
1817                 if (data_buf)
1818                         data_buf += data_len;
1819                 if (oob_buf)
1820                         oob_buf += oob_len + ecc->bytes;
1821         }
1822
1823         if (flash_op_err)
1824                 return -EIO;
1825
1826         if (!uncorrectable_cws)
1827                 return max_bitflips;
1828
1829         return check_for_erased_page(host, data_buf_start, oob_buf_start,
1830                                      uncorrectable_cws, page,
1831                                      max_bitflips);
1832 }
1833
1834 /*
1835  * helper to perform the actual page read operation, used by ecc->read_page(),
1836  * ecc->read_oob()
1837  */
1838 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1839                          u8 *oob_buf, int page)
1840 {
1841         struct nand_chip *chip = &host->chip;
1842         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1843         struct nand_ecc_ctrl *ecc = &chip->ecc;
1844         u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1845         int i, ret;
1846
1847         config_nand_page_read(nandc);
1848
1849         /* queue cmd descs for each codeword */
1850         for (i = 0; i < ecc->steps; i++) {
1851                 int data_size, oob_size;
1852
1853                 if (i == (ecc->steps - 1)) {
1854                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1855                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1856                                    host->spare_bytes;
1857                 } else {
1858                         data_size = host->cw_data;
1859                         oob_size = host->ecc_bytes_hw + host->spare_bytes;
1860                 }
1861
1862                 if (nandc->props->is_bam) {
1863                         if (data_buf && oob_buf) {
1864                                 nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1865                                 nandc_set_read_loc(nandc, 1, data_size,
1866                                                    oob_size, 1);
1867                         } else if (data_buf) {
1868                                 nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1869                         } else {
1870                                 nandc_set_read_loc(nandc, 0, data_size,
1871                                                    oob_size, 1);
1872                         }
1873                 }
1874
1875                 config_nand_cw_read(nandc, true);
1876
1877                 if (data_buf)
1878                         read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1879                                       data_size, 0);
1880
1881                 /*
1882                  * when ecc is enabled, the controller doesn't read the real
1883                  * or dummy bad block markers in each chunk. To maintain a
1884                  * consistent layout across RAW and ECC reads, we just
1885                  * leave the real/dummy BBM offsets empty (i.e, filled with
1886                  * 0xffs)
1887                  */
1888                 if (oob_buf) {
1889                         int j;
1890
1891                         for (j = 0; j < host->bbm_size; j++)
1892                                 *oob_buf++ = 0xff;
1893
1894                         read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1895                                       oob_buf, oob_size, 0);
1896                 }
1897
1898                 if (data_buf)
1899                         data_buf += data_size;
1900                 if (oob_buf)
1901                         oob_buf += oob_size;
1902         }
1903
1904         ret = submit_descs(nandc);
1905         free_descs(nandc);
1906
1907         if (ret) {
1908                 dev_err(nandc->dev, "failure to read page/oob\n");
1909                 return ret;
1910         }
1911
1912         return parse_read_errors(host, data_buf_start, oob_buf_start, page);
1913 }
1914
1915 /*
1916  * a helper that copies the last step/codeword of a page (containing free oob)
1917  * into our local buffer
1918  */
1919 static int copy_last_cw(struct qcom_nand_host *host, int page)
1920 {
1921         struct nand_chip *chip = &host->chip;
1922         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1923         struct nand_ecc_ctrl *ecc = &chip->ecc;
1924         int size;
1925         int ret;
1926
1927         clear_read_regs(nandc);
1928
1929         size = host->use_ecc ? host->cw_data : host->cw_size;
1930
1931         /* prepare a clean read buffer */
1932         memset(nandc->data_buffer, 0xff, size);
1933
1934         set_address(host, host->cw_size * (ecc->steps - 1), page);
1935         update_rw_regs(host, 1, true);
1936
1937         config_nand_single_cw_page_read(nandc, host->use_ecc);
1938
1939         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1940
1941         ret = submit_descs(nandc);
1942         if (ret)
1943                 dev_err(nandc->dev, "failed to copy last codeword\n");
1944
1945         free_descs(nandc);
1946
1947         return ret;
1948 }
1949
1950 /* implements ecc->read_page() */
1951 static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf,
1952                                 int oob_required, int page)
1953 {
1954         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1955         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1956         u8 *data_buf, *oob_buf = NULL;
1957
1958         nand_read_page_op(chip, page, 0, NULL, 0);
1959         data_buf = buf;
1960         oob_buf = oob_required ? chip->oob_poi : NULL;
1961
1962         clear_bam_transaction(nandc);
1963
1964         return read_page_ecc(host, data_buf, oob_buf, page);
1965 }
1966
1967 /* implements ecc->read_page_raw() */
1968 static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
1969                                     int oob_required, int page)
1970 {
1971         struct mtd_info *mtd = nand_to_mtd(chip);
1972         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1973         struct nand_ecc_ctrl *ecc = &chip->ecc;
1974         int cw, ret;
1975         u8 *data_buf = buf, *oob_buf = chip->oob_poi;
1976
1977         for (cw = 0; cw < ecc->steps; cw++) {
1978                 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
1979                                              page, cw);
1980                 if (ret)
1981                         return ret;
1982
1983                 data_buf += host->cw_data;
1984                 oob_buf += ecc->bytes;
1985         }
1986
1987         return 0;
1988 }
1989
1990 /* implements ecc->read_oob() */
1991 static int qcom_nandc_read_oob(struct nand_chip *chip, int page)
1992 {
1993         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1994         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1995         struct nand_ecc_ctrl *ecc = &chip->ecc;
1996
1997         clear_read_regs(nandc);
1998         clear_bam_transaction(nandc);
1999
2000         host->use_ecc = true;
2001         set_address(host, 0, page);
2002         update_rw_regs(host, ecc->steps, true);
2003
2004         return read_page_ecc(host, NULL, chip->oob_poi, page);
2005 }
2006
2007 /* implements ecc->write_page() */
2008 static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf,
2009                                  int oob_required, int page)
2010 {
2011         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2012         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2013         struct nand_ecc_ctrl *ecc = &chip->ecc;
2014         u8 *data_buf, *oob_buf;
2015         int i, ret;
2016
2017         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2018
2019         clear_read_regs(nandc);
2020         clear_bam_transaction(nandc);
2021
2022         data_buf = (u8 *)buf;
2023         oob_buf = chip->oob_poi;
2024
2025         host->use_ecc = true;
2026         update_rw_regs(host, ecc->steps, false);
2027         config_nand_page_write(nandc);
2028
2029         for (i = 0; i < ecc->steps; i++) {
2030                 int data_size, oob_size;
2031
2032                 if (i == (ecc->steps - 1)) {
2033                         data_size = ecc->size - ((ecc->steps - 1) << 2);
2034                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
2035                                    host->spare_bytes;
2036                 } else {
2037                         data_size = host->cw_data;
2038                         oob_size = ecc->bytes;
2039                 }
2040
2041
2042                 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
2043                                i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
2044
2045                 /*
2046                  * when ECC is enabled, we don't really need to write anything
2047                  * to oob for the first n - 1 codewords since these oob regions
2048                  * just contain ECC bytes that's written by the controller
2049                  * itself. For the last codeword, we skip the bbm positions and
2050                  * write to the free oob area.
2051                  */
2052                 if (i == (ecc->steps - 1)) {
2053                         oob_buf += host->bbm_size;
2054
2055                         write_data_dma(nandc, FLASH_BUF_ACC + data_size,
2056                                        oob_buf, oob_size, 0);
2057                 }
2058
2059                 config_nand_cw_write(nandc);
2060
2061                 data_buf += data_size;
2062                 oob_buf += oob_size;
2063         }
2064
2065         ret = submit_descs(nandc);
2066         if (ret)
2067                 dev_err(nandc->dev, "failure to write page\n");
2068
2069         free_descs(nandc);
2070
2071         if (!ret)
2072                 ret = nand_prog_page_end_op(chip);
2073
2074         return ret;
2075 }
2076
2077 /* implements ecc->write_page_raw() */
2078 static int qcom_nandc_write_page_raw(struct nand_chip *chip,
2079                                      const uint8_t *buf, int oob_required,
2080                                      int page)
2081 {
2082         struct mtd_info *mtd = nand_to_mtd(chip);
2083         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2084         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2085         struct nand_ecc_ctrl *ecc = &chip->ecc;
2086         u8 *data_buf, *oob_buf;
2087         int i, ret;
2088
2089         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2090         clear_read_regs(nandc);
2091         clear_bam_transaction(nandc);
2092
2093         data_buf = (u8 *)buf;
2094         oob_buf = chip->oob_poi;
2095
2096         host->use_ecc = false;
2097         update_rw_regs(host, ecc->steps, false);
2098         config_nand_page_write(nandc);
2099
2100         for (i = 0; i < ecc->steps; i++) {
2101                 int data_size1, data_size2, oob_size1, oob_size2;
2102                 int reg_off = FLASH_BUF_ACC;
2103
2104                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
2105                 oob_size1 = host->bbm_size;
2106
2107                 if (i == (ecc->steps - 1)) {
2108                         data_size2 = ecc->size - data_size1 -
2109                                      ((ecc->steps - 1) << 2);
2110                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
2111                                     host->spare_bytes;
2112                 } else {
2113                         data_size2 = host->cw_data - data_size1;
2114                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
2115                 }
2116
2117                 write_data_dma(nandc, reg_off, data_buf, data_size1,
2118                                NAND_BAM_NO_EOT);
2119                 reg_off += data_size1;
2120                 data_buf += data_size1;
2121
2122                 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
2123                                NAND_BAM_NO_EOT);
2124                 reg_off += oob_size1;
2125                 oob_buf += oob_size1;
2126
2127                 write_data_dma(nandc, reg_off, data_buf, data_size2,
2128                                NAND_BAM_NO_EOT);
2129                 reg_off += data_size2;
2130                 data_buf += data_size2;
2131
2132                 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
2133                 oob_buf += oob_size2;
2134
2135                 config_nand_cw_write(nandc);
2136         }
2137
2138         ret = submit_descs(nandc);
2139         if (ret)
2140                 dev_err(nandc->dev, "failure to write raw page\n");
2141
2142         free_descs(nandc);
2143
2144         if (!ret)
2145                 ret = nand_prog_page_end_op(chip);
2146
2147         return ret;
2148 }
2149
2150 /*
2151  * implements ecc->write_oob()
2152  *
2153  * the NAND controller cannot write only data or only OOB within a codeword
2154  * since ECC is calculated for the combined codeword. So update the OOB from
2155  * chip->oob_poi, and pad the data area with OxFF before writing.
2156  */
2157 static int qcom_nandc_write_oob(struct nand_chip *chip, int page)
2158 {
2159         struct mtd_info *mtd = nand_to_mtd(chip);
2160         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2161         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2162         struct nand_ecc_ctrl *ecc = &chip->ecc;
2163         u8 *oob = chip->oob_poi;
2164         int data_size, oob_size;
2165         int ret;
2166
2167         host->use_ecc = true;
2168         clear_bam_transaction(nandc);
2169
2170         /* calculate the data and oob size for the last codeword/step */
2171         data_size = ecc->size - ((ecc->steps - 1) << 2);
2172         oob_size = mtd->oobavail;
2173
2174         memset(nandc->data_buffer, 0xff, host->cw_data);
2175         /* override new oob content to last codeword */
2176         mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2177                                     0, mtd->oobavail);
2178
2179         set_address(host, host->cw_size * (ecc->steps - 1), page);
2180         update_rw_regs(host, 1, false);
2181
2182         config_nand_page_write(nandc);
2183         write_data_dma(nandc, FLASH_BUF_ACC,
2184                        nandc->data_buffer, data_size + oob_size, 0);
2185         config_nand_cw_write(nandc);
2186
2187         ret = submit_descs(nandc);
2188
2189         free_descs(nandc);
2190
2191         if (ret) {
2192                 dev_err(nandc->dev, "failure to write oob\n");
2193                 return -EIO;
2194         }
2195
2196         return nand_prog_page_end_op(chip);
2197 }
2198
2199 static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs)
2200 {
2201         struct mtd_info *mtd = nand_to_mtd(chip);
2202         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2203         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2204         struct nand_ecc_ctrl *ecc = &chip->ecc;
2205         int page, ret, bbpos, bad = 0;
2206
2207         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2208
2209         /*
2210          * configure registers for a raw sub page read, the address is set to
2211          * the beginning of the last codeword, we don't care about reading ecc
2212          * portion of oob. we just want the first few bytes from this codeword
2213          * that contains the BBM
2214          */
2215         host->use_ecc = false;
2216
2217         clear_bam_transaction(nandc);
2218         ret = copy_last_cw(host, page);
2219         if (ret)
2220                 goto err;
2221
2222         if (check_flash_errors(host, 1)) {
2223                 dev_warn(nandc->dev, "error when trying to read BBM\n");
2224                 goto err;
2225         }
2226
2227         bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2228
2229         bad = nandc->data_buffer[bbpos] != 0xff;
2230
2231         if (chip->options & NAND_BUSWIDTH_16)
2232                 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2233 err:
2234         return bad;
2235 }
2236
2237 static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs)
2238 {
2239         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2240         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2241         struct nand_ecc_ctrl *ecc = &chip->ecc;
2242         int page, ret;
2243
2244         clear_read_regs(nandc);
2245         clear_bam_transaction(nandc);
2246
2247         /*
2248          * to mark the BBM as bad, we flash the entire last codeword with 0s.
2249          * we don't care about the rest of the content in the codeword since
2250          * we aren't going to use this block again
2251          */
2252         memset(nandc->data_buffer, 0x00, host->cw_size);
2253
2254         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2255
2256         /* prepare write */
2257         host->use_ecc = false;
2258         set_address(host, host->cw_size * (ecc->steps - 1), page);
2259         update_rw_regs(host, 1, false);
2260
2261         config_nand_page_write(nandc);
2262         write_data_dma(nandc, FLASH_BUF_ACC,
2263                        nandc->data_buffer, host->cw_size, 0);
2264         config_nand_cw_write(nandc);
2265
2266         ret = submit_descs(nandc);
2267
2268         free_descs(nandc);
2269
2270         if (ret) {
2271                 dev_err(nandc->dev, "failure to update BBM\n");
2272                 return -EIO;
2273         }
2274
2275         return nand_prog_page_end_op(chip);
2276 }
2277
2278 /*
2279  * the three functions below implement chip->legacy.read_byte(),
2280  * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these
2281  * aren't used for reading/writing page data, they are used for smaller data
2282  * like reading id, status etc
2283  */
2284 static uint8_t qcom_nandc_read_byte(struct nand_chip *chip)
2285 {
2286         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2287         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2288         u8 *buf = nandc->data_buffer;
2289         u8 ret = 0x0;
2290
2291         if (host->last_command == NAND_CMD_STATUS) {
2292                 ret = host->status;
2293
2294                 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2295
2296                 return ret;
2297         }
2298
2299         if (nandc->buf_start < nandc->buf_count)
2300                 ret = buf[nandc->buf_start++];
2301
2302         return ret;
2303 }
2304
2305 static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len)
2306 {
2307         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2308         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2309
2310         memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2311         nandc->buf_start += real_len;
2312 }
2313
2314 static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf,
2315                                  int len)
2316 {
2317         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2318         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2319
2320         memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2321
2322         nandc->buf_start += real_len;
2323 }
2324
2325 /* we support only one external chip for now */
2326 static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr)
2327 {
2328         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2329
2330         if (chipnr <= 0)
2331                 return;
2332
2333         dev_warn(nandc->dev, "invalid chip select\n");
2334 }
2335
2336 /*
2337  * NAND controller page layout info
2338  *
2339  * Layout with ECC enabled:
2340  *
2341  * |----------------------|  |---------------------------------|
2342  * |           xx.......yy|  |             *********xx.......yy|
2343  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
2344  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
2345  * |           xx.......yy|  |             *********xx.......yy|
2346  * |----------------------|  |---------------------------------|
2347  *     codeword 1,2..n-1                  codeword n
2348  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
2349  *
2350  * n = Number of codewords in the page
2351  * . = ECC bytes
2352  * * = Spare/free bytes
2353  * x = Unused byte(s)
2354  * y = Reserved byte(s)
2355  *
2356  * 2K page: n = 4, spare = 16 bytes
2357  * 4K page: n = 8, spare = 32 bytes
2358  * 8K page: n = 16, spare = 64 bytes
2359  *
2360  * the qcom nand controller operates at a sub page/codeword level. each
2361  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2362  * the number of ECC bytes vary based on the ECC strength and the bus width.
2363  *
2364  * the first n - 1 codewords contains 516 bytes of user data, the remaining
2365  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2366  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2367  *
2368  * When we access a page with ECC enabled, the reserved bytes(s) are not
2369  * accessible at all. When reading, we fill up these unreadable positions
2370  * with 0xffs. When writing, the controller skips writing the inaccessible
2371  * bytes.
2372  *
2373  * Layout with ECC disabled:
2374  *
2375  * |------------------------------|  |---------------------------------------|
2376  * |         yy          xx.......|  |         bb          *********xx.......|
2377  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
2378  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
2379  * |         yy          xx.......|  |         bb          *********xx.......|
2380  * |------------------------------|  |---------------------------------------|
2381  *         codeword 1,2..n-1                        codeword n
2382  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
2383  *
2384  * n = Number of codewords in the page
2385  * . = ECC bytes
2386  * * = Spare/free bytes
2387  * x = Unused byte(s)
2388  * y = Dummy Bad Bock byte(s)
2389  * b = Real Bad Block byte(s)
2390  * size1/size2 = function of codeword size and 'n'
2391  *
2392  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2393  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2394  * Block Markers. In the last codeword, this position contains the real BBM
2395  *
2396  * In order to have a consistent layout between RAW and ECC modes, we assume
2397  * the following OOB layout arrangement:
2398  *
2399  * |-----------|  |--------------------|
2400  * |yyxx.......|  |bb*********xx.......|
2401  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
2402  * |yyxx.......|  |bb*********xx.......|
2403  * |yyxx.......|  |bb*********xx.......|
2404  * |-----------|  |--------------------|
2405  *  first n - 1       nth OOB region
2406  *  OOB regions
2407  *
2408  * n = Number of codewords in the page
2409  * . = ECC bytes
2410  * * = FREE OOB bytes
2411  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2412  * x = Unused byte(s)
2413  * b = Real bad block byte(s) (inaccessible when ECC enabled)
2414  *
2415  * This layout is read as is when ECC is disabled. When ECC is enabled, the
2416  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2417  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2418  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2419  * the sum of the three).
2420  */
2421 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2422                                    struct mtd_oob_region *oobregion)
2423 {
2424         struct nand_chip *chip = mtd_to_nand(mtd);
2425         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2426         struct nand_ecc_ctrl *ecc = &chip->ecc;
2427
2428         if (section > 1)
2429                 return -ERANGE;
2430
2431         if (!section) {
2432                 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2433                                     host->bbm_size;
2434                 oobregion->offset = 0;
2435         } else {
2436                 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2437                 oobregion->offset = mtd->oobsize - oobregion->length;
2438         }
2439
2440         return 0;
2441 }
2442
2443 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2444                                      struct mtd_oob_region *oobregion)
2445 {
2446         struct nand_chip *chip = mtd_to_nand(mtd);
2447         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2448         struct nand_ecc_ctrl *ecc = &chip->ecc;
2449
2450         if (section)
2451                 return -ERANGE;
2452
2453         oobregion->length = ecc->steps * 4;
2454         oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2455
2456         return 0;
2457 }
2458
2459 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2460         .ecc = qcom_nand_ooblayout_ecc,
2461         .free = qcom_nand_ooblayout_free,
2462 };
2463
2464 static int
2465 qcom_nandc_calc_ecc_bytes(int step_size, int strength)
2466 {
2467         return strength == 4 ? 12 : 16;
2468 }
2469 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
2470                      NANDC_STEP_SIZE, 4, 8);
2471
2472 static int qcom_nand_attach_chip(struct nand_chip *chip)
2473 {
2474         struct mtd_info *mtd = nand_to_mtd(chip);
2475         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2476         struct nand_ecc_ctrl *ecc = &chip->ecc;
2477         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2478         int cwperpage, bad_block_byte, ret;
2479         bool wide_bus;
2480         int ecc_mode = 1;
2481
2482         /* controller only supports 512 bytes data steps */
2483         ecc->size = NANDC_STEP_SIZE;
2484         wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2485         cwperpage = mtd->writesize / NANDC_STEP_SIZE;
2486
2487         /*
2488          * Each CW has 4 available OOB bytes which will be protected with ECC
2489          * so remaining bytes can be used for ECC.
2490          */
2491         ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps,
2492                                    mtd->oobsize - (cwperpage * 4));
2493         if (ret) {
2494                 dev_err(nandc->dev, "No valid ECC settings possible\n");
2495                 return ret;
2496         }
2497
2498         if (ecc->strength >= 8) {
2499                 /* 8 bit ECC defaults to BCH ECC on all platforms */
2500                 host->bch_enabled = true;
2501                 ecc_mode = 1;
2502
2503                 if (wide_bus) {
2504                         host->ecc_bytes_hw = 14;
2505                         host->spare_bytes = 0;
2506                         host->bbm_size = 2;
2507                 } else {
2508                         host->ecc_bytes_hw = 13;
2509                         host->spare_bytes = 2;
2510                         host->bbm_size = 1;
2511                 }
2512         } else {
2513                 /*
2514                  * if the controller supports BCH for 4 bit ECC, the controller
2515                  * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2516                  * always 10 bytes
2517                  */
2518                 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2519                         /* BCH */
2520                         host->bch_enabled = true;
2521                         ecc_mode = 0;
2522
2523                         if (wide_bus) {
2524                                 host->ecc_bytes_hw = 8;
2525                                 host->spare_bytes = 2;
2526                                 host->bbm_size = 2;
2527                         } else {
2528                                 host->ecc_bytes_hw = 7;
2529                                 host->spare_bytes = 4;
2530                                 host->bbm_size = 1;
2531                         }
2532                 } else {
2533                         /* RS */
2534                         host->ecc_bytes_hw = 10;
2535
2536                         if (wide_bus) {
2537                                 host->spare_bytes = 0;
2538                                 host->bbm_size = 2;
2539                         } else {
2540                                 host->spare_bytes = 1;
2541                                 host->bbm_size = 1;
2542                         }
2543                 }
2544         }
2545
2546         /*
2547          * we consider ecc->bytes as the sum of all the non-data content in a
2548          * step. It gives us a clean representation of the oob area (even if
2549          * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2550          * ECC and 12 bytes for 4 bit ECC
2551          */
2552         ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2553
2554         ecc->read_page          = qcom_nandc_read_page;
2555         ecc->read_page_raw      = qcom_nandc_read_page_raw;
2556         ecc->read_oob           = qcom_nandc_read_oob;
2557         ecc->write_page         = qcom_nandc_write_page;
2558         ecc->write_page_raw     = qcom_nandc_write_page_raw;
2559         ecc->write_oob          = qcom_nandc_write_oob;
2560
2561         ecc->mode = NAND_ECC_HW;
2562
2563         mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2564
2565         nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2566                                      cwperpage);
2567
2568         /*
2569          * DATA_UD_BYTES varies based on whether the read/write command protects
2570          * spare data with ECC too. We protect spare data by default, so we set
2571          * it to main + spare data, which are 512 and 4 bytes respectively.
2572          */
2573         host->cw_data = 516;
2574
2575         /*
2576          * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2577          * for 8 bit ECC
2578          */
2579         host->cw_size = host->cw_data + ecc->bytes;
2580         bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2581
2582         host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2583                                 | host->cw_data << UD_SIZE_BYTES
2584                                 | 0 << DISABLE_STATUS_AFTER_WRITE
2585                                 | 5 << NUM_ADDR_CYCLES
2586                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2587                                 | 0 << STATUS_BFR_READ
2588                                 | 1 << SET_RD_MODE_AFTER_STATUS
2589                                 | host->spare_bytes << SPARE_SIZE_BYTES;
2590
2591         host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2592                                 | 0 <<  CS_ACTIVE_BSY
2593                                 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2594                                 | 0 << BAD_BLOCK_IN_SPARE_AREA
2595                                 | 2 << WR_RD_BSY_GAP
2596                                 | wide_bus << WIDE_FLASH
2597                                 | host->bch_enabled << ENABLE_BCH_ECC;
2598
2599         host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2600                                 | host->cw_size << UD_SIZE_BYTES
2601                                 | 5 << NUM_ADDR_CYCLES
2602                                 | 0 << SPARE_SIZE_BYTES;
2603
2604         host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2605                                 | 0 << CS_ACTIVE_BSY
2606                                 | 17 << BAD_BLOCK_BYTE_NUM
2607                                 | 1 << BAD_BLOCK_IN_SPARE_AREA
2608                                 | 2 << WR_RD_BSY_GAP
2609                                 | wide_bus << WIDE_FLASH
2610                                 | 1 << DEV0_CFG1_ECC_DISABLE;
2611
2612         host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2613                                 | 0 << ECC_SW_RESET
2614                                 | host->cw_data << ECC_NUM_DATA_BYTES
2615                                 | 1 << ECC_FORCE_CLK_OPEN
2616                                 | ecc_mode << ECC_MODE
2617                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2618
2619         host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2620
2621         host->clrflashstatus = FS_READY_BSY_N;
2622         host->clrreadstatus = 0xc0;
2623         nandc->regs->erased_cw_detect_cfg_clr =
2624                 cpu_to_le32(CLR_ERASED_PAGE_DET);
2625         nandc->regs->erased_cw_detect_cfg_set =
2626                 cpu_to_le32(SET_ERASED_PAGE_DET);
2627
2628         dev_dbg(nandc->dev,
2629                 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2630                 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2631                 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2632                 cwperpage);
2633
2634         return 0;
2635 }
2636
2637 static const struct nand_controller_ops qcom_nandc_ops = {
2638         .attach_chip = qcom_nand_attach_chip,
2639 };
2640
2641 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2642 {
2643         int ret;
2644
2645         ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2646         if (ret) {
2647                 dev_err(nandc->dev, "failed to set DMA mask\n");
2648                 return ret;
2649         }
2650
2651         /*
2652          * we use the internal buffer for reading ONFI params, reading small
2653          * data like ID and status, and preforming read-copy-write operations
2654          * when writing to a codeword partially. 532 is the maximum possible
2655          * size of a codeword for our nand controller
2656          */
2657         nandc->buf_size = 532;
2658
2659         nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2660                                         GFP_KERNEL);
2661         if (!nandc->data_buffer)
2662                 return -ENOMEM;
2663
2664         nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2665                                         GFP_KERNEL);
2666         if (!nandc->regs)
2667                 return -ENOMEM;
2668
2669         nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2670                                 MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2671                                 GFP_KERNEL);
2672         if (!nandc->reg_read_buf)
2673                 return -ENOMEM;
2674
2675         if (nandc->props->is_bam) {
2676                 nandc->reg_read_dma =
2677                         dma_map_single(nandc->dev, nandc->reg_read_buf,
2678                                        MAX_REG_RD *
2679                                        sizeof(*nandc->reg_read_buf),
2680                                        DMA_FROM_DEVICE);
2681                 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2682                         dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2683                         return -EIO;
2684                 }
2685
2686                 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
2687                 if (!nandc->tx_chan) {
2688                         dev_err(nandc->dev, "failed to request tx channel\n");
2689                         return -ENODEV;
2690                 }
2691
2692                 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
2693                 if (!nandc->rx_chan) {
2694                         dev_err(nandc->dev, "failed to request rx channel\n");
2695                         return -ENODEV;
2696                 }
2697
2698                 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
2699                 if (!nandc->cmd_chan) {
2700                         dev_err(nandc->dev, "failed to request cmd channel\n");
2701                         return -ENODEV;
2702                 }
2703
2704                 /*
2705                  * Initially allocate BAM transaction to read ONFI param page.
2706                  * After detecting all the devices, this BAM transaction will
2707                  * be freed and the next BAM tranasction will be allocated with
2708                  * maximum codeword size
2709                  */
2710                 nandc->max_cwperpage = 1;
2711                 nandc->bam_txn = alloc_bam_transaction(nandc);
2712                 if (!nandc->bam_txn) {
2713                         dev_err(nandc->dev,
2714                                 "failed to allocate bam transaction\n");
2715                         return -ENOMEM;
2716                 }
2717         } else {
2718                 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
2719                 if (!nandc->chan) {
2720                         dev_err(nandc->dev,
2721                                 "failed to request slave channel\n");
2722                         return -ENODEV;
2723                 }
2724         }
2725
2726         INIT_LIST_HEAD(&nandc->desc_list);
2727         INIT_LIST_HEAD(&nandc->host_list);
2728
2729         nand_controller_init(&nandc->controller);
2730         nandc->controller.ops = &qcom_nandc_ops;
2731
2732         return 0;
2733 }
2734
2735 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2736 {
2737         if (nandc->props->is_bam) {
2738                 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2739                         dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2740                                          MAX_REG_RD *
2741                                          sizeof(*nandc->reg_read_buf),
2742                                          DMA_FROM_DEVICE);
2743
2744                 if (nandc->tx_chan)
2745                         dma_release_channel(nandc->tx_chan);
2746
2747                 if (nandc->rx_chan)
2748                         dma_release_channel(nandc->rx_chan);
2749
2750                 if (nandc->cmd_chan)
2751                         dma_release_channel(nandc->cmd_chan);
2752         } else {
2753                 if (nandc->chan)
2754                         dma_release_channel(nandc->chan);
2755         }
2756 }
2757
2758 /* one time setup of a few nand controller registers */
2759 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2760 {
2761         u32 nand_ctrl;
2762
2763         /* kill onenand */
2764         nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2765         nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2766                     NAND_DEV_CMD_VLD_VAL);
2767
2768         /* enable ADM or BAM DMA */
2769         if (nandc->props->is_bam) {
2770                 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2771                 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2772         } else {
2773                 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2774         }
2775
2776         /* save the original values of these registers */
2777         nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2778         nandc->vld = NAND_DEV_CMD_VLD_VAL;
2779
2780         return 0;
2781 }
2782
2783 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2784                                             struct qcom_nand_host *host,
2785                                             struct device_node *dn)
2786 {
2787         struct nand_chip *chip = &host->chip;
2788         struct mtd_info *mtd = nand_to_mtd(chip);
2789         struct device *dev = nandc->dev;
2790         int ret;
2791
2792         ret = of_property_read_u32(dn, "reg", &host->cs);
2793         if (ret) {
2794                 dev_err(dev, "can't get chip-select\n");
2795                 return -ENXIO;
2796         }
2797
2798         nand_set_flash_node(chip, dn);
2799         mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2800         if (!mtd->name)
2801                 return -ENOMEM;
2802
2803         mtd->owner = THIS_MODULE;
2804         mtd->dev.parent = dev;
2805
2806         chip->legacy.cmdfunc    = qcom_nandc_command;
2807         chip->legacy.select_chip        = qcom_nandc_select_chip;
2808         chip->legacy.read_byte  = qcom_nandc_read_byte;
2809         chip->legacy.read_buf   = qcom_nandc_read_buf;
2810         chip->legacy.write_buf  = qcom_nandc_write_buf;
2811         chip->legacy.set_features       = nand_get_set_features_notsupp;
2812         chip->legacy.get_features       = nand_get_set_features_notsupp;
2813
2814         /*
2815          * the bad block marker is readable only when we read the last codeword
2816          * of a page with ECC disabled. currently, the nand_base and nand_bbt
2817          * helpers don't allow us to read BB from a nand chip with ECC
2818          * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2819          * and block_markbad helpers until we permanently switch to using
2820          * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2821          */
2822         chip->legacy.block_bad          = qcom_nandc_block_bad;
2823         chip->legacy.block_markbad      = qcom_nandc_block_markbad;
2824
2825         chip->controller = &nandc->controller;
2826         chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2827                          NAND_SKIP_BBTSCAN;
2828
2829         /* set up initial status value */
2830         host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2831
2832         ret = nand_scan(chip, 1);
2833         if (ret)
2834                 return ret;
2835
2836         if (nandc->props->is_bam) {
2837                 free_bam_transaction(nandc);
2838                 nandc->bam_txn = alloc_bam_transaction(nandc);
2839                 if (!nandc->bam_txn) {
2840                         dev_err(nandc->dev,
2841                                 "failed to allocate bam transaction\n");
2842                         return -ENOMEM;
2843                 }
2844         }
2845
2846         ret = mtd_device_register(mtd, NULL, 0);
2847         if (ret)
2848                 nand_cleanup(chip);
2849
2850         return ret;
2851 }
2852
2853 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2854 {
2855         struct device *dev = nandc->dev;
2856         struct device_node *dn = dev->of_node, *child;
2857         struct qcom_nand_host *host;
2858         int ret;
2859
2860         for_each_available_child_of_node(dn, child) {
2861                 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2862                 if (!host) {
2863                         of_node_put(child);
2864                         return -ENOMEM;
2865                 }
2866
2867                 ret = qcom_nand_host_init_and_register(nandc, host, child);
2868                 if (ret) {
2869                         devm_kfree(dev, host);
2870                         continue;
2871                 }
2872
2873                 list_add_tail(&host->node, &nandc->host_list);
2874         }
2875
2876         if (list_empty(&nandc->host_list))
2877                 return -ENODEV;
2878
2879         return 0;
2880 }
2881
2882 /* parse custom DT properties here */
2883 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2884 {
2885         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2886         struct device_node *np = nandc->dev->of_node;
2887         int ret;
2888
2889         if (!nandc->props->is_bam) {
2890                 ret = of_property_read_u32(np, "qcom,cmd-crci",
2891                                            &nandc->cmd_crci);
2892                 if (ret) {
2893                         dev_err(nandc->dev, "command CRCI unspecified\n");
2894                         return ret;
2895                 }
2896
2897                 ret = of_property_read_u32(np, "qcom,data-crci",
2898                                            &nandc->data_crci);
2899                 if (ret) {
2900                         dev_err(nandc->dev, "data CRCI unspecified\n");
2901                         return ret;
2902                 }
2903         }
2904
2905         return 0;
2906 }
2907
2908 static int qcom_nandc_probe(struct platform_device *pdev)
2909 {
2910         struct qcom_nand_controller *nandc;
2911         const void *dev_data;
2912         struct device *dev = &pdev->dev;
2913         struct resource *res;
2914         int ret;
2915
2916         nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2917         if (!nandc)
2918                 return -ENOMEM;
2919
2920         platform_set_drvdata(pdev, nandc);
2921         nandc->dev = dev;
2922
2923         dev_data = of_device_get_match_data(dev);
2924         if (!dev_data) {
2925                 dev_err(&pdev->dev, "failed to get device data\n");
2926                 return -ENODEV;
2927         }
2928
2929         nandc->props = dev_data;
2930
2931         nandc->core_clk = devm_clk_get(dev, "core");
2932         if (IS_ERR(nandc->core_clk))
2933                 return PTR_ERR(nandc->core_clk);
2934
2935         nandc->aon_clk = devm_clk_get(dev, "aon");
2936         if (IS_ERR(nandc->aon_clk))
2937                 return PTR_ERR(nandc->aon_clk);
2938
2939         ret = qcom_nandc_parse_dt(pdev);
2940         if (ret)
2941                 return ret;
2942
2943         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2944         nandc->base = devm_ioremap_resource(dev, res);
2945         if (IS_ERR(nandc->base))
2946                 return PTR_ERR(nandc->base);
2947
2948         nandc->base_phys = res->start;
2949         nandc->base_dma = dma_map_resource(dev, res->start,
2950                                            resource_size(res),
2951                                            DMA_BIDIRECTIONAL, 0);
2952         if (!nandc->base_dma)
2953                 return -ENXIO;
2954
2955         ret = qcom_nandc_alloc(nandc);
2956         if (ret)
2957                 goto err_nandc_alloc;
2958
2959         ret = clk_prepare_enable(nandc->core_clk);
2960         if (ret)
2961                 goto err_core_clk;
2962
2963         ret = clk_prepare_enable(nandc->aon_clk);
2964         if (ret)
2965                 goto err_aon_clk;
2966
2967         ret = qcom_nandc_setup(nandc);
2968         if (ret)
2969                 goto err_setup;
2970
2971         ret = qcom_probe_nand_devices(nandc);
2972         if (ret)
2973                 goto err_setup;
2974
2975         return 0;
2976
2977 err_setup:
2978         clk_disable_unprepare(nandc->aon_clk);
2979 err_aon_clk:
2980         clk_disable_unprepare(nandc->core_clk);
2981 err_core_clk:
2982         qcom_nandc_unalloc(nandc);
2983 err_nandc_alloc:
2984         dma_unmap_resource(dev, res->start, resource_size(res),
2985                            DMA_BIDIRECTIONAL, 0);
2986
2987         return ret;
2988 }
2989
2990 static int qcom_nandc_remove(struct platform_device *pdev)
2991 {
2992         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2993         struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2994         struct qcom_nand_host *host;
2995
2996         list_for_each_entry(host, &nandc->host_list, node)
2997                 nand_release(&host->chip);
2998
2999
3000         qcom_nandc_unalloc(nandc);
3001
3002         clk_disable_unprepare(nandc->aon_clk);
3003         clk_disable_unprepare(nandc->core_clk);
3004
3005         dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
3006                            DMA_BIDIRECTIONAL, 0);
3007
3008         return 0;
3009 }
3010
3011 static const struct qcom_nandc_props ipq806x_nandc_props = {
3012         .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
3013         .is_bam = false,
3014         .dev_cmd_reg_start = 0x0,
3015 };
3016
3017 static const struct qcom_nandc_props ipq4019_nandc_props = {
3018         .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3019         .is_bam = true,
3020         .dev_cmd_reg_start = 0x0,
3021 };
3022
3023 static const struct qcom_nandc_props ipq8074_nandc_props = {
3024         .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3025         .is_bam = true,
3026         .dev_cmd_reg_start = 0x7000,
3027 };
3028
3029 /*
3030  * data will hold a struct pointer containing more differences once we support
3031  * more controller variants
3032  */
3033 static const struct of_device_id qcom_nandc_of_match[] = {
3034         {
3035                 .compatible = "qcom,ipq806x-nand",
3036                 .data = &ipq806x_nandc_props,
3037         },
3038         {
3039                 .compatible = "qcom,ipq4019-nand",
3040                 .data = &ipq4019_nandc_props,
3041         },
3042         {
3043                 .compatible = "qcom,ipq8074-nand",
3044                 .data = &ipq8074_nandc_props,
3045         },
3046         {}
3047 };
3048 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
3049
3050 static struct platform_driver qcom_nandc_driver = {
3051         .driver = {
3052                 .name = "qcom-nandc",
3053                 .of_match_table = qcom_nandc_of_match,
3054         },
3055         .probe   = qcom_nandc_probe,
3056         .remove  = qcom_nandc_remove,
3057 };
3058 module_platform_driver(qcom_nandc_driver);
3059
3060 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3061 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3062 MODULE_LICENSE("GPL v2");