1 // SPDX-License-Identifier: GPL-2.0
3 * Marvell NAND flash controller driver
5 * Copyright (C) 2017 Marvell
6 * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
9 * This NAND controller driver handles two versions of the hardware,
10 * one is called NFCv1 and is available on PXA SoCs and the other is
11 * called NFCv2 and is available on Armada SoCs.
13 * The main visible difference is that NFCv1 only has Hamming ECC
14 * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15 * is not used with NFCv2.
17 * The ECC layouts are depicted in details in Marvell AN-379, but here
18 * is a brief description.
20 * When using Hamming, the data is split in 512B chunks (either 1, 2
21 * or 4) and each chunk will have its own ECC "digest" of 6B at the
22 * beginning of the OOB area and eventually the remaining free OOB
23 * bytes (also called "spare" bytes in the driver). This engine
24 * corrects up to 1 bit per chunk and detects reliably an error if
25 * there are at most 2 bitflips. Here is the page layout used by the
26 * controller when Hamming is chosen:
28 * +-------------------------------------------------------------+
29 * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30 * +-------------------------------------------------------------+
32 * When using the BCH engine, there are N identical (data + free OOB +
33 * ECC) sections and potentially an extra one to deal with
34 * configurations where the chosen (data + free OOB + ECC) sizes do
35 * not align with the page (data + OOB) size. ECC bytes are always
36 * 30B per ECC chunk. Here is the page layout used by the controller
39 * +-----------------------------------------
40 * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41 * +-----------------------------------------
43 * -------------------------------------------
44 * ... | Data N | Free OOB bytes N | ECC N |
45 * -------------------------------------------
47 * --------------------------------------------+
48 * Last Data | Last Free OOB bytes | Last ECC |
49 * --------------------------------------------+
51 * In both cases, the layout seen by the user is always: all data
52 * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53 * ECC bytes are 30B long and are padded with 0xFF to align on 32
56 * The controller has certain limitations that are handled by the
58 * - It can only read 2k at a time. To overcome this limitation, the
59 * driver issues data cycles on the bus, without issuing new
60 * CMD + ADDR cycles. The Marvell term is "naked" operations.
61 * - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62 * bits. What can be tuned is the ECC block size as long as it
63 * stays between 512B and 2kiB. It's usually chosen based on the
64 * chip ECC requirements. For instance, using 2kiB ECC chunks
65 * provides 4b/512B correctability.
66 * - The controller will always treat data bytes, free OOB bytes
67 * and ECC bytes in that order, no matter what the real layout is
68 * (which is usually all data then all OOB bytes). The
69 * marvell_nfc_layouts array below contains the currently
71 * - Because of these weird layouts, the Bad Block Markers can be
72 * located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73 * option must be set to prevent scanning/writing bad block
77 #include <linux/module.h>
78 #include <linux/clk.h>
79 #include <linux/mtd/rawnand.h>
80 #include <linux/of_platform.h>
81 #include <linux/iopoll.h>
82 #include <linux/interrupt.h>
83 #include <linux/slab.h>
84 #include <linux/mfd/syscon.h>
85 #include <linux/regmap.h>
86 #include <asm/unaligned.h>
88 #include <linux/dmaengine.h>
89 #include <linux/dma-mapping.h>
90 #include <linux/dma/pxa-dma.h>
91 #include <linux/platform_data/mtd-nand-pxa3xx.h>
93 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
95 #define FIFO_REP(x) (x / sizeof(u32))
96 #define BCH_SEQ_READS (32 / FIFO_DEPTH)
97 /* NFC does not support transfers of larger chunks at a time */
98 #define MAX_CHUNK_SIZE 2112
99 /* NFCv1 cannot read more that 7 bytes of ID */
100 #define NFCV1_READID_LEN 7
101 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
102 #define POLL_PERIOD 0
103 #define POLL_TIMEOUT 100000
104 /* Interrupt maximum wait period in ms */
105 #define IRQ_TIMEOUT 1000
106 /* Latency in clock cycles between SoC pins and NFC logic */
107 #define MIN_RD_DEL_CNT 3
108 /* Maximum number of contiguous address cycles */
109 #define MAX_ADDRESS_CYC_NFCV1 5
110 #define MAX_ADDRESS_CYC_NFCV2 7
111 /* System control registers/bits to enable the NAND controller on some SoCs */
112 #define GENCONF_SOC_DEVICE_MUX 0x208
113 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
114 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
115 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
116 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
117 #define GENCONF_CLK_GATING_CTRL 0x220
118 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
119 #define GENCONF_ND_CLK_CTRL 0x700
120 #define GENCONF_ND_CLK_CTRL_EN BIT(0)
122 /* NAND controller data flash control register */
124 #define NDCR_ALL_INT GENMASK(11, 0)
125 #define NDCR_CS1_CMDDM BIT(7)
126 #define NDCR_CS0_CMDDM BIT(8)
127 #define NDCR_RDYM BIT(11)
128 #define NDCR_ND_ARB_EN BIT(12)
129 #define NDCR_RA_START BIT(15)
130 #define NDCR_RD_ID_CNT(x) (min_t(unsigned int, x, 0x7) << 16)
131 #define NDCR_PAGE_SZ(x) (x >= 2048 ? BIT(24) : 0)
132 #define NDCR_DWIDTH_M BIT(26)
133 #define NDCR_DWIDTH_C BIT(27)
134 #define NDCR_ND_RUN BIT(28)
135 #define NDCR_DMA_EN BIT(29)
136 #define NDCR_ECC_EN BIT(30)
137 #define NDCR_SPARE_EN BIT(31)
138 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
139 NDCR_DWIDTH_M | NDCR_DWIDTH_C))
141 /* NAND interface timing parameter 0 register */
143 #define NDTR0_TRP(x) ((min_t(unsigned int, x, 0xF) & 0x7) << 0)
144 #define NDTR0_TRH(x) (min_t(unsigned int, x, 0x7) << 3)
145 #define NDTR0_ETRP(x) ((min_t(unsigned int, x, 0xF) & 0x8) << 3)
146 #define NDTR0_SEL_NRE_EDGE BIT(7)
147 #define NDTR0_TWP(x) (min_t(unsigned int, x, 0x7) << 8)
148 #define NDTR0_TWH(x) (min_t(unsigned int, x, 0x7) << 11)
149 #define NDTR0_TCS(x) (min_t(unsigned int, x, 0x7) << 16)
150 #define NDTR0_TCH(x) (min_t(unsigned int, x, 0x7) << 19)
151 #define NDTR0_RD_CNT_DEL(x) (min_t(unsigned int, x, 0xF) << 22)
152 #define NDTR0_SELCNTR BIT(26)
153 #define NDTR0_TADL(x) (min_t(unsigned int, x, 0x1F) << 27)
155 /* NAND interface timing parameter 1 register */
157 #define NDTR1_TAR(x) (min_t(unsigned int, x, 0xF) << 0)
158 #define NDTR1_TWHR(x) (min_t(unsigned int, x, 0xF) << 4)
159 #define NDTR1_TRHW(x) (min_t(unsigned int, x / 16, 0x3) << 8)
160 #define NDTR1_PRESCALE BIT(14)
161 #define NDTR1_WAIT_MODE BIT(15)
162 #define NDTR1_TR(x) (min_t(unsigned int, x, 0xFFFF) << 16)
164 /* NAND controller status register */
166 #define NDSR_WRCMDREQ BIT(0)
167 #define NDSR_RDDREQ BIT(1)
168 #define NDSR_WRDREQ BIT(2)
169 #define NDSR_CORERR BIT(3)
170 #define NDSR_UNCERR BIT(4)
171 #define NDSR_CMDD(cs) BIT(8 - cs)
172 #define NDSR_RDY(rb) BIT(11 + rb)
173 #define NDSR_ERRCNT(x) ((x >> 16) & 0x1F)
175 /* NAND ECC control register */
176 #define NDECCCTRL 0x28
177 #define NDECCCTRL_BCH_EN BIT(0)
179 /* NAND controller data buffer register */
182 /* NAND controller command buffer 0 register */
184 #define NDCB0_CMD1(x) ((x & 0xFF) << 0)
185 #define NDCB0_CMD2(x) ((x & 0xFF) << 8)
186 #define NDCB0_ADDR_CYC(x) ((x & 0x7) << 16)
187 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
188 #define NDCB0_DBC BIT(19)
189 #define NDCB0_CMD_TYPE(x) ((x & 0x7) << 21)
190 #define NDCB0_CSEL BIT(24)
191 #define NDCB0_RDY_BYP BIT(27)
192 #define NDCB0_LEN_OVRD BIT(28)
193 #define NDCB0_CMD_XTYPE(x) ((x & 0x7) << 29)
195 /* NAND controller command buffer 1 register */
197 #define NDCB1_COLS(x) ((x & 0xFFFF) << 0)
198 #define NDCB1_ADDRS_PAGE(x) (x << 16)
200 /* NAND controller command buffer 2 register */
202 #define NDCB2_ADDR5_PAGE(x) (((x >> 16) & 0xFF) << 0)
203 #define NDCB2_ADDR5_CYC(x) ((x & 0xFF) << 0)
205 /* NAND controller command buffer 3 register */
207 #define NDCB3_ADDR6_CYC(x) ((x & 0xFF) << 16)
208 #define NDCB3_ADDR7_CYC(x) ((x & 0xFF) << 24)
210 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
214 #define TYPE_READ_ID 3
215 #define TYPE_STATUS 4
217 #define TYPE_NAKED_CMD 6
218 #define TYPE_NAKED_ADDR 7
220 #define XTYPE_MONOLITHIC_RW 0
221 #define XTYPE_LAST_NAKED_RW 1
222 #define XTYPE_FINAL_COMMAND 3
224 #define XTYPE_WRITE_DISPATCH 4
225 #define XTYPE_NAKED_RW 5
226 #define XTYPE_COMMAND_DISPATCH 6
230 * Marvell ECC engine works differently than the others, in order to limit the
231 * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
232 * per subpage, and depending on a the desired strength needed by the NAND chip,
233 * a particular layout mixing data/spare/ecc is defined, with a possible last
234 * chunk smaller that the others.
236 * @writesize: Full page size on which the layout applies
237 * @chunk: Desired ECC chunk size on which the layout applies
238 * @strength: Desired ECC strength (per chunk size bytes) on which the
240 * @nchunks: Total number of chunks
241 * @full_chunk_cnt: Number of full-sized chunks, which is the number of
242 * repetitions of the pattern:
243 * (data_bytes + spare_bytes + ecc_bytes).
244 * @data_bytes: Number of data bytes per chunk
245 * @spare_bytes: Number of spare bytes per chunk
246 * @ecc_bytes: Number of ecc bytes per chunk
247 * @last_data_bytes: Number of data bytes in the last chunk
248 * @last_spare_bytes: Number of spare bytes in the last chunk
249 * @last_ecc_bytes: Number of ecc bytes in the last chunk
251 struct marvell_hw_ecc_layout {
256 /* Corresponding layout */
263 int last_spare_bytes;
267 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb) \
273 .full_chunk_cnt = fcc, \
277 .last_data_bytes = ldb, \
278 .last_spare_bytes = lsb, \
279 .last_ecc_bytes = leb, \
282 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
283 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
284 MARVELL_LAYOUT( 512, 512, 1, 1, 1, 512, 8, 8, 0, 0, 0),
285 MARVELL_LAYOUT( 2048, 512, 1, 1, 1, 2048, 40, 24, 0, 0, 0),
286 MARVELL_LAYOUT( 2048, 512, 4, 1, 1, 2048, 32, 30, 0, 0, 0),
287 MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,32, 30),
288 MARVELL_LAYOUT( 4096, 512, 4, 2, 2, 2048, 32, 30, 0, 0, 0),
289 MARVELL_LAYOUT( 4096, 512, 8, 5, 4, 1024, 0, 30, 0, 64, 30),
290 MARVELL_LAYOUT( 8192, 512, 4, 4, 4, 2048, 0, 30, 0, 0, 0),
291 MARVELL_LAYOUT( 8192, 512, 8, 9, 8, 1024, 0, 30, 0, 160, 30),
295 * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
296 * is made by a field in NDCB0 register, and in another field in NDCB2 register.
297 * The datasheet describes the logic with an error: ADDR5 field is once
298 * declared at the beginning of NDCB2, and another time at its end. Because the
299 * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
300 * to use the last bit of this field instead of the first ones.
302 * @cs: Wanted CE lane.
303 * @ndcb0_csel: Value of the NDCB0 register with or without the flag
304 * selecting the wanted CE lane. This is set once when
305 * the Device Tree is probed.
306 * @rb: Ready/Busy pin for the flash chip
308 struct marvell_nand_chip_sel {
315 * NAND chip structure: stores NAND chip device related information
317 * @chip: Base NAND chip structure
318 * @node: Used to store NAND chips into a list
319 * @layout NAND layout when using hardware ECC
320 * @ndcr: Controller register value for this NAND chip
321 * @ndtr0: Timing registers 0 value for this NAND chip
322 * @ndtr1: Timing registers 1 value for this NAND chip
323 * @selected_die: Current active CS
324 * @nsels: Number of CS lines required by the NAND chip
325 * @sels: Array of CS lines descriptions
327 struct marvell_nand_chip {
328 struct nand_chip chip;
329 struct list_head node;
330 const struct marvell_hw_ecc_layout *layout;
337 struct marvell_nand_chip_sel sels[0];
340 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
342 return container_of(chip, struct marvell_nand_chip, chip);
345 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
348 return &nand->sels[nand->selected_die];
352 * NAND controller capabilities for distinction between compatible strings
354 * @max_cs_nb: Number of Chip Select lines available
355 * @max_rb_nb: Number of Ready/Busy lines available
356 * @need_system_controller: Indicates if the SoC needs to have access to the
357 * system controller (ie. to enable the NAND controller)
358 * @legacy_of_bindings: Indicates if DT parsing must be done using the old
360 * @is_nfcv2: NFCv2 has numerous enhancements compared to NFCv1, ie.
361 * BCH error detection and correction algorithm,
362 * NDCB3 register has been added
363 * @use_dma: Use dma for data transfers
365 struct marvell_nfc_caps {
366 unsigned int max_cs_nb;
367 unsigned int max_rb_nb;
368 bool need_system_controller;
369 bool legacy_of_bindings;
375 * NAND controller structure: stores Marvell NAND controller information
377 * @controller: Base controller structure
378 * @dev: Parent device (used to print error messages)
379 * @regs: NAND controller registers
380 * @core_clk: Core clock
381 * @reg_clk: Regiters clock
382 * @complete: Completion object to wait for NAND controller events
383 * @assigned_cs: Bitmask describing already assigned CS lines
384 * @chips: List containing all the NAND chips attached to
385 * this NAND controller
386 * @caps: NAND controller capabilities for each compatible string
387 * @dma_chan: DMA channel (NFCv1 only)
388 * @dma_buf: 32-bit aligned buffer for DMA transfers (NFCv1 only)
391 struct nand_controller controller;
394 struct clk *core_clk;
396 struct completion complete;
397 unsigned long assigned_cs;
398 struct list_head chips;
399 struct nand_chip *selected_chip;
400 const struct marvell_nfc_caps *caps;
402 /* DMA (NFCv1 only) */
404 struct dma_chan *dma_chan;
408 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
410 return container_of(ctrl, struct marvell_nfc, controller);
414 * NAND controller timings expressed in NAND Controller clock cycles
416 * @tRP: ND_nRE pulse width
417 * @tRH: ND_nRE high duration
418 * @tWP: ND_nWE pulse time
419 * @tWH: ND_nWE high duration
420 * @tCS: Enable signal setup time
421 * @tCH: Enable signal hold time
422 * @tADL: Address to write data delay
423 * @tAR: ND_ALE low to ND_nRE low delay
424 * @tWHR: ND_nWE high to ND_nRE low for status read
425 * @tRHW: ND_nRE high duration, read to write delay
426 * @tR: ND_nWE high to ND_nRE low for read
428 struct marvell_nfc_timings {
445 * Derives a duration in numbers of clock cycles.
447 * @ps: Duration in pico-seconds
448 * @period_ns: Clock period in nano-seconds
450 * Convert the duration in nano-seconds, then divide by the period and
451 * return the number of clock periods.
453 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
454 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
458 * NAND driver structure filled during the parsing of the ->exec_op() subop
459 * subset of instructions.
461 * @ndcb: Array of values written to NDCBx registers
462 * @cle_ale_delay_ns: Optional delay after the last CMD or ADDR cycle
463 * @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
464 * @rdy_delay_ns: Optional delay after waiting for the RB pin
465 * @data_delay_ns: Optional delay after the data xfer
466 * @data_instr_idx: Index of the data instruction in the subop
467 * @data_instr: Pointer to the data instruction in the subop
469 struct marvell_nfc_op {
471 unsigned int cle_ale_delay_ns;
472 unsigned int rdy_timeout_ms;
473 unsigned int rdy_delay_ns;
474 unsigned int data_delay_ns;
475 unsigned int data_instr_idx;
476 const struct nand_op_instr *data_instr;
480 * Internal helper to conditionnally apply a delay (from the above structure,
483 static void cond_delay(unsigned int ns)
491 udelay(DIV_ROUND_UP(ns, 1000));
495 * The controller has many flags that could generate interrupts, most of them
496 * are disabled and polling is used. For the very slow signals, using interrupts
497 * may relax the CPU charge.
499 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
503 /* Writing 1 disables the interrupt */
504 reg = readl_relaxed(nfc->regs + NDCR);
505 writel_relaxed(reg | int_mask, nfc->regs + NDCR);
508 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
512 /* Writing 0 enables the interrupt */
513 reg = readl_relaxed(nfc->regs + NDCR);
514 writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
517 static void marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
519 writel_relaxed(int_mask, nfc->regs + NDSR);
522 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
525 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
529 * Callers of this function do not verify if the NAND is using a 16-bit
530 * an 8-bit bus for normal operations, so we need to take care of that
531 * here by leaving the configuration unchanged if the NAND does not have
532 * the NAND_BUSWIDTH_16 flag set.
534 if (!(chip->options & NAND_BUSWIDTH_16))
537 ndcr = readl_relaxed(nfc->regs + NDCR);
540 ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
542 ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
544 writel_relaxed(ndcr, nfc->regs + NDCR);
547 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
549 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
554 * The command is being processed, wait for the ND_RUN bit to be
555 * cleared by the NFC. If not, we must clear it by hand.
557 ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
558 (val & NDCR_ND_RUN) == 0,
559 POLL_PERIOD, POLL_TIMEOUT);
561 dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
562 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
571 * Any time a command has to be sent to the controller, the following sequence
572 * has to be followed:
573 * - call marvell_nfc_prepare_cmd()
574 * -> activate the ND_RUN bit that will kind of 'start a job'
575 * -> wait the signal indicating the NFC is waiting for a command
576 * - send the command (cmd and address cycles)
577 * - enventually send or receive the data
578 * - call marvell_nfc_end_cmd() with the corresponding flag
579 * -> wait the flag to be triggered or cancel the job with a timeout
581 * The following helpers are here to factorize the code a bit so that
582 * specialized functions responsible for executing the actual NAND
583 * operations do not have to replicate the same code blocks.
585 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
587 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
591 /* Poll ND_RUN and clear NDSR before issuing any command */
592 ret = marvell_nfc_wait_ndrun(chip);
594 dev_err(nfc->dev, "Last operation did not succeed\n");
598 ndcr = readl_relaxed(nfc->regs + NDCR);
599 writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
601 /* Assert ND_RUN bit and wait the NFC to be ready */
602 writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
603 ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
605 POLL_PERIOD, POLL_TIMEOUT);
607 dev_err(nfc->dev, "Timeout on WRCMDRE\n");
611 /* Command may be written, clear WRCMDREQ status bit */
612 writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
617 static void marvell_nfc_send_cmd(struct nand_chip *chip,
618 struct marvell_nfc_op *nfc_op)
620 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
621 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
623 dev_dbg(nfc->dev, "\nNDCR: 0x%08x\n"
624 "NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
625 (u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
626 nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
628 writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
630 writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
631 writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
634 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
635 * fields are used (only available on NFCv2).
637 if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
638 NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
639 if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
640 writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
644 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
647 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
651 ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
653 POLL_PERIOD, POLL_TIMEOUT);
656 dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
659 dmaengine_terminate_all(nfc->dma_chan);
664 * DMA function uses this helper to poll on CMDD bits without wanting
665 * them to be cleared.
667 if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
670 writel_relaxed(flag, nfc->regs + NDSR);
675 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
677 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
678 int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
680 return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
683 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
685 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
688 /* Timeout is expressed in ms */
690 timeout_ms = IRQ_TIMEOUT;
692 init_completion(&nfc->complete);
694 marvell_nfc_enable_int(nfc, NDCR_RDYM);
695 ret = wait_for_completion_timeout(&nfc->complete,
696 msecs_to_jiffies(timeout_ms));
697 marvell_nfc_disable_int(nfc, NDCR_RDYM);
698 marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
700 dev_err(nfc->dev, "Timeout waiting for RB signal\n");
707 static void marvell_nfc_select_chip(struct nand_chip *chip, int die_nr)
709 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
710 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
713 if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
716 if (die_nr < 0 || die_nr >= marvell_nand->nsels) {
717 nfc->selected_chip = NULL;
718 marvell_nand->selected_die = -1;
722 writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
723 writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
726 * Reset the NDCR register to a clean state for this particular chip,
727 * also clear ND_RUN bit.
729 ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
730 NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
731 writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
733 /* Also reset the interrupt status register */
734 marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
736 nfc->selected_chip = chip;
737 marvell_nand->selected_die = die_nr;
740 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
742 struct marvell_nfc *nfc = dev_id;
743 u32 st = readl_relaxed(nfc->regs + NDSR);
744 u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
747 * RDY interrupt mask is one bit in NDCR while there are two status
748 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
750 if (st & NDSR_RDY(1))
756 marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
758 if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
759 complete(&nfc->complete);
764 /* HW ECC related functions */
765 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
767 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
768 u32 ndcr = readl_relaxed(nfc->regs + NDCR);
770 if (!(ndcr & NDCR_ECC_EN)) {
771 writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
774 * When enabling BCH, set threshold to 0 to always know the
775 * number of corrected bitflips.
777 if (chip->ecc.algo == NAND_ECC_BCH)
778 writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
782 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
784 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
785 u32 ndcr = readl_relaxed(nfc->regs + NDCR);
787 if (ndcr & NDCR_ECC_EN) {
788 writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
789 if (chip->ecc.algo == NAND_ECC_BCH)
790 writel_relaxed(0, nfc->regs + NDECCCTRL);
794 /* DMA related helpers */
795 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
799 reg = readl_relaxed(nfc->regs + NDCR);
800 writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
803 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
807 reg = readl_relaxed(nfc->regs + NDCR);
808 writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
811 /* Read/write PIO/DMA accessors */
812 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
813 enum dma_data_direction direction,
816 unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
817 struct dma_async_tx_descriptor *tx;
818 struct scatterlist sg;
822 marvell_nfc_enable_dma(nfc);
823 /* Prepare the DMA transfer */
824 sg_init_one(&sg, nfc->dma_buf, dma_len);
825 dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
826 tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
827 direction == DMA_FROM_DEVICE ?
828 DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
831 dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
835 /* Do the task and wait for it to finish */
836 cookie = dmaengine_submit(tx);
837 ret = dma_submit_error(cookie);
841 dma_async_issue_pending(nfc->dma_chan);
842 ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
843 dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
844 marvell_nfc_disable_dma(nfc);
846 dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
847 dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
848 dmaengine_terminate_all(nfc->dma_chan);
855 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
858 unsigned int last_len = len % FIFO_DEPTH;
859 unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
862 for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
863 ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
866 u8 tmp_buf[FIFO_DEPTH];
868 ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
869 memcpy(in + last_full_offset, tmp_buf, last_len);
875 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
878 unsigned int last_len = len % FIFO_DEPTH;
879 unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
882 for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
883 iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
886 u8 tmp_buf[FIFO_DEPTH];
888 memcpy(tmp_buf, out + last_full_offset, last_len);
889 iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
895 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
896 u8 *data, int data_len,
897 u8 *spare, int spare_len,
898 u8 *ecc, int ecc_len,
899 unsigned int *max_bitflips)
901 struct mtd_info *mtd = nand_to_mtd(chip);
905 * Blank pages (all 0xFF) that have not been written may be recognized
906 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
907 * check if the entire page (with ECC bytes) is actually blank or not.
916 bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
917 spare, spare_len, chip->ecc.strength);
919 mtd->ecc_stats.failed++;
923 /* Update the stats and max_bitflips */
924 mtd->ecc_stats.corrected += bf;
925 *max_bitflips = max_t(unsigned int, *max_bitflips, bf);
929 * Check a chunk is correct or not according to hardware ECC engine.
930 * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
931 * mtd->ecc_stats.failure is not, the function will instead return a non-zero
932 * value indicating that a check on the emptyness of the subpage must be
933 * performed before declaring the subpage corrupted.
935 static int marvell_nfc_hw_ecc_correct(struct nand_chip *chip,
936 unsigned int *max_bitflips)
938 struct mtd_info *mtd = nand_to_mtd(chip);
939 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
943 ndsr = readl_relaxed(nfc->regs + NDSR);
945 /* Check uncorrectable error flag */
946 if (ndsr & NDSR_UNCERR) {
947 writel_relaxed(ndsr, nfc->regs + NDSR);
950 * Do not increment ->ecc_stats.failed now, instead, return a
951 * non-zero value to indicate that this chunk was apparently
952 * bad, and it should be check to see if it empty or not. If
953 * the chunk (with ECC bytes) is not declared empty, the calling
954 * function must increment the failure count.
959 /* Check correctable error flag */
960 if (ndsr & NDSR_CORERR) {
961 writel_relaxed(ndsr, nfc->regs + NDSR);
963 if (chip->ecc.algo == NAND_ECC_BCH)
964 bf = NDSR_ERRCNT(ndsr);
969 /* Update the stats and max_bitflips */
970 mtd->ecc_stats.corrected += bf;
971 *max_bitflips = max_t(unsigned int, *max_bitflips, bf);
976 /* Hamming read helpers */
977 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
978 u8 *data_buf, u8 *oob_buf,
981 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
982 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
983 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
984 struct marvell_nfc_op nfc_op = {
985 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
986 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
988 NDCB0_CMD1(NAND_CMD_READ0) |
989 NDCB0_CMD2(NAND_CMD_READSTART),
990 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
991 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
993 unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
996 /* NFCv2 needs more information about the operation being executed */
997 if (nfc->caps->is_nfcv2)
998 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1000 ret = marvell_nfc_prepare_cmd(chip);
1004 marvell_nfc_send_cmd(chip, &nfc_op);
1005 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1006 "RDDREQ while draining FIFO (data/oob)");
1011 * Read the page then the OOB area. Unlike what is shown in current
1012 * documentation, spare bytes are protected by the ECC engine, and must
1013 * be at the beginning of the OOB area or running this driver on legacy
1014 * systems will prevent the discovery of the BBM/BBT.
1017 marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1018 lt->data_bytes + oob_bytes);
1019 memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1020 memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1022 marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1023 marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1026 ret = marvell_nfc_wait_cmdd(chip);
1031 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1032 int oob_required, int page)
1034 return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1038 static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1039 int oob_required, int page)
1041 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1042 unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1043 int max_bitflips = 0, ret;
1046 marvell_nfc_enable_hw_ecc(chip);
1047 marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1049 ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1050 marvell_nfc_disable_hw_ecc(chip);
1053 return max_bitflips;
1056 * When ECC failures are detected, check if the full page has been
1057 * written or not. Ignore the failure if it is actually empty.
1059 raw_buf = kmalloc(full_sz, GFP_KERNEL);
1063 marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1064 lt->data_bytes, true, page);
1065 marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1069 return max_bitflips;
1073 * Spare area in Hamming layouts is not protected by the ECC engine (even if
1074 * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1075 * also stands for ->read_oob().
1077 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1079 /* Invalidate page cache */
1082 return marvell_nfc_hw_ecc_hmg_do_read_page(chip, chip->data_buf,
1083 chip->oob_poi, true, page);
1086 /* Hamming write helpers */
1087 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1089 const u8 *oob_buf, bool raw,
1092 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1093 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1094 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1095 struct marvell_nfc_op nfc_op = {
1096 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1097 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1098 NDCB0_CMD1(NAND_CMD_SEQIN) |
1099 NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1101 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
1102 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
1104 unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1107 /* NFCv2 needs more information about the operation being executed */
1108 if (nfc->caps->is_nfcv2)
1109 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1111 ret = marvell_nfc_prepare_cmd(chip);
1115 marvell_nfc_send_cmd(chip, &nfc_op);
1116 ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1117 "WRDREQ while loading FIFO (data)");
1121 /* Write the page then the OOB area */
1123 memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1124 memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1125 marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1126 lt->ecc_bytes + lt->spare_bytes);
1128 marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1129 marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1132 ret = marvell_nfc_wait_cmdd(chip);
1136 ret = marvell_nfc_wait_op(chip,
1137 PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
1141 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1143 int oob_required, int page)
1145 return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1149 static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1151 int oob_required, int page)
1155 marvell_nfc_enable_hw_ecc(chip);
1156 ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1158 marvell_nfc_disable_hw_ecc(chip);
1164 * Spare area in Hamming layouts is not protected by the ECC engine (even if
1165 * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1166 * also stands for ->write_oob().
1168 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1171 struct mtd_info *mtd = nand_to_mtd(chip);
1173 /* Invalidate page cache */
1176 memset(chip->data_buf, 0xFF, mtd->writesize);
1178 return marvell_nfc_hw_ecc_hmg_do_write_page(chip, chip->data_buf,
1179 chip->oob_poi, true, page);
1182 /* BCH read helpers */
1183 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1184 int oob_required, int page)
1186 struct mtd_info *mtd = nand_to_mtd(chip);
1187 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1188 u8 *oob = chip->oob_poi;
1189 int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1190 int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1191 lt->last_spare_bytes;
1192 int data_len = lt->data_bytes;
1193 int spare_len = lt->spare_bytes;
1194 int ecc_len = lt->ecc_bytes;
1198 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1200 nand_read_page_op(chip, page, 0, NULL, 0);
1202 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1203 /* Update last chunk length */
1204 if (chunk >= lt->full_chunk_cnt) {
1205 data_len = lt->last_data_bytes;
1206 spare_len = lt->last_spare_bytes;
1207 ecc_len = lt->last_ecc_bytes;
1210 /* Read data bytes*/
1211 nand_change_read_column_op(chip, chunk * chunk_size,
1212 buf + (lt->data_bytes * chunk),
1215 /* Read spare bytes */
1216 nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1219 /* Read ECC bytes */
1220 nand_read_data_op(chip, oob + ecc_offset +
1221 (ALIGN(lt->ecc_bytes, 32) * chunk),
1228 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1229 u8 *data, unsigned int data_len,
1230 u8 *spare, unsigned int spare_len,
1233 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1234 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1235 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1237 struct marvell_nfc_op nfc_op = {
1238 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1239 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1241 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
1242 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
1243 .ndcb[3] = data_len + spare_len,
1246 ret = marvell_nfc_prepare_cmd(chip);
1251 nfc_op.ndcb[0] |= NDCB0_DBC |
1252 NDCB0_CMD1(NAND_CMD_READ0) |
1253 NDCB0_CMD2(NAND_CMD_READSTART);
1256 * Trigger the monolithic read on the first chunk, then naked read on
1257 * intermediate chunks and finally a last naked read on the last chunk.
1260 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1261 else if (chunk < lt->nchunks - 1)
1262 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1264 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1266 marvell_nfc_send_cmd(chip, &nfc_op);
1269 * According to the datasheet, when reading from NDDB
1270 * with BCH enabled, after each 32 bytes reads, we
1271 * have to make sure that the NDSR.RDDREQ bit is set.
1273 * Drain the FIFO, 8 32-bit reads at a time, and skip
1274 * the polling on the last read.
1276 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1278 for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1279 marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1280 "RDDREQ while draining FIFO (data)");
1281 marvell_nfc_xfer_data_in_pio(nfc, data,
1282 FIFO_DEPTH * BCH_SEQ_READS);
1283 data += FIFO_DEPTH * BCH_SEQ_READS;
1286 for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1287 marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1288 "RDDREQ while draining FIFO (OOB)");
1289 marvell_nfc_xfer_data_in_pio(nfc, spare,
1290 FIFO_DEPTH * BCH_SEQ_READS);
1291 spare += FIFO_DEPTH * BCH_SEQ_READS;
1295 static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1296 u8 *buf, int oob_required,
1299 struct mtd_info *mtd = nand_to_mtd(chip);
1300 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1301 int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1302 u8 *data = buf, *spare = chip->oob_poi;
1303 int max_bitflips = 0;
1304 u32 failure_mask = 0;
1308 * With BCH, OOB is not fully used (and thus not read entirely), not
1309 * expected bytes could show up at the end of the OOB buffer if not
1310 * explicitly erased.
1313 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1315 marvell_nfc_enable_hw_ecc(chip);
1317 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1318 /* Update length for the last chunk */
1319 if (chunk >= lt->full_chunk_cnt) {
1320 data_len = lt->last_data_bytes;
1321 spare_len = lt->last_spare_bytes;
1324 /* Read the chunk and detect number of bitflips */
1325 marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1326 spare, spare_len, page);
1327 ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1329 failure_mask |= BIT(chunk);
1335 marvell_nfc_disable_hw_ecc(chip);
1338 return max_bitflips;
1341 * Please note that dumping the ECC bytes during a normal read with OOB
1342 * area would add a significant overhead as ECC bytes are "consumed" by
1343 * the controller in normal mode and must be re-read in raw mode. To
1344 * avoid dropping the performances, we prefer not to include them. The
1345 * user should re-read the page in raw mode if ECC bytes are required.
1349 * In case there is any subpage read error reported by ->correct(), we
1350 * usually re-read only ECC bytes in raw mode and check if the whole
1351 * page is empty. In this case, it is normal that the ECC check failed
1352 * and we just ignore the error.
1354 * However, it has been empirically observed that for some layouts (e.g
1355 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1356 * bits and may create itself bitflips in the erased area. To overcome
1357 * this strange behavior, the whole page is re-read in raw mode, not
1358 * only the ECC bytes.
1360 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1361 int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1362 int data_off, spare_off, ecc_off;
1363 int data_len, spare_len, ecc_len;
1365 /* No failure reported for this chunk, move to the next one */
1366 if (!(failure_mask & BIT(chunk)))
1369 data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1371 spare_off_in_page = data_off_in_page +
1372 (chunk < lt->full_chunk_cnt ? lt->data_bytes :
1373 lt->last_data_bytes);
1374 ecc_off_in_page = spare_off_in_page +
1375 (chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1376 lt->last_spare_bytes);
1378 data_off = chunk * lt->data_bytes;
1379 spare_off = chunk * lt->spare_bytes;
1380 ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1381 lt->last_spare_bytes +
1382 (chunk * (lt->ecc_bytes + 2));
1384 data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1385 lt->last_data_bytes;
1386 spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1387 lt->last_spare_bytes;
1388 ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1392 * Only re-read the ECC bytes, unless we are using the 2k/8b
1393 * layout which is buggy in the sense that the ECC engine will
1394 * try to correct data bytes anyway, creating bitflips. In this
1395 * case, re-read the entire page.
1397 if (lt->writesize == 2048 && lt->strength == 8) {
1398 nand_change_read_column_op(chip, data_off_in_page,
1399 buf + data_off, data_len,
1401 nand_change_read_column_op(chip, spare_off_in_page,
1402 chip->oob_poi + spare_off, spare_len,
1406 nand_change_read_column_op(chip, ecc_off_in_page,
1407 chip->oob_poi + ecc_off, ecc_len,
1410 /* Check the entire chunk (data + spare + ecc) for emptyness */
1411 marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1412 chip->oob_poi + spare_off, spare_len,
1413 chip->oob_poi + ecc_off, ecc_len,
1417 return max_bitflips;
1420 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1422 /* Invalidate page cache */
1425 return chip->ecc.read_page_raw(chip, chip->data_buf, true, page);
1428 static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1430 /* Invalidate page cache */
1433 return chip->ecc.read_page(chip, chip->data_buf, true, page);
1436 /* BCH write helpers */
1437 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1439 int oob_required, int page)
1441 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1442 int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1443 int data_len = lt->data_bytes;
1444 int spare_len = lt->spare_bytes;
1445 int ecc_len = lt->ecc_bytes;
1446 int spare_offset = 0;
1447 int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1448 lt->last_spare_bytes;
1451 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1453 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1454 if (chunk >= lt->full_chunk_cnt) {
1455 data_len = lt->last_data_bytes;
1456 spare_len = lt->last_spare_bytes;
1457 ecc_len = lt->last_ecc_bytes;
1460 /* Point to the column of the next chunk */
1461 nand_change_write_column_op(chip, chunk * full_chunk_size,
1464 /* Write the data */
1465 nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1471 /* Write the spare bytes */
1473 nand_write_data_op(chip, chip->oob_poi + spare_offset,
1476 /* Write the ECC bytes */
1478 nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1481 spare_offset += spare_len;
1482 ecc_offset += ALIGN(ecc_len, 32);
1485 return nand_prog_page_end_op(chip);
1489 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1490 const u8 *data, unsigned int data_len,
1491 const u8 *spare, unsigned int spare_len,
1494 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1495 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1496 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1499 struct marvell_nfc_op nfc_op = {
1500 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1501 .ndcb[3] = data_len + spare_len,
1505 * First operation dispatches the CMD_SEQIN command, issue the address
1506 * cycles and asks for the first chunk of data.
1507 * All operations in the middle (if any) will issue a naked write and
1508 * also ask for data.
1509 * Last operation (if any) asks for the last chunk of data through a
1513 if (lt->nchunks == 1)
1514 xtype = XTYPE_MONOLITHIC_RW;
1516 xtype = XTYPE_WRITE_DISPATCH;
1518 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1519 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1520 NDCB0_CMD1(NAND_CMD_SEQIN);
1521 nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1522 nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1523 } else if (chunk < lt->nchunks - 1) {
1524 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1526 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1529 /* Always dispatch the PAGEPROG command on the last chunk */
1530 if (chunk == lt->nchunks - 1)
1531 nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1533 ret = marvell_nfc_prepare_cmd(chip);
1537 marvell_nfc_send_cmd(chip, &nfc_op);
1538 ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1539 "WRDREQ while loading FIFO (data)");
1543 /* Transfer the contents */
1544 iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1545 iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1550 static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1552 int oob_required, int page)
1554 struct mtd_info *mtd = nand_to_mtd(chip);
1555 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1556 const u8 *data = buf;
1557 const u8 *spare = chip->oob_poi;
1558 int data_len = lt->data_bytes;
1559 int spare_len = lt->spare_bytes;
1562 /* Spare data will be written anyway, so clear it to avoid garbage */
1564 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1566 marvell_nfc_enable_hw_ecc(chip);
1568 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1569 if (chunk >= lt->full_chunk_cnt) {
1570 data_len = lt->last_data_bytes;
1571 spare_len = lt->last_spare_bytes;
1574 marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1575 spare, spare_len, page);
1580 * Waiting only for CMDD or PAGED is not enough, ECC are
1581 * partially written. No flag is set once the operation is
1582 * really finished but the ND_RUN bit is cleared, so wait for it
1583 * before stepping into the next command.
1585 marvell_nfc_wait_ndrun(chip);
1588 ret = marvell_nfc_wait_op(chip,
1589 PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
1591 marvell_nfc_disable_hw_ecc(chip);
1599 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1602 struct mtd_info *mtd = nand_to_mtd(chip);
1604 /* Invalidate page cache */
1607 memset(chip->data_buf, 0xFF, mtd->writesize);
1609 return chip->ecc.write_page_raw(chip, chip->data_buf, true, page);
1612 static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1614 struct mtd_info *mtd = nand_to_mtd(chip);
1616 /* Invalidate page cache */
1619 memset(chip->data_buf, 0xFF, mtd->writesize);
1621 return chip->ecc.write_page(chip, chip->data_buf, true, page);
1624 /* NAND framework ->exec_op() hooks and related helpers */
1625 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1626 const struct nand_subop *subop,
1627 struct marvell_nfc_op *nfc_op)
1629 const struct nand_op_instr *instr = NULL;
1630 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1631 bool first_cmd = true;
1635 /* Reset the input structure as most of its fields will be OR'ed */
1636 memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1638 for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1639 unsigned int offset, naddrs;
1643 instr = &subop->instrs[op_id];
1645 switch (instr->type) {
1646 case NAND_OP_CMD_INSTR:
1649 NDCB0_CMD1(instr->ctx.cmd.opcode);
1652 NDCB0_CMD2(instr->ctx.cmd.opcode) |
1655 nfc_op->cle_ale_delay_ns = instr->delay_ns;
1659 case NAND_OP_ADDR_INSTR:
1660 offset = nand_subop_get_addr_start_off(subop, op_id);
1661 naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1662 addrs = &instr->ctx.addr.addrs[offset];
1664 nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1666 for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1667 nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1670 nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1672 nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1674 nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1676 nfc_op->cle_ale_delay_ns = instr->delay_ns;
1679 case NAND_OP_DATA_IN_INSTR:
1680 nfc_op->data_instr = instr;
1681 nfc_op->data_instr_idx = op_id;
1682 nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1683 if (nfc->caps->is_nfcv2) {
1685 NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1687 len = nand_subop_get_data_len(subop, op_id);
1688 nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1690 nfc_op->data_delay_ns = instr->delay_ns;
1693 case NAND_OP_DATA_OUT_INSTR:
1694 nfc_op->data_instr = instr;
1695 nfc_op->data_instr_idx = op_id;
1696 nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1697 if (nfc->caps->is_nfcv2) {
1699 NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1701 len = nand_subop_get_data_len(subop, op_id);
1702 nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1704 nfc_op->data_delay_ns = instr->delay_ns;
1707 case NAND_OP_WAITRDY_INSTR:
1708 nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1709 nfc_op->rdy_delay_ns = instr->delay_ns;
1715 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1716 const struct nand_subop *subop,
1717 struct marvell_nfc_op *nfc_op)
1719 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1720 const struct nand_op_instr *instr = nfc_op->data_instr;
1721 unsigned int op_id = nfc_op->data_instr_idx;
1722 unsigned int len = nand_subop_get_data_len(subop, op_id);
1723 unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1724 bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1727 if (instr->ctx.data.force_8bit)
1728 marvell_nfc_force_byte_access(chip, true);
1731 u8 *in = instr->ctx.data.buf.in + offset;
1733 ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1735 const u8 *out = instr->ctx.data.buf.out + offset;
1737 ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1740 if (instr->ctx.data.force_8bit)
1741 marvell_nfc_force_byte_access(chip, false);
1746 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1747 const struct nand_subop *subop)
1749 struct marvell_nfc_op nfc_op;
1753 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1754 reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1756 ret = marvell_nfc_prepare_cmd(chip);
1760 marvell_nfc_send_cmd(chip, &nfc_op);
1761 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1762 "RDDREQ/WRDREQ while draining raw data");
1766 cond_delay(nfc_op.cle_ale_delay_ns);
1769 if (nfc_op.rdy_timeout_ms) {
1770 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1775 cond_delay(nfc_op.rdy_delay_ns);
1778 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1779 ret = marvell_nfc_wait_cmdd(chip);
1783 cond_delay(nfc_op.data_delay_ns);
1786 if (nfc_op.rdy_timeout_ms) {
1787 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1792 cond_delay(nfc_op.rdy_delay_ns);
1796 * NDCR ND_RUN bit should be cleared automatically at the end of each
1797 * operation but experience shows that the behavior is buggy when it
1798 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1801 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1803 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1810 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1811 const struct nand_subop *subop)
1813 struct marvell_nfc_op nfc_op;
1816 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1819 * Naked access are different in that they need to be flagged as naked
1820 * by the controller. Reset the controller registers fields that inform
1821 * on the type and refill them according to the ongoing operation.
1823 nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1824 NDCB0_CMD_XTYPE(XTYPE_MASK));
1825 switch (subop->instrs[0].type) {
1826 case NAND_OP_CMD_INSTR:
1827 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1829 case NAND_OP_ADDR_INSTR:
1830 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1832 case NAND_OP_DATA_IN_INSTR:
1833 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1834 NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1836 case NAND_OP_DATA_OUT_INSTR:
1837 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1838 NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1841 /* This should never happen */
1845 ret = marvell_nfc_prepare_cmd(chip);
1849 marvell_nfc_send_cmd(chip, &nfc_op);
1851 if (!nfc_op.data_instr) {
1852 ret = marvell_nfc_wait_cmdd(chip);
1853 cond_delay(nfc_op.cle_ale_delay_ns);
1857 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1858 "RDDREQ/WRDREQ while draining raw data");
1862 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1863 ret = marvell_nfc_wait_cmdd(chip);
1868 * NDCR ND_RUN bit should be cleared automatically at the end of each
1869 * operation but experience shows that the behavior is buggy when it
1870 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1872 if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1873 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1875 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1882 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1883 const struct nand_subop *subop)
1885 struct marvell_nfc_op nfc_op;
1888 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1890 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1891 cond_delay(nfc_op.rdy_delay_ns);
1896 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1897 const struct nand_subop *subop)
1899 struct marvell_nfc_op nfc_op;
1902 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1903 nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1904 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1906 ret = marvell_nfc_prepare_cmd(chip);
1910 marvell_nfc_send_cmd(chip, &nfc_op);
1911 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1912 "RDDREQ while reading ID");
1916 cond_delay(nfc_op.cle_ale_delay_ns);
1918 if (nfc_op.rdy_timeout_ms) {
1919 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1924 cond_delay(nfc_op.rdy_delay_ns);
1926 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1927 ret = marvell_nfc_wait_cmdd(chip);
1931 cond_delay(nfc_op.data_delay_ns);
1936 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
1937 const struct nand_subop *subop)
1939 struct marvell_nfc_op nfc_op;
1942 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1943 nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1944 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
1946 ret = marvell_nfc_prepare_cmd(chip);
1950 marvell_nfc_send_cmd(chip, &nfc_op);
1951 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1952 "RDDREQ while reading status");
1956 cond_delay(nfc_op.cle_ale_delay_ns);
1958 if (nfc_op.rdy_timeout_ms) {
1959 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1964 cond_delay(nfc_op.rdy_delay_ns);
1966 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1967 ret = marvell_nfc_wait_cmdd(chip);
1971 cond_delay(nfc_op.data_delay_ns);
1976 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
1977 const struct nand_subop *subop)
1979 struct marvell_nfc_op nfc_op;
1982 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1983 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
1985 ret = marvell_nfc_prepare_cmd(chip);
1989 marvell_nfc_send_cmd(chip, &nfc_op);
1990 ret = marvell_nfc_wait_cmdd(chip);
1994 cond_delay(nfc_op.cle_ale_delay_ns);
1996 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2000 cond_delay(nfc_op.rdy_delay_ns);
2005 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2006 const struct nand_subop *subop)
2008 struct marvell_nfc_op nfc_op;
2011 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2012 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2014 ret = marvell_nfc_prepare_cmd(chip);
2018 marvell_nfc_send_cmd(chip, &nfc_op);
2019 ret = marvell_nfc_wait_cmdd(chip);
2023 cond_delay(nfc_op.cle_ale_delay_ns);
2025 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2029 cond_delay(nfc_op.rdy_delay_ns);
2034 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2035 /* Monolithic reads/writes */
2036 NAND_OP_PARSER_PATTERN(
2037 marvell_nfc_monolithic_access_exec,
2038 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2039 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2040 NAND_OP_PARSER_PAT_CMD_ELEM(true),
2041 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2042 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2043 NAND_OP_PARSER_PATTERN(
2044 marvell_nfc_monolithic_access_exec,
2045 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2046 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2047 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2048 NAND_OP_PARSER_PAT_CMD_ELEM(true),
2049 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2050 /* Naked commands */
2051 NAND_OP_PARSER_PATTERN(
2052 marvell_nfc_naked_access_exec,
2053 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2054 NAND_OP_PARSER_PATTERN(
2055 marvell_nfc_naked_access_exec,
2056 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2057 NAND_OP_PARSER_PATTERN(
2058 marvell_nfc_naked_access_exec,
2059 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2060 NAND_OP_PARSER_PATTERN(
2061 marvell_nfc_naked_access_exec,
2062 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2063 NAND_OP_PARSER_PATTERN(
2064 marvell_nfc_naked_waitrdy_exec,
2065 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2068 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2069 /* Naked commands not supported, use a function for each pattern */
2070 NAND_OP_PARSER_PATTERN(
2071 marvell_nfc_read_id_type_exec,
2072 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2073 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2074 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2075 NAND_OP_PARSER_PATTERN(
2076 marvell_nfc_erase_cmd_type_exec,
2077 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2078 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2079 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2080 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2081 NAND_OP_PARSER_PATTERN(
2082 marvell_nfc_read_status_exec,
2083 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2084 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2085 NAND_OP_PARSER_PATTERN(
2086 marvell_nfc_reset_cmd_type_exec,
2087 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2088 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2089 NAND_OP_PARSER_PATTERN(
2090 marvell_nfc_naked_waitrdy_exec,
2091 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2094 static int marvell_nfc_exec_op(struct nand_chip *chip,
2095 const struct nand_operation *op,
2098 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2100 if (nfc->caps->is_nfcv2)
2101 return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2104 return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2109 * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2112 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2113 struct mtd_oob_region *oobregion)
2115 struct nand_chip *chip = mtd_to_nand(mtd);
2116 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2121 oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2123 oobregion->offset = mtd->oobsize - oobregion->length;
2128 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2129 struct mtd_oob_region *oobregion)
2131 struct nand_chip *chip = mtd_to_nand(mtd);
2132 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2138 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2139 * 4KB page / 4bit BCH combination.
2141 if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2142 oobregion->offset = 6;
2144 oobregion->offset = 2;
2146 oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2147 lt->last_spare_bytes - oobregion->offset;
2152 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2153 .ecc = marvell_nand_ooblayout_ecc,
2154 .free = marvell_nand_ooblayout_free,
2157 static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
2158 struct nand_ecc_ctrl *ecc)
2160 struct nand_chip *chip = mtd_to_nand(mtd);
2161 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2162 const struct marvell_hw_ecc_layout *l;
2165 if (!nfc->caps->is_nfcv2 &&
2166 (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2168 "NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2169 mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2173 to_marvell_nand(chip)->layout = NULL;
2174 for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2175 l = &marvell_nfc_layouts[i];
2176 if (mtd->writesize == l->writesize &&
2177 ecc->size == l->chunk && ecc->strength == l->strength) {
2178 to_marvell_nand(chip)->layout = l;
2183 if (!to_marvell_nand(chip)->layout ||
2184 (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2186 "ECC strength %d at page size %d is not supported\n",
2187 ecc->strength, mtd->writesize);
2191 /* Special care for the layout 2k/8-bit/512B */
2192 if (l->writesize == 2048 && l->strength == 8) {
2193 if (mtd->oobsize < 128) {
2194 dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2197 chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2201 mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2202 ecc->steps = l->nchunks;
2203 ecc->size = l->data_bytes;
2205 if (ecc->strength == 1) {
2206 chip->ecc.algo = NAND_ECC_HAMMING;
2207 ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2208 ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2209 ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2210 ecc->read_oob = ecc->read_oob_raw;
2211 ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2212 ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2213 ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2214 ecc->write_oob = ecc->write_oob_raw;
2216 chip->ecc.algo = NAND_ECC_BCH;
2218 ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2219 ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2220 ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2221 ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2222 ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2223 ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2224 ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2225 ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2231 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2232 struct nand_ecc_ctrl *ecc)
2234 struct nand_chip *chip = mtd_to_nand(mtd);
2235 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2238 if (ecc->mode != NAND_ECC_NONE && (!ecc->size || !ecc->strength)) {
2239 if (chip->ecc_step_ds && chip->ecc_strength_ds) {
2240 ecc->size = chip->ecc_step_ds;
2241 ecc->strength = chip->ecc_strength_ds;
2244 "No minimum ECC strength, using 1b/512B\n");
2250 switch (ecc->mode) {
2252 ret = marvell_nand_hw_ecc_ctrl_init(mtd, ecc);
2258 case NAND_ECC_ON_DIE:
2259 if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2260 mtd->writesize != SZ_2K) {
2261 dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2273 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2274 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2276 static struct nand_bbt_descr bbt_main_descr = {
2277 .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2278 NAND_BBT_2BIT | NAND_BBT_VERSION,
2282 .maxblocks = 8, /* Last 8 blocks in each chip */
2283 .pattern = bbt_pattern
2286 static struct nand_bbt_descr bbt_mirror_descr = {
2287 .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2288 NAND_BBT_2BIT | NAND_BBT_VERSION,
2292 .maxblocks = 8, /* Last 8 blocks in each chip */
2293 .pattern = bbt_mirror_pattern
2296 static int marvell_nfc_setup_data_interface(struct nand_chip *chip, int chipnr,
2297 const struct nand_data_interface
2300 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2301 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2302 unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2303 const struct nand_sdr_timings *sdr;
2304 struct marvell_nfc_timings nfc_tmg;
2307 sdr = nand_get_sdr_timings(conf);
2309 return PTR_ERR(sdr);
2312 * SDR timings are given in pico-seconds while NFC timings must be
2313 * expressed in NAND controller clock cycles, which is half of the
2314 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2315 * This is not written anywhere in the datasheet but was observed
2316 * with an oscilloscope.
2318 * NFC datasheet gives equations from which thoses calculations
2319 * are derived, they tend to be slightly more restrictives than the
2320 * given core timings and may improve the overall speed.
2322 nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2323 nfc_tmg.tRH = nfc_tmg.tRP;
2324 nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2325 nfc_tmg.tWH = nfc_tmg.tWP;
2326 nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2327 nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2328 nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2330 * Read delay is the time of propagation from SoC pins to NFC internal
2331 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2332 * EDO mode, an additional delay of tRH must be taken into account so
2333 * the data is sampled on the falling edge instead of the rising edge.
2335 read_delay = sdr->tRC_min >= 30000 ?
2336 MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2338 nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2340 * tWHR and tRHW are supposed to be read to write delays (and vice
2341 * versa) but in some cases, ie. when doing a change column, they must
2342 * be greater than that to be sure tCCS delay is respected.
2344 nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2346 nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2350 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2351 * NFCv1: No WAIT_MODE, tR must be maximal.
2353 if (nfc->caps->is_nfcv2) {
2354 nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2356 nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2358 if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2359 nfc_tmg.tR = nfc_tmg.tCH - 3;
2367 marvell_nand->ndtr0 =
2368 NDTR0_TRP(nfc_tmg.tRP) |
2369 NDTR0_TRH(nfc_tmg.tRH) |
2370 NDTR0_ETRP(nfc_tmg.tRP) |
2371 NDTR0_TWP(nfc_tmg.tWP) |
2372 NDTR0_TWH(nfc_tmg.tWH) |
2373 NDTR0_TCS(nfc_tmg.tCS) |
2374 NDTR0_TCH(nfc_tmg.tCH);
2376 marvell_nand->ndtr1 =
2377 NDTR1_TAR(nfc_tmg.tAR) |
2378 NDTR1_TWHR(nfc_tmg.tWHR) |
2379 NDTR1_TR(nfc_tmg.tR);
2381 if (nfc->caps->is_nfcv2) {
2382 marvell_nand->ndtr0 |=
2383 NDTR0_RD_CNT_DEL(read_delay) |
2385 NDTR0_TADL(nfc_tmg.tADL);
2387 marvell_nand->ndtr1 |=
2388 NDTR1_TRHW(nfc_tmg.tRHW) |
2395 static int marvell_nand_attach_chip(struct nand_chip *chip)
2397 struct mtd_info *mtd = nand_to_mtd(chip);
2398 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2399 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2400 struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2403 if (pdata && pdata->flash_bbt)
2404 chip->bbt_options |= NAND_BBT_USE_FLASH;
2406 if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2408 * We'll use a bad block table stored in-flash and don't
2409 * allow writing the bad block marker to the flash.
2411 chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2412 chip->bbt_td = &bbt_main_descr;
2413 chip->bbt_md = &bbt_mirror_descr;
2416 /* Save the chip-specific fields of NDCR */
2417 marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2418 if (chip->options & NAND_BUSWIDTH_16)
2419 marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2422 * On small page NANDs, only one cycle is needed to pass the
2425 if (mtd->writesize <= 512) {
2426 marvell_nand->addr_cyc = 1;
2428 marvell_nand->addr_cyc = 2;
2429 marvell_nand->ndcr |= NDCR_RA_START;
2433 * Now add the number of cycles needed to pass the row
2436 * Addressing a chip using CS 2 or 3 should also need the third row
2437 * cycle but due to inconsistance in the documentation and lack of
2438 * hardware to test this situation, this case is not supported.
2440 if (chip->options & NAND_ROW_ADDR_3)
2441 marvell_nand->addr_cyc += 3;
2443 marvell_nand->addr_cyc += 2;
2446 chip->ecc.size = pdata->ecc_step_size;
2447 chip->ecc.strength = pdata->ecc_strength;
2450 ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2452 dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2456 if (chip->ecc.mode == NAND_ECC_HW) {
2458 * Subpage write not available with hardware ECC, prohibit also
2459 * subpage read as in userspace subpage access would still be
2460 * allowed and subpage write, if used, would lead to numerous
2461 * uncorrectable ECC errors.
2463 chip->options |= NAND_NO_SUBPAGE_WRITE;
2466 if (pdata || nfc->caps->legacy_of_bindings) {
2468 * We keep the MTD name unchanged to avoid breaking platforms
2469 * where the MTD cmdline parser is used and the bootloader
2470 * has not been updated to use the new naming scheme.
2472 mtd->name = "pxa3xx_nand-0";
2473 } else if (!mtd->name) {
2475 * If the new bindings are used and the bootloader has not been
2476 * updated to pass a new mtdparts parameter on the cmdline, you
2477 * should define the following property in your NAND node, ie:
2479 * label = "main-storage";
2481 * This way, mtd->name will be set by the core when
2482 * nand_set_flash_node() is called.
2484 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2485 "%s:nand.%d", dev_name(nfc->dev),
2486 marvell_nand->sels[0].cs);
2488 dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2496 static const struct nand_controller_ops marvell_nand_controller_ops = {
2497 .attach_chip = marvell_nand_attach_chip,
2500 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2501 struct device_node *np)
2503 struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2504 struct marvell_nand_chip *marvell_nand;
2505 struct mtd_info *mtd;
2506 struct nand_chip *chip;
2511 * The legacy "num-cs" property indicates the number of CS on the only
2512 * chip connected to the controller (legacy bindings does not support
2513 * more than one chip). The CS and RB pins are always the #0.
2515 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2516 * properties must be filled. For each chip, expressed as a subnode,
2517 * "reg" points to the CS lines and "nand-rb" to the RB line.
2519 if (pdata || nfc->caps->legacy_of_bindings) {
2522 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2524 dev_err(dev, "missing/invalid reg property\n");
2529 /* Alloc the nand chip structure */
2530 marvell_nand = devm_kzalloc(dev, sizeof(*marvell_nand) +
2532 sizeof(struct marvell_nand_chip_sel)),
2534 if (!marvell_nand) {
2535 dev_err(dev, "could not allocate chip structure\n");
2539 marvell_nand->nsels = nsels;
2540 marvell_nand->selected_die = -1;
2542 for (i = 0; i < nsels; i++) {
2543 if (pdata || nfc->caps->legacy_of_bindings) {
2545 * Legacy bindings use the CS lines in natural
2550 /* Retrieve CS id */
2551 ret = of_property_read_u32_index(np, "reg", i, &cs);
2553 dev_err(dev, "could not retrieve reg property: %d\n",
2559 if (cs >= nfc->caps->max_cs_nb) {
2560 dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2561 cs, nfc->caps->max_cs_nb);
2565 if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2566 dev_err(dev, "CS %d already assigned\n", cs);
2571 * The cs variable represents the chip select id, which must be
2572 * converted in bit fields for NDCB0 and NDCB2 to select the
2573 * right chip. Unfortunately, due to a lack of information on
2574 * the subject and incoherent documentation, the user should not
2575 * use CS1 and CS3 at all as asserting them is not supported in
2576 * a reliable way (due to multiplexing inside ADDR5 field).
2578 marvell_nand->sels[i].cs = cs;
2582 marvell_nand->sels[i].ndcb0_csel = 0;
2586 marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2592 /* Retrieve RB id */
2593 if (pdata || nfc->caps->legacy_of_bindings) {
2594 /* Legacy bindings always use RB #0 */
2597 ret = of_property_read_u32_index(np, "nand-rb", i,
2601 "could not retrieve RB property: %d\n",
2607 if (rb >= nfc->caps->max_rb_nb) {
2608 dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2609 rb, nfc->caps->max_rb_nb);
2613 marvell_nand->sels[i].rb = rb;
2616 chip = &marvell_nand->chip;
2617 chip->controller = &nfc->controller;
2618 nand_set_flash_node(chip, np);
2620 chip->exec_op = marvell_nfc_exec_op;
2621 chip->select_chip = marvell_nfc_select_chip;
2622 if (!of_property_read_bool(np, "marvell,nand-keep-config"))
2623 chip->setup_data_interface = marvell_nfc_setup_data_interface;
2625 mtd = nand_to_mtd(chip);
2626 mtd->dev.parent = dev;
2629 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2630 * in the DT node, this entry will be overwritten in nand_scan_ident().
2632 chip->ecc.mode = NAND_ECC_HW;
2635 * Save a reference value for timing registers before
2636 * ->setup_data_interface() is called.
2638 marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2639 marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2641 chip->options |= NAND_BUSWIDTH_AUTO;
2643 ret = nand_scan(chip, marvell_nand->nsels);
2645 dev_err(dev, "could not scan the nand chip\n");
2650 /* Legacy bindings support only one chip */
2651 ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2653 ret = mtd_device_register(mtd, NULL, 0);
2655 dev_err(dev, "failed to register mtd device: %d\n", ret);
2660 list_add_tail(&marvell_nand->node, &nfc->chips);
2665 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2667 struct device_node *np = dev->of_node;
2668 struct device_node *nand_np;
2669 int max_cs = nfc->caps->max_cs_nb;
2676 nchips = of_get_child_count(np);
2678 if (nchips > max_cs) {
2679 dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2685 * Legacy bindings do not use child nodes to exhibit NAND chip
2686 * properties and layout. Instead, NAND properties are mixed with the
2687 * controller ones, and partitions are defined as direct subnodes of the
2688 * NAND controller node.
2690 if (nfc->caps->legacy_of_bindings) {
2691 ret = marvell_nand_chip_init(dev, nfc, np);
2695 for_each_child_of_node(np, nand_np) {
2696 ret = marvell_nand_chip_init(dev, nfc, nand_np);
2698 of_node_put(nand_np);
2706 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2708 struct marvell_nand_chip *entry, *temp;
2710 list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2711 nand_release(&entry->chip);
2712 list_del(&entry->node);
2716 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2718 struct platform_device *pdev = container_of(nfc->dev,
2719 struct platform_device,
2721 struct dma_slave_config config = {};
2725 if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2727 "DMA not enabled in configuration\n");
2731 ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2735 nfc->dma_chan = dma_request_slave_channel(nfc->dev, "data");
2736 if (!nfc->dma_chan) {
2738 "Unable to request data DMA channel\n");
2742 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2746 config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2747 config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2748 config.src_addr = r->start + NDDB;
2749 config.dst_addr = r->start + NDDB;
2750 config.src_maxburst = 32;
2751 config.dst_maxburst = 32;
2752 ret = dmaengine_slave_config(nfc->dma_chan, &config);
2754 dev_err(nfc->dev, "Failed to configure DMA channel\n");
2759 * DMA must act on length multiple of 32 and this length may be
2760 * bigger than the destination buffer. Use this buffer instead
2761 * for DMA transfers and then copy the desired amount of data to
2762 * the provided buffer.
2764 nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2768 nfc->use_dma = true;
2773 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2776 * ECC operations and interruptions are only enabled when specifically
2777 * needed. ECC shall not be activated in the early stages (fails probe).
2778 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2779 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2780 * offset in the read page and this will fail the protection.
2782 writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2783 NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2784 writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2785 writel_relaxed(0, nfc->regs + NDECCCTRL);
2788 static int marvell_nfc_init(struct marvell_nfc *nfc)
2790 struct device_node *np = nfc->dev->of_node;
2793 * Some SoCs like A7k/A8k need to enable manually the NAND
2794 * controller, gated clocks and reset bits to avoid being bootloader
2795 * dependent. This is done through the use of the System Functions
2798 if (nfc->caps->need_system_controller) {
2799 struct regmap *sysctrl_base =
2800 syscon_regmap_lookup_by_phandle(np,
2801 "marvell,system-controller");
2803 if (IS_ERR(sysctrl_base))
2804 return PTR_ERR(sysctrl_base);
2806 regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2807 GENCONF_SOC_DEVICE_MUX_NFC_EN |
2808 GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2809 GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2810 GENCONF_SOC_DEVICE_MUX_NFC_INT_EN);
2812 regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2813 GENCONF_CLK_GATING_CTRL_ND_GATE,
2814 GENCONF_CLK_GATING_CTRL_ND_GATE);
2816 regmap_update_bits(sysctrl_base, GENCONF_ND_CLK_CTRL,
2817 GENCONF_ND_CLK_CTRL_EN,
2818 GENCONF_ND_CLK_CTRL_EN);
2821 /* Configure the DMA if appropriate */
2822 if (!nfc->caps->is_nfcv2)
2823 marvell_nfc_init_dma(nfc);
2825 marvell_nfc_reset(nfc);
2830 static int marvell_nfc_probe(struct platform_device *pdev)
2832 struct device *dev = &pdev->dev;
2834 struct marvell_nfc *nfc;
2838 nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2844 nand_controller_init(&nfc->controller);
2845 nfc->controller.ops = &marvell_nand_controller_ops;
2846 INIT_LIST_HEAD(&nfc->chips);
2848 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2849 nfc->regs = devm_ioremap_resource(dev, r);
2850 if (IS_ERR(nfc->regs))
2851 return PTR_ERR(nfc->regs);
2853 irq = platform_get_irq(pdev, 0);
2855 dev_err(dev, "failed to retrieve irq\n");
2859 nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2861 /* Managed the legacy case (when the first clock was not named) */
2862 if (nfc->core_clk == ERR_PTR(-ENOENT))
2863 nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2865 if (IS_ERR(nfc->core_clk))
2866 return PTR_ERR(nfc->core_clk);
2868 ret = clk_prepare_enable(nfc->core_clk);
2872 nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2873 if (IS_ERR(nfc->reg_clk)) {
2874 if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2875 ret = PTR_ERR(nfc->reg_clk);
2876 goto unprepare_core_clk;
2879 nfc->reg_clk = NULL;
2882 ret = clk_prepare_enable(nfc->reg_clk);
2884 goto unprepare_core_clk;
2886 marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2887 marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2888 ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2889 0, "marvell-nfc", nfc);
2891 goto unprepare_reg_clk;
2893 /* Get NAND controller capabilities */
2895 nfc->caps = (void *)pdev->id_entry->driver_data;
2897 nfc->caps = of_device_get_match_data(&pdev->dev);
2900 dev_err(dev, "Could not retrieve NFC caps\n");
2902 goto unprepare_reg_clk;
2905 /* Init the controller and then probe the chips */
2906 ret = marvell_nfc_init(nfc);
2908 goto unprepare_reg_clk;
2910 platform_set_drvdata(pdev, nfc);
2912 ret = marvell_nand_chips_init(dev, nfc);
2914 goto unprepare_reg_clk;
2919 clk_disable_unprepare(nfc->reg_clk);
2921 clk_disable_unprepare(nfc->core_clk);
2926 static int marvell_nfc_remove(struct platform_device *pdev)
2928 struct marvell_nfc *nfc = platform_get_drvdata(pdev);
2930 marvell_nand_chips_cleanup(nfc);
2933 dmaengine_terminate_all(nfc->dma_chan);
2934 dma_release_channel(nfc->dma_chan);
2937 clk_disable_unprepare(nfc->reg_clk);
2938 clk_disable_unprepare(nfc->core_clk);
2943 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
2945 struct marvell_nfc *nfc = dev_get_drvdata(dev);
2946 struct marvell_nand_chip *chip;
2948 list_for_each_entry(chip, &nfc->chips, node)
2949 marvell_nfc_wait_ndrun(&chip->chip);
2951 clk_disable_unprepare(nfc->reg_clk);
2952 clk_disable_unprepare(nfc->core_clk);
2957 static int __maybe_unused marvell_nfc_resume(struct device *dev)
2959 struct marvell_nfc *nfc = dev_get_drvdata(dev);
2962 ret = clk_prepare_enable(nfc->core_clk);
2966 ret = clk_prepare_enable(nfc->reg_clk);
2971 * Reset nfc->selected_chip so the next command will cause the timing
2972 * registers to be restored in marvell_nfc_select_chip().
2974 nfc->selected_chip = NULL;
2976 /* Reset registers that have lost their contents */
2977 marvell_nfc_reset(nfc);
2982 static const struct dev_pm_ops marvell_nfc_pm_ops = {
2983 SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
2986 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
2989 .need_system_controller = true,
2993 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
2999 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3005 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3008 .need_system_controller = true,
3009 .legacy_of_bindings = true,
3013 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3016 .legacy_of_bindings = true,
3020 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3023 .legacy_of_bindings = true,
3027 static const struct platform_device_id marvell_nfc_platform_ids[] = {
3029 .name = "pxa3xx-nand",
3030 .driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3034 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3036 static const struct of_device_id marvell_nfc_of_ids[] = {
3038 .compatible = "marvell,armada-8k-nand-controller",
3039 .data = &marvell_armada_8k_nfc_caps,
3042 .compatible = "marvell,armada370-nand-controller",
3043 .data = &marvell_armada370_nfc_caps,
3046 .compatible = "marvell,pxa3xx-nand-controller",
3047 .data = &marvell_pxa3xx_nfc_caps,
3049 /* Support for old/deprecated bindings: */
3051 .compatible = "marvell,armada-8k-nand",
3052 .data = &marvell_armada_8k_nfc_legacy_caps,
3055 .compatible = "marvell,armada370-nand",
3056 .data = &marvell_armada370_nfc_legacy_caps,
3059 .compatible = "marvell,pxa3xx-nand",
3060 .data = &marvell_pxa3xx_nfc_legacy_caps,
3064 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3066 static struct platform_driver marvell_nfc_driver = {
3068 .name = "marvell-nfc",
3069 .of_match_table = marvell_nfc_of_ids,
3070 .pm = &marvell_nfc_pm_ops,
3072 .id_table = marvell_nfc_platform_ids,
3073 .probe = marvell_nfc_probe,
3074 .remove = marvell_nfc_remove,
3076 module_platform_driver(marvell_nfc_driver);
3078 MODULE_LICENSE("GPL");
3079 MODULE_DESCRIPTION("Marvell NAND controller driver");