1 // SPDX-License-Identifier: GPL-2.0+
3 * Freescale GPMI NAND Flash Driver
5 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc.
6 * Copyright (C) 2008 Embedded Alley Solutions, Inc.
9 #include <linux/delay.h>
10 #include <linux/slab.h>
11 #include <linux/sched/task_stack.h>
12 #include <linux/interrupt.h>
13 #include <linux/module.h>
14 #include <linux/mtd/partitions.h>
16 #include <linux/of_device.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/dma/mxs-dma.h>
19 #include "gpmi-nand.h"
20 #include "gpmi-regs.h"
23 /* Resource names for the GPMI NAND driver. */
24 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME "gpmi-nand"
25 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME "bch"
26 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME "bch"
28 /* Converts time to clock cycles */
29 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period)
31 #define MXS_SET_ADDR 0x4
32 #define MXS_CLR_ADDR 0x8
34 * Clear the bit and poll it cleared. This is usually called with
35 * a reset address and mask being either SFTRST(bit 31) or CLKGATE
38 static int clear_poll_bit(void __iomem *addr, u32 mask)
43 writel(mask, addr + MXS_CLR_ADDR);
46 * SFTRST needs 3 GPMI clocks to settle, the reference manual
47 * recommends to wait 1us.
51 /* poll the bit becoming clear */
52 while ((readl(addr) & mask) && --timeout)
58 #define MODULE_CLKGATE (1 << 30)
59 #define MODULE_SFTRST (1 << 31)
61 * The current mxs_reset_block() will do two things:
62 * [1] enable the module.
63 * [2] reset the module.
65 * In most of the cases, it's ok.
66 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847).
67 * If you try to soft reset the BCH block, it becomes unusable until
68 * the next hard reset. This case occurs in the NAND boot mode. When the board
69 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself.
70 * So If the driver tries to reset the BCH again, the BCH will not work anymore.
71 * You will see a DMA timeout in this case. The bug has been fixed
72 * in the following chips, such as MX28.
74 * To avoid this bug, just add a new parameter `just_enable` for
75 * the mxs_reset_block(), and rewrite it here.
77 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable)
82 /* clear and poll SFTRST */
83 ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
88 writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR);
91 /* set SFTRST to reset the block */
92 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR);
95 /* poll CLKGATE becoming set */
96 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout)
98 if (unlikely(!timeout))
102 /* clear and poll SFTRST */
103 ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
107 /* clear and poll CLKGATE */
108 ret = clear_poll_bit(reset_addr, MODULE_CLKGATE);
115 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);
119 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v)
125 for (i = 0; i < GPMI_CLK_MAX; i++) {
126 clk = this->resources.clock[i];
131 ret = clk_prepare_enable(clk);
135 clk_disable_unprepare(clk);
142 clk_disable_unprepare(this->resources.clock[i - 1]);
146 static int gpmi_init(struct gpmi_nand_data *this)
148 struct resources *r = &this->resources;
151 ret = pm_runtime_resume_and_get(this->dev);
155 ret = gpmi_reset_block(r->gpmi_regs, false);
160 * Reset BCH here, too. We got failures otherwise :(
161 * See later BCH reset for explanation of MX23 and MX28 handling
163 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));
167 /* Choose NAND mode. */
168 writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR);
170 /* Set the IRQ polarity. */
171 writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY,
172 r->gpmi_regs + HW_GPMI_CTRL1_SET);
174 /* Disable Write-Protection. */
175 writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET);
177 /* Select BCH ECC. */
178 writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET);
181 * Decouple the chip select from dma channel. We use dma0 for all
182 * the chips, force all NAND RDY_BUSY inputs to be sourced from
185 writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY,
186 r->gpmi_regs + HW_GPMI_CTRL1_SET);
189 pm_runtime_mark_last_busy(this->dev);
190 pm_runtime_put_autosuspend(this->dev);
194 /* This function is very useful. It is called only when the bug occur. */
195 static void gpmi_dump_info(struct gpmi_nand_data *this)
197 struct resources *r = &this->resources;
198 struct bch_geometry *geo = &this->bch_geometry;
202 dev_err(this->dev, "Show GPMI registers :\n");
203 for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) {
204 reg = readl(r->gpmi_regs + i * 0x10);
205 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
208 /* start to print out the BCH info */
209 dev_err(this->dev, "Show BCH registers :\n");
210 for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) {
211 reg = readl(r->bch_regs + i * 0x10);
212 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
214 dev_err(this->dev, "BCH Geometry :\n"
216 "ECC Strength : %u\n"
217 "Page Size in Bytes : %u\n"
218 "Metadata Size in Bytes : %u\n"
219 "ECC0 Chunk Size in Bytes: %u\n"
220 "ECCn Chunk Size in Bytes: %u\n"
221 "ECC Chunk Count : %u\n"
222 "Payload Size in Bytes : %u\n"
223 "Auxiliary Size in Bytes: %u\n"
224 "Auxiliary Status Offset: %u\n"
225 "Block Mark Byte Offset : %u\n"
226 "Block Mark Bit Offset : %u\n",
231 geo->ecc0_chunk_size,
232 geo->eccn_chunk_size,
233 geo->ecc_chunk_count,
236 geo->auxiliary_status_offset,
237 geo->block_mark_byte_offset,
238 geo->block_mark_bit_offset);
241 static bool gpmi_check_ecc(struct gpmi_nand_data *this)
243 struct nand_chip *chip = &this->nand;
244 struct bch_geometry *geo = &this->bch_geometry;
245 struct nand_device *nand = &chip->base;
246 struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
248 conf->step_size = geo->eccn_chunk_size;
249 conf->strength = geo->ecc_strength;
251 /* Do the sanity check. */
252 if (GPMI_IS_MXS(this)) {
253 /* The mx23/mx28 only support the GF13. */
254 if (geo->gf_len == 14)
258 if (geo->ecc_strength > this->devdata->bch_max_ecc_strength)
261 if (!nand_ecc_is_strong_enough(nand))
267 /* check if bbm locates in data chunk rather than ecc chunk */
268 static bool bbm_in_data_chunk(struct gpmi_nand_data *this,
269 unsigned int *chunk_num)
271 struct bch_geometry *geo = &this->bch_geometry;
272 struct nand_chip *chip = &this->nand;
273 struct mtd_info *mtd = nand_to_mtd(chip);
276 if (geo->ecc0_chunk_size != geo->eccn_chunk_size) {
278 "The size of ecc0_chunk must equal to eccn_chunk\n");
282 i = (mtd->writesize * 8 - geo->metadata_size * 8) /
283 (geo->gf_len * geo->ecc_strength +
284 geo->eccn_chunk_size * 8);
286 j = (mtd->writesize * 8 - geo->metadata_size * 8) -
287 (geo->gf_len * geo->ecc_strength +
288 geo->eccn_chunk_size * 8) * i;
290 if (j < geo->eccn_chunk_size * 8) {
292 dev_dbg(this->dev, "Set ecc to %d and bbm in chunk %d\n",
293 geo->ecc_strength, *chunk_num);
301 * If we can get the ECC information from the nand chip, we do not
302 * need to calculate them ourselves.
304 * We may have available oob space in this case.
306 static int set_geometry_by_ecc_info(struct gpmi_nand_data *this,
307 unsigned int ecc_strength,
308 unsigned int ecc_step)
310 struct bch_geometry *geo = &this->bch_geometry;
311 struct nand_chip *chip = &this->nand;
312 struct mtd_info *mtd = nand_to_mtd(chip);
313 unsigned int block_mark_bit_offset;
324 "unsupported nand chip. ecc bits : %d, ecc size : %d\n",
325 nanddev_get_ecc_requirements(&chip->base)->strength,
326 nanddev_get_ecc_requirements(&chip->base)->step_size);
329 geo->ecc0_chunk_size = ecc_step;
330 geo->eccn_chunk_size = ecc_step;
331 geo->ecc_strength = round_up(ecc_strength, 2);
332 if (!gpmi_check_ecc(this))
335 /* Keep the C >= O */
336 if (geo->eccn_chunk_size < mtd->oobsize) {
338 "unsupported nand chip. ecc size: %d, oob size : %d\n",
339 ecc_step, mtd->oobsize);
343 /* The default value, see comment in the legacy_set_geometry(). */
344 geo->metadata_size = 10;
346 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;
349 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below:
352 * |<----------------------------------------------------->|
356 * |<-------------------------------------------->| D | | O' |
359 * +---+----------+-+----------+-+----------+-+----------+-+-----+
360 * | M | data |E| data |E| data |E| data |E| |
361 * +---+----------+-+----------+-+----------+-+----------+-+-----+
367 * P : the page size for BCH module.
368 * E : The ECC strength.
369 * G : the length of Galois Field.
370 * N : The chunk count of per page.
371 * M : the metasize of per page.
372 * C : the ecc chunk size, aka the "data" above.
373 * P': the nand chip's page size.
374 * O : the nand chip's oob size.
377 * The formula for P is :
380 * P = ------------ + P' + M
383 * The position of block mark moves forward in the ECC-based view
384 * of page, and the delta is:
387 * D = (---------------- + M)
390 * Please see the comment in legacy_set_geometry().
391 * With the condition C >= O , we still can get same result.
392 * So the bit position of the physical block mark within the ECC-based
393 * view of the page is :
396 geo->page_size = mtd->writesize + geo->metadata_size +
397 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
399 geo->payload_size = mtd->writesize;
401 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
402 geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
403 + ALIGN(geo->ecc_chunk_count, 4);
405 if (!this->swap_block_mark)
409 block_mark_bit_offset = mtd->writesize * 8 -
410 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
411 + geo->metadata_size * 8);
413 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
414 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
419 * Calculate the ECC strength by hand:
420 * E : The ECC strength.
421 * G : the length of Galois Field.
422 * N : The chunk count of per page.
423 * O : the oobsize of the NAND chip.
424 * M : the metasize of per page.
428 * ------------ <= (O - M)
436 static inline int get_ecc_strength(struct gpmi_nand_data *this)
438 struct bch_geometry *geo = &this->bch_geometry;
439 struct mtd_info *mtd = nand_to_mtd(&this->nand);
442 ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8)
443 / (geo->gf_len * geo->ecc_chunk_count);
445 /* We need the minor even number. */
446 return round_down(ecc_strength, 2);
449 static int set_geometry_for_large_oob(struct gpmi_nand_data *this)
451 struct bch_geometry *geo = &this->bch_geometry;
452 struct nand_chip *chip = &this->nand;
453 struct mtd_info *mtd = nand_to_mtd(chip);
454 const struct nand_ecc_props *requirements =
455 nanddev_get_ecc_requirements(&chip->base);
456 unsigned int block_mark_bit_offset;
457 unsigned int max_ecc;
458 unsigned int bbm_chunk;
461 /* sanity check for the minimum ecc nand required */
462 if (!(requirements->strength > 0 &&
463 requirements->step_size > 0))
465 geo->ecc_strength = requirements->strength;
467 /* check if platform can support this nand */
468 if (!gpmi_check_ecc(this)) {
470 "unsupported NAND chip, minimum ecc required %d\n",
475 /* calculate the maximum ecc platform can support*/
476 geo->metadata_size = 10;
478 geo->ecc0_chunk_size = 1024;
479 geo->eccn_chunk_size = 1024;
480 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;
481 max_ecc = min(get_ecc_strength(this),
482 this->devdata->bch_max_ecc_strength);
485 * search a supported ecc strength that makes bbm
486 * located in data chunk
488 geo->ecc_strength = max_ecc;
489 while (!(geo->ecc_strength < requirements->strength)) {
490 if (bbm_in_data_chunk(this, &bbm_chunk))
492 geo->ecc_strength -= 2;
495 /* if none of them works, keep using the minimum ecc */
496 /* nand required but changing ecc page layout */
497 geo->ecc_strength = requirements->strength;
498 /* add extra ecc for meta data */
499 geo->ecc0_chunk_size = 0;
500 geo->ecc_chunk_count = (mtd->writesize / geo->eccn_chunk_size) + 1;
501 geo->ecc_for_meta = 1;
502 /* check if oob can afford this extra ecc chunk */
503 if (mtd->oobsize * 8 < geo->metadata_size * 8 +
504 geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) {
505 dev_err(this->dev, "unsupported NAND chip with new layout\n");
509 /* calculate in which chunk bbm located */
510 bbm_chunk = (mtd->writesize * 8 - geo->metadata_size * 8 -
511 geo->gf_len * geo->ecc_strength) /
512 (geo->gf_len * geo->ecc_strength +
513 geo->eccn_chunk_size * 8) + 1;
517 geo->page_size = mtd->writesize + geo->metadata_size +
518 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
519 geo->payload_size = mtd->writesize;
522 * The auxiliary buffer contains the metadata and the ECC status. The
523 * metadata is padded to the nearest 32-bit boundary. The ECC status
524 * contains one byte for every ECC chunk, and is also padded to the
525 * nearest 32-bit boundary.
527 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
528 geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
529 + ALIGN(geo->ecc_chunk_count, 4);
531 if (!this->swap_block_mark)
534 /* calculate the number of ecc chunk behind the bbm */
535 i = (mtd->writesize / geo->eccn_chunk_size) - bbm_chunk + 1;
537 block_mark_bit_offset = mtd->writesize * 8 -
538 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - i)
539 + geo->metadata_size * 8);
541 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
542 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
544 dev_dbg(this->dev, "BCH Geometry :\n"
546 "ECC Strength : %u\n"
547 "Page Size in Bytes : %u\n"
548 "Metadata Size in Bytes : %u\n"
549 "ECC0 Chunk Size in Bytes: %u\n"
550 "ECCn Chunk Size in Bytes: %u\n"
551 "ECC Chunk Count : %u\n"
552 "Payload Size in Bytes : %u\n"
553 "Auxiliary Size in Bytes: %u\n"
554 "Auxiliary Status Offset: %u\n"
555 "Block Mark Byte Offset : %u\n"
556 "Block Mark Bit Offset : %u\n"
557 "Block Mark in chunk : %u\n"
558 "Ecc for Meta data : %u\n",
563 geo->ecc0_chunk_size,
564 geo->eccn_chunk_size,
565 geo->ecc_chunk_count,
568 geo->auxiliary_status_offset,
569 geo->block_mark_byte_offset,
570 geo->block_mark_bit_offset,
577 static int legacy_set_geometry(struct gpmi_nand_data *this)
579 struct bch_geometry *geo = &this->bch_geometry;
580 struct mtd_info *mtd = nand_to_mtd(&this->nand);
581 unsigned int metadata_size;
582 unsigned int status_size;
583 unsigned int block_mark_bit_offset;
586 * The size of the metadata can be changed, though we set it to 10
587 * bytes now. But it can't be too large, because we have to save
588 * enough space for BCH.
590 geo->metadata_size = 10;
592 /* The default for the length of Galois Field. */
595 /* The default for chunk size. */
596 geo->ecc0_chunk_size = 512;
597 geo->eccn_chunk_size = 512;
598 while (geo->eccn_chunk_size < mtd->oobsize) {
599 geo->ecc0_chunk_size *= 2; /* keep C >= O */
600 geo->eccn_chunk_size *= 2; /* keep C >= O */
604 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;
606 /* We use the same ECC strength for all chunks. */
607 geo->ecc_strength = get_ecc_strength(this);
608 if (!gpmi_check_ecc(this)) {
610 "ecc strength: %d cannot be supported by the controller (%d)\n"
611 "try to use minimum ecc strength that NAND chip required\n",
613 this->devdata->bch_max_ecc_strength);
617 geo->page_size = mtd->writesize + geo->metadata_size +
618 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
619 geo->payload_size = mtd->writesize;
622 * The auxiliary buffer contains the metadata and the ECC status. The
623 * metadata is padded to the nearest 32-bit boundary. The ECC status
624 * contains one byte for every ECC chunk, and is also padded to the
625 * nearest 32-bit boundary.
627 metadata_size = ALIGN(geo->metadata_size, 4);
628 status_size = ALIGN(geo->ecc_chunk_count, 4);
630 geo->auxiliary_size = metadata_size + status_size;
631 geo->auxiliary_status_offset = metadata_size;
633 if (!this->swap_block_mark)
637 * We need to compute the byte and bit offsets of
638 * the physical block mark within the ECC-based view of the page.
640 * NAND chip with 2K page shows below:
646 * +---+----------+-+----------+-+----------+-+----------+-+
647 * | M | data |E| data |E| data |E| data |E|
648 * +---+----------+-+----------+-+----------+-+----------+-+
650 * The position of block mark moves forward in the ECC-based view
651 * of page, and the delta is:
654 * D = (---------------- + M)
657 * With the formula to compute the ECC strength, and the condition
658 * : C >= O (C is the ecc chunk size)
660 * It's easy to deduce to the following result:
662 * E * G (O - M) C - M C - M
663 * ----------- <= ------- <= -------- < ---------
669 * D = (---------------- + M) < C
672 * The above inequality means the position of block mark
673 * within the ECC-based view of the page is still in the data chunk,
674 * and it's NOT in the ECC bits of the chunk.
676 * Use the following to compute the bit position of the
677 * physical block mark within the ECC-based view of the page:
678 * (page_size - D) * 8
682 block_mark_bit_offset = mtd->writesize * 8 -
683 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
684 + geo->metadata_size * 8);
686 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
687 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
691 static int common_nfc_set_geometry(struct gpmi_nand_data *this)
693 struct nand_chip *chip = &this->nand;
694 struct mtd_info *mtd = nand_to_mtd(&this->nand);
695 const struct nand_ecc_props *requirements =
696 nanddev_get_ecc_requirements(&chip->base);
697 bool use_minimun_ecc;
700 use_minimun_ecc = of_property_read_bool(this->dev->of_node,
701 "fsl,use-minimum-ecc");
703 /* use legacy bch geometry settings by default*/
704 if ((!use_minimun_ecc && mtd->oobsize < 1024) ||
705 !(requirements->strength > 0 && requirements->step_size > 0)) {
706 dev_dbg(this->dev, "use legacy bch geometry\n");
707 err = legacy_set_geometry(this);
712 /* for large oob nand */
713 if (mtd->oobsize > 1024) {
714 dev_dbg(this->dev, "use large oob bch geometry\n");
715 err = set_geometry_for_large_oob(this);
720 /* otherwise use the minimum ecc nand chip required */
721 dev_dbg(this->dev, "use minimum ecc bch geometry\n");
722 err = set_geometry_by_ecc_info(this, requirements->strength,
723 requirements->step_size);
725 dev_err(this->dev, "none of the bch geometry setting works\n");
730 /* Configures the geometry for BCH. */
731 static int bch_set_geometry(struct gpmi_nand_data *this)
733 struct resources *r = &this->resources;
736 ret = common_nfc_set_geometry(this);
740 ret = pm_runtime_get_sync(this->dev);
742 pm_runtime_put_autosuspend(this->dev);
747 * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this
748 * chip, otherwise it will lock up. So we skip resetting BCH on the MX23.
751 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));
755 /* Set *all* chip selects to use layout 0. */
756 writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT);
760 pm_runtime_mark_last_busy(this->dev);
761 pm_runtime_put_autosuspend(this->dev);
767 * <1> Firstly, we should know what's the GPMI-clock means.
768 * The GPMI-clock is the internal clock in the gpmi nand controller.
769 * If you set 100MHz to gpmi nand controller, the GPMI-clock's period
770 * is 10ns. Mark the GPMI-clock's period as GPMI-clock-period.
772 * <2> Secondly, we should know what's the frequency on the nand chip pins.
773 * The frequency on the nand chip pins is derived from the GPMI-clock.
774 * We can get it from the following equation:
778 * F : the frequency on the nand chip pins.
779 * G : the GPMI clock, such as 100MHz.
780 * DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP
781 * DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD
783 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz,
784 * the nand EDO(extended Data Out) timing could be applied.
785 * The GPMI implements a feedback read strobe to sample the read data.
786 * The feedback read strobe can be delayed to support the nand EDO timing
787 * where the read strobe may deasserts before the read data is valid, and
788 * read data is valid for some time after read strobe.
790 * The following figure illustrates some aspects of a NAND Flash read:
797 * __ ___|__________________________________
801 * Read Data --------------< >---------
805 * FeedbackRDN ________ ____________
808 * D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY.
811 * <4> Now, we begin to describe how to compute the right RDN_DELAY.
813 * 4.1) From the aspect of the nand chip pins:
814 * Delay = (tREA + C - tRP) {1}
816 * tREA : the maximum read access time.
817 * C : a constant to adjust the delay. default is 4000ps.
818 * tRP : the read pulse width, which is exactly:
819 * tRP = (GPMI-clock-period) * DATA_SETUP
821 * 4.2) From the aspect of the GPMI nand controller:
822 * Delay = RDN_DELAY * 0.125 * RP {2}
824 * RP : the DLL reference period.
825 * if (GPMI-clock-period > DLL_THRETHOLD)
826 * RP = GPMI-clock-period / 2;
828 * RP = GPMI-clock-period;
830 * Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period
831 * is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD
832 * is 16000ps, but in mx6q, we use 12000ps.
834 * 4.3) since {1} equals {2}, we get:
836 * (tREA + 4000 - tRP) * 8
837 * RDN_DELAY = ----------------------- {3}
840 static int gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
841 const struct nand_sdr_timings *sdr)
843 struct gpmi_nfc_hardware_timing *hw = &this->hw;
844 struct resources *r = &this->resources;
845 unsigned int dll_threshold_ps = this->devdata->max_chain_delay;
846 unsigned int period_ps, reference_period_ps;
847 unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;
849 bool use_half_period;
850 int sample_delay_ps, sample_delay_factor;
851 unsigned int busy_timeout_cycles;
853 unsigned long clk_rate, min_rate;
856 if (sdr->tRC_min >= 30000) {
857 /* ONFI non-EDO modes [0-3] */
858 hw->clk_rate = 22000000;
860 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS;
861 } else if (sdr->tRC_min >= 25000) {
862 /* ONFI EDO mode 4 */
863 hw->clk_rate = 80000000;
865 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
867 /* ONFI EDO mode 5 */
868 hw->clk_rate = 100000000;
870 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
873 clk_rate = clk_round_rate(r->clock[0], hw->clk_rate);
874 if (clk_rate <= min_rate) {
875 dev_err(this->dev, "clock setting: expected %ld, got %ld\n",
876 hw->clk_rate, clk_rate);
880 hw->clk_rate = clk_rate;
881 /* SDR core timings are given in picoseconds */
882 period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);
884 addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps);
885 data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps);
886 data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps);
887 busy_timeout_ps = max(sdr->tBERS_max, sdr->tPROG_max);
888 busy_timeout_cycles = TO_CYCLES(busy_timeout_ps, period_ps);
890 hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) |
891 BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) |
892 BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles);
893 hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(DIV_ROUND_UP(busy_timeout_cycles, 4096));
896 * Derive NFC ideal delay from {3}:
898 * (tREA + 4000 - tRP) * 8
899 * RDN_DELAY = -----------------------
902 if (period_ps > dll_threshold_ps) {
903 use_half_period = true;
904 reference_period_ps = period_ps / 2;
906 use_half_period = false;
907 reference_period_ps = period_ps;
910 tRP_ps = data_setup_cycles * period_ps;
911 sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8;
912 if (sample_delay_ps > 0)
913 sample_delay_factor = sample_delay_ps / reference_period_ps;
915 sample_delay_factor = 0;
917 hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel);
918 if (sample_delay_factor)
919 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) |
920 BM_GPMI_CTRL1_DLL_ENABLE |
921 (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);
925 static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
927 struct gpmi_nfc_hardware_timing *hw = &this->hw;
928 struct resources *r = &this->resources;
929 void __iomem *gpmi_regs = r->gpmi_regs;
930 unsigned int dll_wait_time_us;
933 /* Clock dividers do NOT guarantee a clean clock signal on its output
934 * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8,
935 * all clock dividers provide these guarantee.
937 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this))
938 clk_disable_unprepare(r->clock[0]);
940 ret = clk_set_rate(r->clock[0], hw->clk_rate);
942 dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret);
946 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) {
947 ret = clk_prepare_enable(r->clock[0]);
952 writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);
953 writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);
956 * Clear several CTRL1 fields, DLL must be disabled when setting
957 * RDN_DELAY or HALF_PERIOD.
959 writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR);
960 writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET);
962 /* Wait 64 clock cycles before using the GPMI after enabling the DLL */
963 dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64;
964 if (!dll_wait_time_us)
965 dll_wait_time_us = 1;
967 /* Wait for the DLL to settle. */
968 udelay(dll_wait_time_us);
973 static int gpmi_setup_interface(struct nand_chip *chip, int chipnr,
974 const struct nand_interface_config *conf)
976 struct gpmi_nand_data *this = nand_get_controller_data(chip);
977 const struct nand_sdr_timings *sdr;
980 /* Retrieve required NAND timings */
981 sdr = nand_get_sdr_timings(conf);
985 /* Only MX28/MX6 GPMI controller can reach EDO timings */
986 if (sdr->tRC_min <= 25000 && !GPMI_IS_MX28(this) && !GPMI_IS_MX6(this))
989 /* Stop here if this call was just a check */
993 /* Do the actual derivation of the controller timings */
994 ret = gpmi_nfc_compute_timings(this, sdr);
998 this->hw.must_apply_timings = true;
1003 /* Clears a BCH interrupt. */
1004 static void gpmi_clear_bch(struct gpmi_nand_data *this)
1006 struct resources *r = &this->resources;
1007 writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR);
1010 static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)
1012 /* We use the DMA channel 0 to access all the nand chips. */
1013 return this->dma_chans[0];
1016 /* This will be called after the DMA operation is finished. */
1017 static void dma_irq_callback(void *param)
1019 struct gpmi_nand_data *this = param;
1020 struct completion *dma_c = &this->dma_done;
1025 static irqreturn_t bch_irq(int irq, void *cookie)
1027 struct gpmi_nand_data *this = cookie;
1029 gpmi_clear_bch(this);
1030 complete(&this->bch_done);
1034 static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len)
1037 * raw_len is the length to read/write including bch data which
1038 * we are passed in exec_op. Calculate the data length from it.
1041 return ALIGN_DOWN(raw_len, this->bch_geometry.eccn_chunk_size);
1046 /* Can we use the upper's buffer directly for DMA? */
1047 static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf,
1048 int raw_len, struct scatterlist *sgl,
1049 enum dma_data_direction dr)
1052 int len = gpmi_raw_len_to_len(this, raw_len);
1054 /* first try to map the upper buffer directly */
1055 if (virt_addr_valid(buf) && !object_is_on_stack(buf)) {
1056 sg_init_one(sgl, buf, len);
1057 ret = dma_map_sg(this->dev, sgl, 1, dr);
1065 /* We have to use our own DMA buffer. */
1066 sg_init_one(sgl, this->data_buffer_dma, len);
1068 if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma)
1069 memcpy(this->data_buffer_dma, buf, len);
1071 dma_map_sg(this->dev, sgl, 1, dr);
1076 /* add our owner bbt descriptor */
1077 static uint8_t scan_ff_pattern[] = { 0xff };
1078 static struct nand_bbt_descr gpmi_bbt_descr = {
1082 .pattern = scan_ff_pattern
1086 * We may change the layout if we can get the ECC info from the datasheet,
1087 * else we will use all the (page + OOB).
1089 static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section,
1090 struct mtd_oob_region *oobregion)
1092 struct nand_chip *chip = mtd_to_nand(mtd);
1093 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1094 struct bch_geometry *geo = &this->bch_geometry;
1099 oobregion->offset = 0;
1100 oobregion->length = geo->page_size - mtd->writesize;
1105 static int gpmi_ooblayout_free(struct mtd_info *mtd, int section,
1106 struct mtd_oob_region *oobregion)
1108 struct nand_chip *chip = mtd_to_nand(mtd);
1109 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1110 struct bch_geometry *geo = &this->bch_geometry;
1115 /* The available oob size we have. */
1116 if (geo->page_size < mtd->writesize + mtd->oobsize) {
1117 oobregion->offset = geo->page_size - mtd->writesize;
1118 oobregion->length = mtd->oobsize - oobregion->offset;
1124 static const char * const gpmi_clks_for_mx2x[] = {
1128 static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = {
1129 .ecc = gpmi_ooblayout_ecc,
1130 .free = gpmi_ooblayout_free,
1133 static const struct gpmi_devdata gpmi_devdata_imx23 = {
1135 .bch_max_ecc_strength = 20,
1136 .max_chain_delay = 16000,
1137 .clks = gpmi_clks_for_mx2x,
1138 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
1141 static const struct gpmi_devdata gpmi_devdata_imx28 = {
1143 .bch_max_ecc_strength = 20,
1144 .max_chain_delay = 16000,
1145 .clks = gpmi_clks_for_mx2x,
1146 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
1149 static const char * const gpmi_clks_for_mx6[] = {
1150 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch",
1153 static const struct gpmi_devdata gpmi_devdata_imx6q = {
1155 .bch_max_ecc_strength = 40,
1156 .max_chain_delay = 12000,
1157 .clks = gpmi_clks_for_mx6,
1158 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
1161 static const struct gpmi_devdata gpmi_devdata_imx6sx = {
1163 .bch_max_ecc_strength = 62,
1164 .max_chain_delay = 12000,
1165 .clks = gpmi_clks_for_mx6,
1166 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
1169 static const char * const gpmi_clks_for_mx7d[] = {
1170 "gpmi_io", "gpmi_bch_apb",
1173 static const struct gpmi_devdata gpmi_devdata_imx7d = {
1175 .bch_max_ecc_strength = 62,
1176 .max_chain_delay = 12000,
1177 .clks = gpmi_clks_for_mx7d,
1178 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d),
1181 static int acquire_register_block(struct gpmi_nand_data *this,
1182 const char *res_name)
1184 struct platform_device *pdev = this->pdev;
1185 struct resources *res = &this->resources;
1188 p = devm_platform_ioremap_resource_byname(pdev, res_name);
1192 if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME))
1194 else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME))
1197 dev_err(this->dev, "unknown resource name : %s\n", res_name);
1202 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h)
1204 struct platform_device *pdev = this->pdev;
1205 const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME;
1208 err = platform_get_irq_byname(pdev, res_name);
1212 err = devm_request_irq(this->dev, err, irq_h, 0, res_name, this);
1214 dev_err(this->dev, "error requesting BCH IRQ\n");
1219 static void release_dma_channels(struct gpmi_nand_data *this)
1222 for (i = 0; i < DMA_CHANS; i++)
1223 if (this->dma_chans[i]) {
1224 dma_release_channel(this->dma_chans[i]);
1225 this->dma_chans[i] = NULL;
1229 static int acquire_dma_channels(struct gpmi_nand_data *this)
1231 struct platform_device *pdev = this->pdev;
1232 struct dma_chan *dma_chan;
1235 /* request dma channel */
1236 dma_chan = dma_request_chan(&pdev->dev, "rx-tx");
1237 if (IS_ERR(dma_chan)) {
1238 ret = dev_err_probe(this->dev, PTR_ERR(dma_chan),
1239 "DMA channel request failed\n");
1240 release_dma_channels(this);
1242 this->dma_chans[0] = dma_chan;
1248 static int gpmi_get_clks(struct gpmi_nand_data *this)
1250 struct resources *r = &this->resources;
1254 for (i = 0; i < this->devdata->clks_count; i++) {
1255 clk = devm_clk_get(this->dev, this->devdata->clks[i]);
1267 dev_dbg(this->dev, "failed in finding the clocks.\n");
1271 static int acquire_resources(struct gpmi_nand_data *this)
1275 ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME);
1279 ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME);
1283 ret = acquire_bch_irq(this, bch_irq);
1287 ret = acquire_dma_channels(this);
1291 ret = gpmi_get_clks(this);
1297 release_dma_channels(this);
1302 static void release_resources(struct gpmi_nand_data *this)
1304 release_dma_channels(this);
1307 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this)
1309 struct device *dev = this->dev;
1310 struct bch_geometry *geo = &this->bch_geometry;
1312 if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt))
1313 dma_free_coherent(dev, geo->auxiliary_size,
1314 this->auxiliary_virt,
1315 this->auxiliary_phys);
1316 kfree(this->data_buffer_dma);
1317 kfree(this->raw_buffer);
1319 this->data_buffer_dma = NULL;
1320 this->raw_buffer = NULL;
1323 /* Allocate the DMA buffers */
1324 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)
1326 struct bch_geometry *geo = &this->bch_geometry;
1327 struct device *dev = this->dev;
1328 struct mtd_info *mtd = nand_to_mtd(&this->nand);
1331 * [2] Allocate a read/write data buffer.
1332 * The gpmi_alloc_dma_buffer can be called twice.
1333 * We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer
1334 * is called before the NAND identification; and we allocate a
1335 * buffer of the real NAND page size when the gpmi_alloc_dma_buffer
1338 this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE,
1339 GFP_DMA | GFP_KERNEL);
1340 if (this->data_buffer_dma == NULL)
1343 this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size,
1344 &this->auxiliary_phys, GFP_DMA);
1345 if (!this->auxiliary_virt)
1348 this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL);
1349 if (!this->raw_buffer)
1355 gpmi_free_dma_buffer(this);
1360 * Handles block mark swapping.
1361 * It can be called in swapping the block mark, or swapping it back,
1362 * because the operations are the same.
1364 static void block_mark_swapping(struct gpmi_nand_data *this,
1365 void *payload, void *auxiliary)
1367 struct bch_geometry *nfc_geo = &this->bch_geometry;
1372 unsigned char from_data;
1373 unsigned char from_oob;
1375 if (!this->swap_block_mark)
1379 * If control arrives here, we're swapping. Make some convenience
1382 bit = nfc_geo->block_mark_bit_offset;
1383 p = payload + nfc_geo->block_mark_byte_offset;
1387 * Get the byte from the data area that overlays the block mark. Since
1388 * the ECC engine applies its own view to the bits in the page, the
1389 * physical block mark won't (in general) appear on a byte boundary in
1392 from_data = (p[0] >> bit) | (p[1] << (8 - bit));
1394 /* Get the byte from the OOB. */
1400 mask = (0x1 << bit) - 1;
1401 p[0] = (p[0] & mask) | (from_oob << bit);
1404 p[1] = (p[1] & mask) | (from_oob >> (8 - bit));
1407 static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first,
1410 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1411 struct bch_geometry *nfc_geo = &this->bch_geometry;
1412 struct mtd_info *mtd = nand_to_mtd(chip);
1414 unsigned char *status;
1415 unsigned int max_bitflips = 0;
1417 /* Loop over status bytes, accumulating ECC status. */
1418 status = this->auxiliary_virt + ALIGN(meta, 4);
1420 for (i = first; i < last; i++, status++) {
1421 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))
1424 if (*status == STATUS_UNCORRECTABLE) {
1425 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1426 u8 *eccbuf = this->raw_buffer;
1427 int offset, bitoffset;
1431 /* Read ECC bytes into our internal raw_buffer */
1432 offset = nfc_geo->metadata_size * 8;
1433 offset += ((8 * nfc_geo->eccn_chunk_size) + eccbits) * (i + 1);
1435 bitoffset = offset % 8;
1436 eccbytes = DIV_ROUND_UP(offset + eccbits, 8);
1439 nand_change_read_column_op(chip, offset, eccbuf,
1443 * ECC data are not byte aligned and we may have
1444 * in-band data in the first and last byte of
1445 * eccbuf. Set non-eccbits to one so that
1446 * nand_check_erased_ecc_chunk() does not count them
1450 eccbuf[0] |= GENMASK(bitoffset - 1, 0);
1452 bitoffset = (bitoffset + eccbits) % 8;
1454 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset);
1457 * The ECC hardware has an uncorrectable ECC status
1458 * code in case we have bitflips in an erased page. As
1459 * nothing was written into this subpage the ECC is
1460 * obviously wrong and we can not trust it. We assume
1461 * at this point that we are reading an erased page and
1462 * try to correct the bitflips in buffer up to
1463 * ecc_strength bitflips. If this is a page with random
1464 * data, we exceed this number of bitflips and have a
1465 * ECC failure. Otherwise we use the corrected buffer.
1468 /* The first block includes metadata */
1469 flips = nand_check_erased_ecc_chunk(
1470 buf + i * nfc_geo->eccn_chunk_size,
1471 nfc_geo->eccn_chunk_size,
1473 this->auxiliary_virt,
1474 nfc_geo->metadata_size,
1475 nfc_geo->ecc_strength);
1477 flips = nand_check_erased_ecc_chunk(
1478 buf + i * nfc_geo->eccn_chunk_size,
1479 nfc_geo->eccn_chunk_size,
1482 nfc_geo->ecc_strength);
1486 max_bitflips = max_t(unsigned int, max_bitflips,
1488 mtd->ecc_stats.corrected += flips;
1492 mtd->ecc_stats.failed++;
1496 mtd->ecc_stats.corrected += *status;
1497 max_bitflips = max_t(unsigned int, max_bitflips, *status);
1500 return max_bitflips;
1503 static void gpmi_bch_layout_std(struct gpmi_nand_data *this)
1505 struct bch_geometry *geo = &this->bch_geometry;
1506 unsigned int ecc_strength = geo->ecc_strength >> 1;
1507 unsigned int gf_len = geo->gf_len;
1508 unsigned int block0_size = geo->ecc0_chunk_size;
1509 unsigned int blockn_size = geo->eccn_chunk_size;
1511 this->bch_flashlayout0 =
1512 BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) |
1513 BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) |
1514 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |
1515 BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) |
1516 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block0_size, this);
1518 this->bch_flashlayout1 =
1519 BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) |
1520 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |
1521 BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) |
1522 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(blockn_size, this);
1525 static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf,
1526 int oob_required, int page)
1528 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1529 struct mtd_info *mtd = nand_to_mtd(chip);
1530 struct bch_geometry *geo = &this->bch_geometry;
1531 unsigned int max_bitflips;
1534 gpmi_bch_layout_std(this);
1537 ret = nand_read_page_op(chip, page, 0, buf, geo->page_size);
1541 max_bitflips = gpmi_count_bitflips(chip, buf, 0,
1542 geo->ecc_chunk_count,
1543 geo->auxiliary_status_offset);
1545 /* handle the block mark swapping */
1546 block_mark_swapping(this, buf, this->auxiliary_virt);
1550 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob()
1551 * for details about our policy for delivering the OOB.
1553 * We fill the caller's buffer with set bits, and then copy the
1554 * block mark to th caller's buffer. Note that, if block mark
1555 * swapping was necessary, it has already been done, so we can
1556 * rely on the first byte of the auxiliary buffer to contain
1559 memset(chip->oob_poi, ~0, mtd->oobsize);
1560 chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0];
1563 return max_bitflips;
1566 /* Fake a virtual small page for the subpage read */
1567 static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs,
1568 uint32_t len, uint8_t *buf, int page)
1570 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1571 struct bch_geometry *geo = &this->bch_geometry;
1572 int size = chip->ecc.size; /* ECC chunk size */
1573 int meta, n, page_size;
1574 unsigned int max_bitflips;
1575 unsigned int ecc_strength;
1576 int first, last, marker_pos;
1577 int ecc_parity_size;
1581 /* The size of ECC parity */
1582 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1584 /* Align it with the chunk size */
1585 first = offs / size;
1586 last = (offs + len - 1) / size;
1588 if (this->swap_block_mark) {
1590 * Find the chunk which contains the Block Marker.
1591 * If this chunk is in the range of [first, last],
1592 * we have to read out the whole page.
1593 * Why? since we had swapped the data at the position of Block
1594 * Marker to the metadata which is bound with the chunk 0.
1596 marker_pos = geo->block_mark_byte_offset / size;
1597 if (last >= marker_pos && first <= marker_pos) {
1599 "page:%d, first:%d, last:%d, marker at:%d\n",
1600 page, first, last, marker_pos);
1601 return gpmi_ecc_read_page(chip, buf, 0, page);
1606 * if there is an ECC dedicate for meta:
1607 * - need to add an extra ECC size when calculating col and page_size,
1608 * if the meta size is NOT zero.
1609 * - ecc0_chunk size need to set to the same size as other chunks,
1610 * if the meta size is zero.
1613 meta = geo->metadata_size;
1615 if (geo->ecc_for_meta)
1616 col = meta + ecc_parity_size
1617 + (size + ecc_parity_size) * first;
1619 col = meta + (size + ecc_parity_size) * first;
1622 buf = buf + first * size;
1625 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1626 n = last - first + 1;
1628 if (geo->ecc_for_meta && meta)
1629 page_size = meta + ecc_parity_size
1630 + (size + ecc_parity_size) * n;
1632 page_size = meta + (size + ecc_parity_size) * n;
1634 ecc_strength = geo->ecc_strength >> 1;
1636 this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS(
1637 (geo->ecc_for_meta ? n : n - 1)) |
1638 BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) |
1639 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |
1640 BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) |
1641 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE((geo->ecc_for_meta ?
1642 0 : geo->ecc0_chunk_size), this);
1644 this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) |
1645 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |
1646 BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) |
1647 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->eccn_chunk_size, this);
1651 ret = nand_read_page_op(chip, page, col, buf, page_size);
1655 dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n",
1656 page, offs, len, col, first, n, page_size);
1658 max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta);
1660 return max_bitflips;
1663 static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf,
1664 int oob_required, int page)
1666 struct mtd_info *mtd = nand_to_mtd(chip);
1667 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1668 struct bch_geometry *nfc_geo = &this->bch_geometry;
1670 dev_dbg(this->dev, "ecc write page.\n");
1672 gpmi_bch_layout_std(this);
1675 memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size);
1677 if (this->swap_block_mark) {
1679 * When doing bad block marker swapping we must always copy the
1680 * input buffer as we can't modify the const buffer.
1682 memcpy(this->data_buffer_dma, buf, mtd->writesize);
1683 buf = this->data_buffer_dma;
1684 block_mark_swapping(this, this->data_buffer_dma,
1685 this->auxiliary_virt);
1688 return nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size);
1692 * There are several places in this driver where we have to handle the OOB and
1693 * block marks. This is the function where things are the most complicated, so
1694 * this is where we try to explain it all. All the other places refer back to
1697 * These are the rules, in order of decreasing importance:
1699 * 1) Nothing the caller does can be allowed to imperil the block mark.
1701 * 2) In read operations, the first byte of the OOB we return must reflect the
1702 * true state of the block mark, no matter where that block mark appears in
1703 * the physical page.
1705 * 3) ECC-based read operations return an OOB full of set bits (since we never
1706 * allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads
1709 * 4) "Raw" read operations return a direct view of the physical bytes in the
1710 * page, using the conventional definition of which bytes are data and which
1711 * are OOB. This gives the caller a way to see the actual, physical bytes
1712 * in the page, without the distortions applied by our ECC engine.
1715 * What we do for this specific read operation depends on two questions:
1717 * 1) Are we doing a "raw" read, or an ECC-based read?
1719 * 2) Are we using block mark swapping or transcription?
1721 * There are four cases, illustrated by the following Karnaugh map:
1723 * | Raw | ECC-based |
1724 * -------------+-------------------------+-------------------------+
1725 * | Read the conventional | |
1726 * | OOB at the end of the | |
1727 * Swapping | page and return it. It | |
1728 * | contains exactly what | |
1729 * | we want. | Read the block mark and |
1730 * -------------+-------------------------+ return it in a buffer |
1731 * | Read the conventional | full of set bits. |
1732 * | OOB at the end of the | |
1733 * | page and also the block | |
1734 * Transcribing | mark in the metadata. | |
1735 * | Copy the block mark | |
1736 * | into the first byte of | |
1738 * -------------+-------------------------+-------------------------+
1740 * Note that we break rule #4 in the Transcribing/Raw case because we're not
1741 * giving an accurate view of the actual, physical bytes in the page (we're
1742 * overwriting the block mark). That's OK because it's more important to follow
1745 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not
1746 * easy. When reading a page, for example, the NAND Flash MTD code calls our
1747 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an
1748 * ECC-based or raw view of the page is implicit in which function it calls
1749 * (there is a similar pair of ECC-based/raw functions for writing).
1751 static int gpmi_ecc_read_oob(struct nand_chip *chip, int page)
1753 struct mtd_info *mtd = nand_to_mtd(chip);
1754 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1757 /* clear the OOB buffer */
1758 memset(chip->oob_poi, ~0, mtd->oobsize);
1760 /* Read out the conventional OOB. */
1761 ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi,
1767 * Now, we want to make sure the block mark is correct. In the
1768 * non-transcribing case (!GPMI_IS_MX23()), we already have it.
1769 * Otherwise, we need to explicitly read it.
1771 if (GPMI_IS_MX23(this)) {
1772 /* Read the block mark into the first byte of the OOB buffer. */
1773 ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1);
1781 static int gpmi_ecc_write_oob(struct nand_chip *chip, int page)
1783 struct mtd_info *mtd = nand_to_mtd(chip);
1784 struct mtd_oob_region of = { };
1786 /* Do we have available oob area? */
1787 mtd_ooblayout_free(mtd, 0, &of);
1791 if (!nand_is_slc(chip))
1794 return nand_prog_page_op(chip, page, mtd->writesize + of.offset,
1795 chip->oob_poi + of.offset, of.length);
1799 * This function reads a NAND page without involving the ECC engine (no HW
1801 * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1802 * inline (interleaved with payload DATA), and do not align data chunk on
1804 * We thus need to take care moving the payload data and ECC bits stored in the
1805 * page into the provided buffers, which is why we're using nand_extract_bits().
1807 * See set_geometry_by_ecc_info inline comments to have a full description
1808 * of the layout used by the GPMI controller.
1810 static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
1811 int oob_required, int page)
1813 struct mtd_info *mtd = nand_to_mtd(chip);
1814 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1815 struct bch_geometry *nfc_geo = &this->bch_geometry;
1816 int eccsize = nfc_geo->eccn_chunk_size;
1817 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1818 u8 *tmp_buf = this->raw_buffer;
1821 size_t oob_byte_off;
1822 uint8_t *oob = chip->oob_poi;
1826 ret = nand_read_page_op(chip, page, 0, tmp_buf,
1827 mtd->writesize + mtd->oobsize);
1832 * If required, swap the bad block marker and the data stored in the
1833 * metadata section, so that we don't wrongly consider a block as bad.
1835 * See the layout description for a detailed explanation on why this
1838 if (this->swap_block_mark)
1839 swap(tmp_buf[0], tmp_buf[mtd->writesize]);
1842 * Copy the metadata section into the oob buffer (this section is
1843 * guaranteed to be aligned on a byte boundary).
1846 memcpy(oob, tmp_buf, nfc_geo->metadata_size);
1848 oob_bit_off = nfc_geo->metadata_size * 8;
1849 src_bit_off = oob_bit_off;
1851 /* Extract interleaved payload data and ECC bits */
1852 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1854 nand_extract_bits(buf, step * eccsize * 8, tmp_buf,
1855 src_bit_off, eccsize * 8);
1856 src_bit_off += eccsize * 8;
1858 /* Align last ECC block to align a byte boundary */
1859 if (step == nfc_geo->ecc_chunk_count - 1 &&
1860 (oob_bit_off + eccbits) % 8)
1861 eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1864 nand_extract_bits(oob, oob_bit_off, tmp_buf,
1865 src_bit_off, eccbits);
1867 src_bit_off += eccbits;
1868 oob_bit_off += eccbits;
1872 oob_byte_off = oob_bit_off / 8;
1874 if (oob_byte_off < mtd->oobsize)
1875 memcpy(oob + oob_byte_off,
1876 tmp_buf + mtd->writesize + oob_byte_off,
1877 mtd->oobsize - oob_byte_off);
1884 * This function writes a NAND page without involving the ECC engine (no HW
1886 * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1887 * inline (interleaved with payload DATA), and do not align data chunk on
1889 * We thus need to take care moving the OOB area at the right place in the
1890 * final page, which is why we're using nand_extract_bits().
1892 * See set_geometry_by_ecc_info inline comments to have a full description
1893 * of the layout used by the GPMI controller.
1895 static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
1896 int oob_required, int page)
1898 struct mtd_info *mtd = nand_to_mtd(chip);
1899 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1900 struct bch_geometry *nfc_geo = &this->bch_geometry;
1901 int eccsize = nfc_geo->eccn_chunk_size;
1902 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1903 u8 *tmp_buf = this->raw_buffer;
1904 uint8_t *oob = chip->oob_poi;
1907 size_t oob_byte_off;
1911 * Initialize all bits to 1 in case we don't have a buffer for the
1912 * payload or oob data in order to leave unspecified bits of data
1913 * to their initial state.
1915 if (!buf || !oob_required)
1916 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize);
1919 * First copy the metadata section (stored in oob buffer) at the
1920 * beginning of the page, as imposed by the GPMI layout.
1922 memcpy(tmp_buf, oob, nfc_geo->metadata_size);
1923 oob_bit_off = nfc_geo->metadata_size * 8;
1924 dst_bit_off = oob_bit_off;
1926 /* Interleave payload data and ECC bits */
1927 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1929 nand_extract_bits(tmp_buf, dst_bit_off, buf,
1930 step * eccsize * 8, eccsize * 8);
1931 dst_bit_off += eccsize * 8;
1933 /* Align last ECC block to align a byte boundary */
1934 if (step == nfc_geo->ecc_chunk_count - 1 &&
1935 (oob_bit_off + eccbits) % 8)
1936 eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1939 nand_extract_bits(tmp_buf, dst_bit_off, oob,
1940 oob_bit_off, eccbits);
1942 dst_bit_off += eccbits;
1943 oob_bit_off += eccbits;
1946 oob_byte_off = oob_bit_off / 8;
1948 if (oob_required && oob_byte_off < mtd->oobsize)
1949 memcpy(tmp_buf + mtd->writesize + oob_byte_off,
1950 oob + oob_byte_off, mtd->oobsize - oob_byte_off);
1953 * If required, swap the bad block marker and the first byte of the
1954 * metadata section, so that we don't modify the bad block marker.
1956 * See the layout description for a detailed explanation on why this
1959 if (this->swap_block_mark)
1960 swap(tmp_buf[0], tmp_buf[mtd->writesize]);
1962 return nand_prog_page_op(chip, page, 0, tmp_buf,
1963 mtd->writesize + mtd->oobsize);
1966 static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page)
1968 return gpmi_ecc_read_page_raw(chip, NULL, 1, page);
1971 static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page)
1973 return gpmi_ecc_write_page_raw(chip, NULL, 1, page);
1976 static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs)
1978 struct mtd_info *mtd = nand_to_mtd(chip);
1979 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1981 uint8_t *block_mark;
1982 int column, page, chipnr;
1984 chipnr = (int)(ofs >> chip->chip_shift);
1985 nand_select_target(chip, chipnr);
1987 column = !GPMI_IS_MX23(this) ? mtd->writesize : 0;
1989 /* Write the block mark. */
1990 block_mark = this->data_buffer_dma;
1991 block_mark[0] = 0; /* bad block marker */
1993 /* Shift to get page */
1994 page = (int)(ofs >> chip->page_shift);
1996 ret = nand_prog_page_op(chip, page, column, block_mark, 1);
1998 nand_deselect_target(chip);
2003 static int nand_boot_set_geometry(struct gpmi_nand_data *this)
2005 struct boot_rom_geometry *geometry = &this->rom_geometry;
2008 * Set the boot block stride size.
2010 * In principle, we should be reading this from the OTP bits, since
2011 * that's where the ROM is going to get it. In fact, we don't have any
2012 * way to read the OTP bits, so we go with the default and hope for the
2015 geometry->stride_size_in_pages = 64;
2018 * Set the search area stride exponent.
2020 * In principle, we should be reading this from the OTP bits, since
2021 * that's where the ROM is going to get it. In fact, we don't have any
2022 * way to read the OTP bits, so we go with the default and hope for the
2025 geometry->search_area_stride_exponent = 2;
2029 static const char *fingerprint = "STMP";
2030 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
2032 struct boot_rom_geometry *rom_geo = &this->rom_geometry;
2033 struct device *dev = this->dev;
2034 struct nand_chip *chip = &this->nand;
2035 unsigned int search_area_size_in_strides;
2036 unsigned int stride;
2038 u8 *buffer = nand_get_data_buf(chip);
2039 int found_an_ncb_fingerprint = false;
2042 /* Compute the number of strides in a search area. */
2043 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
2045 nand_select_target(chip, 0);
2048 * Loop through the first search area, looking for the NCB fingerprint.
2050 dev_dbg(dev, "Scanning for an NCB fingerprint...\n");
2052 for (stride = 0; stride < search_area_size_in_strides; stride++) {
2053 /* Compute the page addresses. */
2054 page = stride * rom_geo->stride_size_in_pages;
2056 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page);
2059 * Read the NCB fingerprint. The fingerprint is four bytes long
2060 * and starts in the 12th byte of the page.
2062 ret = nand_read_page_op(chip, page, 12, buffer,
2063 strlen(fingerprint));
2067 /* Look for the fingerprint. */
2068 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) {
2069 found_an_ncb_fingerprint = true;
2075 nand_deselect_target(chip);
2077 if (found_an_ncb_fingerprint)
2078 dev_dbg(dev, "\tFound a fingerprint\n");
2080 dev_dbg(dev, "\tNo fingerprint found\n");
2081 return found_an_ncb_fingerprint;
2084 /* Writes a transcription stamp. */
2085 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
2087 struct device *dev = this->dev;
2088 struct boot_rom_geometry *rom_geo = &this->rom_geometry;
2089 struct nand_chip *chip = &this->nand;
2090 struct mtd_info *mtd = nand_to_mtd(chip);
2091 unsigned int block_size_in_pages;
2092 unsigned int search_area_size_in_strides;
2093 unsigned int search_area_size_in_pages;
2094 unsigned int search_area_size_in_blocks;
2096 unsigned int stride;
2098 u8 *buffer = nand_get_data_buf(chip);
2101 /* Compute the search area geometry. */
2102 block_size_in_pages = mtd->erasesize / mtd->writesize;
2103 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
2104 search_area_size_in_pages = search_area_size_in_strides *
2105 rom_geo->stride_size_in_pages;
2106 search_area_size_in_blocks =
2107 (search_area_size_in_pages + (block_size_in_pages - 1)) /
2108 block_size_in_pages;
2110 dev_dbg(dev, "Search Area Geometry :\n");
2111 dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks);
2112 dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides);
2113 dev_dbg(dev, "\tin Pages : %u\n", search_area_size_in_pages);
2115 nand_select_target(chip, 0);
2117 /* Loop over blocks in the first search area, erasing them. */
2118 dev_dbg(dev, "Erasing the search area...\n");
2120 for (block = 0; block < search_area_size_in_blocks; block++) {
2121 /* Erase this block. */
2122 dev_dbg(dev, "\tErasing block 0x%x\n", block);
2123 status = nand_erase_op(chip, block);
2125 dev_err(dev, "[%s] Erase failed.\n", __func__);
2128 /* Write the NCB fingerprint into the page buffer. */
2129 memset(buffer, ~0, mtd->writesize);
2130 memcpy(buffer + 12, fingerprint, strlen(fingerprint));
2132 /* Loop through the first search area, writing NCB fingerprints. */
2133 dev_dbg(dev, "Writing NCB fingerprints...\n");
2134 for (stride = 0; stride < search_area_size_in_strides; stride++) {
2135 /* Compute the page addresses. */
2136 page = stride * rom_geo->stride_size_in_pages;
2138 /* Write the first page of the current stride. */
2139 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);
2141 status = chip->ecc.write_page_raw(chip, buffer, 0, page);
2143 dev_err(dev, "[%s] Write failed.\n", __func__);
2146 nand_deselect_target(chip);
2151 static int mx23_boot_init(struct gpmi_nand_data *this)
2153 struct device *dev = this->dev;
2154 struct nand_chip *chip = &this->nand;
2155 struct mtd_info *mtd = nand_to_mtd(chip);
2156 unsigned int block_count;
2165 * If control arrives here, we can't use block mark swapping, which
2166 * means we're forced to use transcription. First, scan for the
2167 * transcription stamp. If we find it, then we don't have to do
2168 * anything -- the block marks are already transcribed.
2170 if (mx23_check_transcription_stamp(this))
2174 * If control arrives here, we couldn't find a transcription stamp, so
2175 * so we presume the block marks are in the conventional location.
2177 dev_dbg(dev, "Transcribing bad block marks...\n");
2179 /* Compute the number of blocks in the entire medium. */
2180 block_count = nanddev_eraseblocks_per_target(&chip->base);
2183 * Loop over all the blocks in the medium, transcribing block marks as
2186 for (block = 0; block < block_count; block++) {
2188 * Compute the chip, page and byte addresses for this block's
2189 * conventional mark.
2191 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift);
2192 page = block << (chip->phys_erase_shift - chip->page_shift);
2193 byte = block << chip->phys_erase_shift;
2195 /* Send the command to read the conventional block mark. */
2196 nand_select_target(chip, chipnr);
2197 ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark,
2199 nand_deselect_target(chip);
2205 * Check if the block is marked bad. If so, we need to mark it
2206 * again, but this time the result will be a mark in the
2207 * location where we transcribe block marks.
2209 if (block_mark != 0xff) {
2210 dev_dbg(dev, "Transcribing mark in block %u\n", block);
2211 ret = chip->legacy.block_markbad(chip, byte);
2214 "Failed to mark block bad with ret %d\n",
2219 /* Write the stamp that indicates we've transcribed the block marks. */
2220 mx23_write_transcription_stamp(this);
2224 static int nand_boot_init(struct gpmi_nand_data *this)
2226 nand_boot_set_geometry(this);
2228 /* This is ROM arch-specific initilization before the BBT scanning. */
2229 if (GPMI_IS_MX23(this))
2230 return mx23_boot_init(this);
2234 static int gpmi_set_geometry(struct gpmi_nand_data *this)
2238 /* Free the temporary DMA memory for reading ID. */
2239 gpmi_free_dma_buffer(this);
2241 /* Set up the NFC geometry which is used by BCH. */
2242 ret = bch_set_geometry(this);
2244 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret);
2248 /* Alloc the new DMA buffers according to the pagesize and oobsize */
2249 return gpmi_alloc_dma_buffer(this);
2252 static int gpmi_init_last(struct gpmi_nand_data *this)
2254 struct nand_chip *chip = &this->nand;
2255 struct mtd_info *mtd = nand_to_mtd(chip);
2256 struct nand_ecc_ctrl *ecc = &chip->ecc;
2257 struct bch_geometry *bch_geo = &this->bch_geometry;
2260 /* Set up the medium geometry */
2261 ret = gpmi_set_geometry(this);
2265 /* Init the nand_ecc_ctrl{} */
2266 ecc->read_page = gpmi_ecc_read_page;
2267 ecc->write_page = gpmi_ecc_write_page;
2268 ecc->read_oob = gpmi_ecc_read_oob;
2269 ecc->write_oob = gpmi_ecc_write_oob;
2270 ecc->read_page_raw = gpmi_ecc_read_page_raw;
2271 ecc->write_page_raw = gpmi_ecc_write_page_raw;
2272 ecc->read_oob_raw = gpmi_ecc_read_oob_raw;
2273 ecc->write_oob_raw = gpmi_ecc_write_oob_raw;
2274 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2275 ecc->size = bch_geo->eccn_chunk_size;
2276 ecc->strength = bch_geo->ecc_strength;
2277 mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops);
2280 * We only enable the subpage read when:
2281 * (1) the chip is imx6, and
2282 * (2) the size of the ECC parity is byte aligned.
2284 if (GPMI_IS_MX6(this) &&
2285 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) {
2286 ecc->read_subpage = gpmi_ecc_read_subpage;
2287 chip->options |= NAND_SUBPAGE_READ;
2293 static int gpmi_nand_attach_chip(struct nand_chip *chip)
2295 struct gpmi_nand_data *this = nand_get_controller_data(chip);
2298 if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2299 chip->bbt_options |= NAND_BBT_NO_OOB;
2301 if (of_property_read_bool(this->dev->of_node,
2302 "fsl,no-blockmark-swap"))
2303 this->swap_block_mark = false;
2305 dev_dbg(this->dev, "Blockmark swapping %sabled\n",
2306 this->swap_block_mark ? "en" : "dis");
2308 ret = gpmi_init_last(this);
2312 chip->options |= NAND_SKIP_BBTSCAN;
2317 static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this)
2319 struct gpmi_transfer *transfer = &this->transfers[this->ntransfers];
2323 if (this->ntransfers == GPMI_MAX_TRANSFERS)
2329 static struct dma_async_tx_descriptor *gpmi_chain_command(
2330 struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr)
2332 struct dma_chan *channel = get_dma_chan(this);
2333 struct dma_async_tx_descriptor *desc;
2334 struct gpmi_transfer *transfer;
2335 int chip = this->nand.cur_cs;
2338 /* [1] send out the PIO words */
2339 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
2340 | BM_GPMI_CTRL0_WORD_LENGTH
2341 | BF_GPMI_CTRL0_CS(chip, this)
2342 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2343 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE)
2344 | BM_GPMI_CTRL0_ADDRESS_INCREMENT
2345 | BF_GPMI_CTRL0_XFER_COUNT(naddr + 1);
2348 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2353 transfer = get_next_transfer(this);
2357 transfer->cmdbuf[0] = cmd;
2359 memcpy(&transfer->cmdbuf[1], addr, naddr);
2361 sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1);
2362 dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE);
2364 transfer->direction = DMA_TO_DEVICE;
2366 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV,
2367 MXS_DMA_CTRL_WAIT4END);
2371 static struct dma_async_tx_descriptor *gpmi_chain_wait_ready(
2372 struct gpmi_nand_data *this)
2374 struct dma_chan *channel = get_dma_chan(this);
2377 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY)
2378 | BM_GPMI_CTRL0_WORD_LENGTH
2379 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2380 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2381 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2382 | BF_GPMI_CTRL0_XFER_COUNT(0);
2385 return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE,
2386 MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY);
2389 static struct dma_async_tx_descriptor *gpmi_chain_data_read(
2390 struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct)
2392 struct dma_async_tx_descriptor *desc;
2393 struct dma_chan *channel = get_dma_chan(this);
2394 struct gpmi_transfer *transfer;
2397 transfer = get_next_transfer(this);
2401 transfer->direction = DMA_FROM_DEVICE;
2403 *direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl,
2406 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)
2407 | BM_GPMI_CTRL0_WORD_LENGTH
2408 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2409 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2410 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2411 | BF_GPMI_CTRL0_XFER_COUNT(raw_len);
2414 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
2415 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE)
2416 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE
2417 | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);
2419 pio[4] = transfer->sgl.dma_address;
2420 pio[5] = this->auxiliary_phys;
2423 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2429 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,
2431 MXS_DMA_CTRL_WAIT4END);
2436 static struct dma_async_tx_descriptor *gpmi_chain_data_write(
2437 struct gpmi_nand_data *this, const void *buf, int raw_len)
2439 struct dma_chan *channel = get_dma_chan(this);
2440 struct dma_async_tx_descriptor *desc;
2441 struct gpmi_transfer *transfer;
2444 transfer = get_next_transfer(this);
2448 transfer->direction = DMA_TO_DEVICE;
2450 prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE);
2452 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
2453 | BM_GPMI_CTRL0_WORD_LENGTH
2454 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2455 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2456 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2457 | BF_GPMI_CTRL0_XFER_COUNT(raw_len);
2460 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
2461 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE)
2462 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE |
2463 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);
2465 pio[4] = transfer->sgl.dma_address;
2466 pio[5] = this->auxiliary_phys;
2469 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2471 (this->bch ? MXS_DMA_CTRL_WAIT4END : 0));
2476 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,
2478 MXS_DMA_CTRL_WAIT4END);
2483 static int gpmi_nfc_exec_op(struct nand_chip *chip,
2484 const struct nand_operation *op,
2487 const struct nand_op_instr *instr;
2488 struct gpmi_nand_data *this = nand_get_controller_data(chip);
2489 struct dma_async_tx_descriptor *desc = NULL;
2490 int i, ret, buf_len = 0, nbufs = 0;
2492 void *buf_read = NULL;
2493 const void *buf_write = NULL;
2494 bool direct = false;
2495 struct completion *dma_completion, *bch_completion;
2501 this->ntransfers = 0;
2502 for (i = 0; i < GPMI_MAX_TRANSFERS; i++)
2503 this->transfers[i].direction = DMA_NONE;
2505 ret = pm_runtime_resume_and_get(this->dev);
2510 * This driver currently supports only one NAND chip. Plus, dies share
2511 * the same configuration. So once timings have been applied on the
2512 * controller side, they will not change anymore. When the time will
2513 * come, the check on must_apply_timings will have to be dropped.
2515 if (this->hw.must_apply_timings) {
2516 this->hw.must_apply_timings = false;
2517 ret = gpmi_nfc_apply_timings(this);
2522 dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs);
2524 for (i = 0; i < op->ninstrs; i++) {
2525 instr = &op->instrs[i];
2527 nand_op_trace(" ", instr);
2529 switch (instr->type) {
2530 case NAND_OP_WAITRDY_INSTR:
2531 desc = gpmi_chain_wait_ready(this);
2533 case NAND_OP_CMD_INSTR:
2534 cmd = instr->ctx.cmd.opcode;
2537 * When this command has an address cycle chain it
2538 * together with the address cycle
2540 if (i + 1 != op->ninstrs &&
2541 op->instrs[i + 1].type == NAND_OP_ADDR_INSTR)
2544 desc = gpmi_chain_command(this, cmd, NULL, 0);
2547 case NAND_OP_ADDR_INSTR:
2548 desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs,
2549 instr->ctx.addr.naddrs);
2551 case NAND_OP_DATA_OUT_INSTR:
2552 buf_write = instr->ctx.data.buf.out;
2553 buf_len = instr->ctx.data.len;
2556 desc = gpmi_chain_data_write(this, buf_write, buf_len);
2559 case NAND_OP_DATA_IN_INSTR:
2560 if (!instr->ctx.data.len)
2562 buf_read = instr->ctx.data.buf.in;
2563 buf_len = instr->ctx.data.len;
2566 desc = gpmi_chain_data_read(this, buf_read, buf_len,
2577 dev_dbg(this->dev, "%s setup done\n", __func__);
2580 dev_err(this->dev, "Multiple data instructions not supported\n");
2586 writel(this->bch_flashlayout0,
2587 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0);
2588 writel(this->bch_flashlayout1,
2589 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1);
2592 desc->callback = dma_irq_callback;
2593 desc->callback_param = this;
2594 dma_completion = &this->dma_done;
2595 bch_completion = NULL;
2597 init_completion(dma_completion);
2599 if (this->bch && buf_read) {
2600 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
2601 this->resources.bch_regs + HW_BCH_CTRL_SET);
2602 bch_completion = &this->bch_done;
2603 init_completion(bch_completion);
2606 dmaengine_submit(desc);
2607 dma_async_issue_pending(get_dma_chan(this));
2609 to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000));
2611 dev_err(this->dev, "DMA timeout, last DMA\n");
2612 gpmi_dump_info(this);
2617 if (this->bch && buf_read) {
2618 to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000));
2620 dev_err(this->dev, "BCH timeout, last DMA\n");
2621 gpmi_dump_info(this);
2627 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
2628 this->resources.bch_regs + HW_BCH_CTRL_CLR);
2629 gpmi_clear_bch(this);
2634 for (i = 0; i < this->ntransfers; i++) {
2635 struct gpmi_transfer *transfer = &this->transfers[i];
2637 if (transfer->direction != DMA_NONE)
2638 dma_unmap_sg(this->dev, &transfer->sgl, 1,
2639 transfer->direction);
2642 if (!ret && buf_read && !direct)
2643 memcpy(buf_read, this->data_buffer_dma,
2644 gpmi_raw_len_to_len(this, buf_len));
2649 pm_runtime_mark_last_busy(this->dev);
2650 pm_runtime_put_autosuspend(this->dev);
2655 static const struct nand_controller_ops gpmi_nand_controller_ops = {
2656 .attach_chip = gpmi_nand_attach_chip,
2657 .setup_interface = gpmi_setup_interface,
2658 .exec_op = gpmi_nfc_exec_op,
2661 static int gpmi_nand_init(struct gpmi_nand_data *this)
2663 struct nand_chip *chip = &this->nand;
2664 struct mtd_info *mtd = nand_to_mtd(chip);
2667 /* init the MTD data structures */
2668 mtd->name = "gpmi-nand";
2669 mtd->dev.parent = this->dev;
2671 /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */
2672 nand_set_controller_data(chip, this);
2673 nand_set_flash_node(chip, this->pdev->dev.of_node);
2674 chip->legacy.block_markbad = gpmi_block_markbad;
2675 chip->badblock_pattern = &gpmi_bbt_descr;
2676 chip->options |= NAND_NO_SUBPAGE_WRITE;
2678 /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */
2679 this->swap_block_mark = !GPMI_IS_MX23(this);
2682 * Allocate a temporary DMA buffer for reading ID in the
2683 * nand_scan_ident().
2685 this->bch_geometry.payload_size = 1024;
2686 this->bch_geometry.auxiliary_size = 128;
2687 ret = gpmi_alloc_dma_buffer(this);
2691 nand_controller_init(&this->base);
2692 this->base.ops = &gpmi_nand_controller_ops;
2693 chip->controller = &this->base;
2695 ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1);
2699 ret = nand_boot_init(this);
2701 goto err_nand_cleanup;
2702 ret = nand_create_bbt(chip);
2704 goto err_nand_cleanup;
2706 ret = mtd_device_register(mtd, NULL, 0);
2708 goto err_nand_cleanup;
2714 gpmi_free_dma_buffer(this);
2718 static const struct of_device_id gpmi_nand_id_table[] = {
2719 { .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, },
2720 { .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, },
2721 { .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, },
2722 { .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, },
2723 { .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,},
2726 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table);
2728 static int gpmi_nand_probe(struct platform_device *pdev)
2730 struct gpmi_nand_data *this;
2733 this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL);
2737 this->devdata = of_device_get_match_data(&pdev->dev);
2738 platform_set_drvdata(pdev, this);
2740 this->dev = &pdev->dev;
2742 ret = acquire_resources(this);
2744 goto exit_acquire_resources;
2746 ret = __gpmi_enable_clk(this, true);
2748 goto exit_acquire_resources;
2750 pm_runtime_set_autosuspend_delay(&pdev->dev, 500);
2751 pm_runtime_use_autosuspend(&pdev->dev);
2752 pm_runtime_set_active(&pdev->dev);
2753 pm_runtime_enable(&pdev->dev);
2754 pm_runtime_get_sync(&pdev->dev);
2756 ret = gpmi_init(this);
2760 ret = gpmi_nand_init(this);
2764 pm_runtime_mark_last_busy(&pdev->dev);
2765 pm_runtime_put_autosuspend(&pdev->dev);
2767 dev_info(this->dev, "driver registered.\n");
2772 pm_runtime_put(&pdev->dev);
2773 pm_runtime_disable(&pdev->dev);
2774 release_resources(this);
2775 exit_acquire_resources:
2780 static void gpmi_nand_remove(struct platform_device *pdev)
2782 struct gpmi_nand_data *this = platform_get_drvdata(pdev);
2783 struct nand_chip *chip = &this->nand;
2786 pm_runtime_put_sync(&pdev->dev);
2787 pm_runtime_disable(&pdev->dev);
2789 ret = mtd_device_unregister(nand_to_mtd(chip));
2792 gpmi_free_dma_buffer(this);
2793 release_resources(this);
2796 #ifdef CONFIG_PM_SLEEP
2797 static int gpmi_pm_suspend(struct device *dev)
2799 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2801 release_dma_channels(this);
2805 static int gpmi_pm_resume(struct device *dev)
2807 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2810 ret = acquire_dma_channels(this);
2814 /* re-init the GPMI registers */
2815 ret = gpmi_init(this);
2817 dev_err(this->dev, "Error setting GPMI : %d\n", ret);
2821 /* Set flag to get timing setup restored for next exec_op */
2822 if (this->hw.clk_rate)
2823 this->hw.must_apply_timings = true;
2825 /* re-init the BCH registers */
2826 ret = bch_set_geometry(this);
2828 dev_err(this->dev, "Error setting BCH : %d\n", ret);
2834 #endif /* CONFIG_PM_SLEEP */
2836 static int __maybe_unused gpmi_runtime_suspend(struct device *dev)
2838 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2840 return __gpmi_enable_clk(this, false);
2843 static int __maybe_unused gpmi_runtime_resume(struct device *dev)
2845 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2847 return __gpmi_enable_clk(this, true);
2850 static const struct dev_pm_ops gpmi_pm_ops = {
2851 SET_SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume)
2852 SET_RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL)
2855 static struct platform_driver gpmi_nand_driver = {
2857 .name = "gpmi-nand",
2859 .of_match_table = gpmi_nand_id_table,
2861 .probe = gpmi_nand_probe,
2862 .remove_new = gpmi_nand_remove,
2864 module_platform_driver(gpmi_nand_driver);
2866 MODULE_AUTHOR("Freescale Semiconductor, Inc.");
2867 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver");
2868 MODULE_LICENSE("GPL");