drivers/mtd/nand/atmel/pmecc.c

   1 /*
   2  * Copyright 2017 ATMEL
   3  * Copyright 2017 Free Electrons
   4  *
   5  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
   6  *
   7  * Derived from the atmel_nand.c driver which contained the following
   8  * copyrights:
   9  *
  10  *   Copyright 2003 Rick Bronson
  11  *
  12  *   Derived from drivers/mtd/nand/autcpu12.c
  13  *      Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
  14  *
  15  *   Derived from drivers/mtd/spia.c
  16  *      Copyright 2000 Steven J. Hill (sjhill@cotw.com)
  17  *
  18  *   Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
  19  *      Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
  20  *
  21  *   Derived from Das U-Boot source code
  22  *      (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
  23  *      Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
  24  *
  25  *   Add Programmable Multibit ECC support for various AT91 SoC
  26  *      Copyright 2012 ATMEL, Hong Xu
  27  *
  28  *   Add Nand Flash Controller support for SAMA5 SoC
  29  *      Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
  30  *
  31  * This program is free software; you can redistribute it and/or modify
  32  * it under the terms of the GNU General Public License version 2 as
  33  * published by the Free Software Foundation.
  34  *
  35  * The PMECC is an hardware assisted BCH engine, which means part of the
  36  * ECC algorithm is left to the software. The hardware/software repartition
  37  * is explained in the "PMECC Controller Functional Description" chapter in
  38  * Atmel datasheets, and some of the functions in this file are directly
  39  * implementing the algorithms described in the "Software Implementation"
  40  * sub-section.
  41  *
  42  * TODO: it seems that the software BCH implementation in lib/bch.c is already
  43  * providing some of the logic we are implementing here. It would be smart
  44  * to expose the needed lib/bch.c helpers/functions and re-use them here.
  45  */
  46
  47 #include <linux/genalloc.h>
  48 #include <linux/iopoll.h>
  49 #include <linux/module.h>
  50 #include <linux/mtd/nand.h>
  51 #include <linux/of_irq.h>
  52 #include <linux/of_platform.h>
  53 #include <linux/platform_device.h>
  54 #include <linux/slab.h>
  55
  56 #include "pmecc.h"
  57
  58 /* Galois field dimension */
  59 #define PMECC_GF_DIMENSION_13                   13
  60 #define PMECC_GF_DIMENSION_14                   14
  61
  62 /* Primitive Polynomial used by PMECC */
  63 #define PMECC_GF_13_PRIMITIVE_POLY              0x201b
  64 #define PMECC_GF_14_PRIMITIVE_POLY              0x4443
  65
  66 #define PMECC_LOOKUP_TABLE_SIZE_512             0x2000
  67 #define PMECC_LOOKUP_TABLE_SIZE_1024            0x4000
  68
  69 /* Time out value for reading PMECC status register */
  70 #define PMECC_MAX_TIMEOUT_MS                    100
  71
  72 /* PMECC Register Definitions */
  73 #define ATMEL_PMECC_CFG                         0x0
  74 #define PMECC_CFG_BCH_STRENGTH(x)               (x)
  75 #define PMECC_CFG_BCH_STRENGTH_MASK             GENMASK(2, 0)
  76 #define PMECC_CFG_SECTOR512                     (0 << 4)
  77 #define PMECC_CFG_SECTOR1024                    (1 << 4)
  78 #define PMECC_CFG_NSECTORS(x)                   ((fls(x) - 1) << 8)
  79 #define PMECC_CFG_READ_OP                       (0 << 12)
  80 #define PMECC_CFG_WRITE_OP                      (1 << 12)
  81 #define PMECC_CFG_SPARE_ENABLE                  BIT(16)
  82 #define PMECC_CFG_AUTO_ENABLE                   BIT(20)
  83
  84 #define ATMEL_PMECC_SAREA                       0x4
  85 #define ATMEL_PMECC_SADDR                       0x8
  86 #define ATMEL_PMECC_EADDR                       0xc
  87
  88 #define ATMEL_PMECC_CLK                         0x10
  89 #define PMECC_CLK_133MHZ                        (2 << 0)
  90
  91 #define ATMEL_PMECC_CTRL                        0x14
  92 #define PMECC_CTRL_RST                          BIT(0)
  93 #define PMECC_CTRL_DATA                         BIT(1)
  94 #define PMECC_CTRL_USER                         BIT(2)
  95 #define PMECC_CTRL_ENABLE                       BIT(4)
  96 #define PMECC_CTRL_DISABLE                      BIT(5)
  97
  98 #define ATMEL_PMECC_SR                          0x18
  99 #define PMECC_SR_BUSY                           BIT(0)
 100 #define PMECC_SR_ENABLE                         BIT(4)
 101
 102 #define ATMEL_PMECC_IER                         0x1c
 103 #define ATMEL_PMECC_IDR                         0x20
 104 #define ATMEL_PMECC_IMR                         0x24
 105 #define ATMEL_PMECC_ISR                         0x28
 106 #define PMECC_ERROR_INT                         BIT(0)
 107
 108 #define ATMEL_PMECC_ECC(sector, n)              \
 109         ((((sector) + 1) * 0x40) + (n))
 110
 111 #define ATMEL_PMECC_REM(sector, n)              \
 112         ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
 113
 114 /* PMERRLOC Register Definitions */
 115 #define ATMEL_PMERRLOC_ELCFG                    0x0
 116 #define PMERRLOC_ELCFG_SECTOR_512               (0 << 0)
 117 #define PMERRLOC_ELCFG_SECTOR_1024              (1 << 0)
 118 #define PMERRLOC_ELCFG_NUM_ERRORS(n)            ((n) << 16)
 119
 120 #define ATMEL_PMERRLOC_ELPRIM                   0x4
 121 #define ATMEL_PMERRLOC_ELEN                     0x8
 122 #define ATMEL_PMERRLOC_ELDIS                    0xc
 123 #define PMERRLOC_DISABLE                        BIT(0)
 124
 125 #define ATMEL_PMERRLOC_ELSR                     0x10
 126 #define PMERRLOC_ELSR_BUSY                      BIT(0)
 127
 128 #define ATMEL_PMERRLOC_ELIER                    0x14
 129 #define ATMEL_PMERRLOC_ELIDR                    0x18
 130 #define ATMEL_PMERRLOC_ELIMR                    0x1c
 131 #define ATMEL_PMERRLOC_ELISR                    0x20
 132 #define PMERRLOC_ERR_NUM_MASK                   GENMASK(12, 8)
 133 #define PMERRLOC_CALC_DONE                      BIT(0)
 134
 135 #define ATMEL_PMERRLOC_SIGMA(x)                 (((x) * 0x4) + 0x28)
 136
 137 #define ATMEL_PMERRLOC_EL(offs, x)              (((x) * 0x4) + (offs))
 138
 139 struct atmel_pmecc_gf_tables {
 140         u16 *alpha_to;
 141         u16 *index_of;
 142 };
 143
 144 struct atmel_pmecc_caps {
 145         const int *strengths;
 146         int nstrengths;
 147         int el_offset;
 148         bool correct_erased_chunks;
 149 };
 150
 151 struct atmel_pmecc {
 152         struct device *dev;
 153         const struct atmel_pmecc_caps *caps;
 154
 155         struct {
 156                 void __iomem *base;
 157                 void __iomem *errloc;
 158         } regs;
 159
 160         struct mutex lock;
 161 };
 162
 163 struct atmel_pmecc_user_conf_cache {
 164         u32 cfg;
 165         u32 sarea;
 166         u32 saddr;
 167         u32 eaddr;
 168 };
 169
 170 struct atmel_pmecc_user {
 171         struct atmel_pmecc_user_conf_cache cache;
 172         struct atmel_pmecc *pmecc;
 173         const struct atmel_pmecc_gf_tables *gf_tables;
 174         int eccbytes;
 175         s16 *partial_syn;
 176         s16 *si;
 177         s16 *lmu;
 178         s16 *smu;
 179         s32 *mu;
 180         s32 *dmu;
 181         s32 *delta;
 182         u32 isr;
 183 };
 184
 185 static DEFINE_MUTEX(pmecc_gf_tables_lock);
 186 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
 187 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
 188
 189 static inline int deg(unsigned int poly)
 190 {
 191         /* polynomial degree is the most-significant bit index */
 192         return fls(poly) - 1;
 193 }
 194
 195 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
 196                                        struct atmel_pmecc_gf_tables *gf_tables)
 197 {
 198         unsigned int i, x = 1;
 199         const unsigned int k = BIT(deg(poly));
 200         unsigned int nn = BIT(mm) - 1;
 201
 202         /* primitive polynomial must be of degree m */
 203         if (k != (1u << mm))
 204                 return -EINVAL;
 205
 206         for (i = 0; i < nn; i++) {
 207                 gf_tables->alpha_to[i] = x;
 208                 gf_tables->index_of[x] = i;
 209                 if (i && (x == 1))
 210                         /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
 211                         return -EINVAL;
 212                 x <<= 1;
 213                 if (x & k)
 214                         x ^= poly;
 215         }
 216         gf_tables->alpha_to[nn] = 1;
 217         gf_tables->index_of[0] = 0;
 218
 219         return 0;
 220 }
 221
 222 static const struct atmel_pmecc_gf_tables *
 223 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
 224 {
 225         struct atmel_pmecc_gf_tables *gf_tables;
 226         unsigned int poly, degree, table_size;
 227         int ret;
 228
 229         if (req->ecc.sectorsize == 512) {
 230                 degree = PMECC_GF_DIMENSION_13;
 231                 poly = PMECC_GF_13_PRIMITIVE_POLY;
 232                 table_size = PMECC_LOOKUP_TABLE_SIZE_512;
 233         } else {
 234                 degree = PMECC_GF_DIMENSION_14;
 235                 poly = PMECC_GF_14_PRIMITIVE_POLY;
 236                 table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
 237         }
 238
 239         gf_tables = kzalloc(sizeof(*gf_tables) +
 240                             (2 * table_size * sizeof(u16)),
 241                             GFP_KERNEL);
 242         if (!gf_tables)
 243                 return ERR_PTR(-ENOMEM);
 244
 245         gf_tables->alpha_to = (void *)(gf_tables + 1);
 246         gf_tables->index_of = gf_tables->alpha_to + table_size;
 247
 248         ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
 249         if (ret) {
 250                 kfree(gf_tables);
 251                 return ERR_PTR(ret);
 252         }
 253
 254         return gf_tables;
 255 }
 256
 257 static const struct atmel_pmecc_gf_tables *
 258 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
 259 {
 260         const struct atmel_pmecc_gf_tables **gf_tables, *ret;
 261
 262         mutex_lock(&pmecc_gf_tables_lock);
 263         if (req->ecc.sectorsize == 512)
 264                 gf_tables = &pmecc_gf_tables_512;
 265         else
 266                 gf_tables = &pmecc_gf_tables_1024;
 267
 268         ret = *gf_tables;
 269
 270         if (!ret) {
 271                 ret = atmel_pmecc_create_gf_tables(req);
 272                 if (!IS_ERR(ret))
 273                         *gf_tables = ret;
 274         }
 275         mutex_unlock(&pmecc_gf_tables_lock);
 276
 277         return ret;
 278 }
 279
 280 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
 281                                         struct atmel_pmecc_user_req *req)
 282 {
 283         int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
 284
 285         if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
 286                 return -EINVAL;
 287
 288         if (req->ecc.ooboffset >= 0 &&
 289             req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
 290                 return -EINVAL;
 291
 292         if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
 293                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
 294                         return -EINVAL;
 295
 296                 if (req->pagesize > 512)
 297                         req->ecc.sectorsize = 1024;
 298                 else
 299                         req->ecc.sectorsize = 512;
 300         }
 301
 302         if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
 303                 return -EINVAL;
 304
 305         if (req->pagesize % req->ecc.sectorsize)
 306                 return -EINVAL;
 307
 308         req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
 309
 310         max_eccbytes = req->ecc.bytes;
 311
 312         for (i = 0; i < pmecc->caps->nstrengths; i++) {
 313                 int nbytes, strength = pmecc->caps->strengths[i];
 314
 315                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
 316                     strength < req->ecc.strength)
 317                         continue;
 318
 319                 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
 320                                       8);
 321                 nbytes *= req->ecc.nsectors;
 322
 323                 if (nbytes > max_eccbytes)
 324                         break;
 325
 326                 eccstrength = strength;
 327                 eccbytes = nbytes;
 328
 329                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
 330                         break;
 331         }
 332
 333         if (!eccstrength)
 334                 return -EINVAL;
 335
 336         req->ecc.bytes = eccbytes;
 337         req->ecc.strength = eccstrength;
 338
 339         if (req->ecc.ooboffset < 0)
 340                 req->ecc.ooboffset = req->oobsize - eccbytes;
 341
 342         return 0;
 343 }
 344
 345 struct atmel_pmecc_user *
 346 atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
 347                         struct atmel_pmecc_user_req *req)
 348 {
 349         struct atmel_pmecc_user *user;
 350         const struct atmel_pmecc_gf_tables *gf_tables;
 351         int strength, size, ret;
 352
 353         ret = atmel_pmecc_prepare_user_req(pmecc, req);
 354         if (ret)
 355                 return ERR_PTR(ret);
 356
 357         size = sizeof(*user);
 358         size = ALIGN(size, sizeof(u16));
 359         /* Reserve space for partial_syn, si and smu */
 360         size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
 361                 (2 + req->ecc.strength + 2);
 362         /* Reserve space for lmu. */
 363         size += (req->ecc.strength + 1) * sizeof(u16);
 364         /* Reserve space for mu, dmu and delta. */
 365         size = ALIGN(size, sizeof(s32));
 366         size += (req->ecc.strength + 1) * sizeof(s32);
 367
 368         user = kzalloc(size, GFP_KERNEL);
 369         if (!user)
 370                 return ERR_PTR(-ENOMEM);
 371
 372         user->pmecc = pmecc;
 373
 374         user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
 375         user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
 376         user->lmu = user->si + ((2 * req->ecc.strength) + 1);
 377         user->smu = user->lmu + (req->ecc.strength + 1);
 378         user->mu = (s32 *)PTR_ALIGN(user->smu +
 379                                     (((2 * req->ecc.strength) + 1) *
 380                                      (req->ecc.strength + 2)),
 381                                     sizeof(s32));
 382         user->dmu = user->mu + req->ecc.strength + 1;
 383         user->delta = user->dmu + req->ecc.strength + 1;
 384
 385         gf_tables = atmel_pmecc_get_gf_tables(req);
 386         if (IS_ERR(gf_tables)) {
 387                 kfree(user);
 388                 return ERR_CAST(gf_tables);
 389         }
 390
 391         user->gf_tables = gf_tables;
 392
 393         user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
 394
 395         for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
 396                 if (pmecc->caps->strengths[strength] == req->ecc.strength)
 397                         break;
 398         }
 399
 400         user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
 401                           PMECC_CFG_NSECTORS(req->ecc.nsectors);
 402
 403         if (req->ecc.sectorsize == 1024)
 404                 user->cache.cfg |= PMECC_CFG_SECTOR1024;
 405
 406         user->cache.sarea = req->oobsize - 1;
 407         user->cache.saddr = req->ecc.ooboffset;
 408         user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
 409
 410         return user;
 411 }
 412 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
 413
 414 void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user)
 415 {
 416         kfree(user);
 417 }
 418 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user);
 419
 420 static int get_strength(struct atmel_pmecc_user *user)
 421 {
 422         const int *strengths = user->pmecc->caps->strengths;
 423
 424         return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
 425 }
 426
 427 static int get_sectorsize(struct atmel_pmecc_user *user)
 428 {
 429         return user->cache.cfg & PMECC_LOOKUP_TABLE_SIZE_1024 ? 1024 : 512;
 430 }
 431
 432 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
 433 {
 434         int strength = get_strength(user);
 435         u32 value;
 436         int i;
 437
 438         /* Fill odd syndromes */
 439         for (i = 0; i < strength; i++) {
 440                 value = readl_relaxed(user->pmecc->regs.base +
 441                                       ATMEL_PMECC_REM(sector, i / 2));
 442                 if (i & 1)
 443                         value >>= 16;
 444
 445                 user->partial_syn[(2 * i) + 1] = value;
 446         }
 447 }
 448
 449 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
 450 {
 451         int degree = get_sectorsize(user) == 512 ? 13 : 14;
 452         int cw_len = BIT(degree) - 1;
 453         int strength = get_strength(user);
 454         s16 *alpha_to = user->gf_tables->alpha_to;
 455         s16 *index_of = user->gf_tables->index_of;
 456         s16 *partial_syn = user->partial_syn;
 457         s16 *si;
 458         int i, j;
 459
 460         /*
 461          * si[] is a table that holds the current syndrome value,
 462          * an element of that table belongs to the field
 463          */
 464         si = user->si;
 465
 466         memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
 467
 468         /* Computation 2t syndromes based on S(x) */
 469         /* Odd syndromes */
 470         for (i = 1; i < 2 * strength; i += 2) {
 471                 for (j = 0; j < degree; j++) {
 472                         if (partial_syn[i] & BIT(j))
 473                                 si[i] = alpha_to[i * j] ^ si[i];
 474                 }
 475         }
 476         /* Even syndrome = (Odd syndrome) ** 2 */
 477         for (i = 2, j = 1; j <= strength; i = ++j << 1) {
 478                 if (si[j] == 0) {
 479                         si[i] = 0;
 480                 } else {
 481                         s16 tmp;
 482
 483                         tmp = index_of[si[j]];
 484                         tmp = (tmp * 2) % cw_len;
 485                         si[i] = alpha_to[tmp];
 486                 }
 487         }
 488 }
 489
 490 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
 491 {
 492         s16 *lmu = user->lmu;
 493         s16 *si = user->si;
 494         s32 *mu = user->mu;
 495         s32 *dmu = user->dmu;
 496         s32 *delta = user->delta;
 497         int degree = get_sectorsize(user) == 512 ? 13 : 14;
 498         int cw_len = BIT(degree) - 1;
 499         int strength = get_strength(user);
 500         int num = 2 * strength + 1;
 501         s16 *index_of = user->gf_tables->index_of;
 502         s16 *alpha_to = user->gf_tables->alpha_to;
 503         int i, j, k;
 504         u32 dmu_0_count, tmp;
 505         s16 *smu = user->smu;
 506
 507         /* index of largest delta */
 508         int ro;
 509         int largest;
 510         int diff;
 511
 512         dmu_0_count = 0;
 513
 514         /* First Row */
 515
 516         /* Mu */
 517         mu[0] = -1;
 518
 519         memset(smu, 0, sizeof(s16) * num);
 520         smu[0] = 1;
 521
 522         /* discrepancy set to 1 */
 523         dmu[0] = 1;
 524         /* polynom order set to 0 */
 525         lmu[0] = 0;
 526         delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
 527
 528         /* Second Row */
 529
 530         /* Mu */
 531         mu[1] = 0;
 532         /* Sigma(x) set to 1 */
 533         memset(&smu[num], 0, sizeof(s16) * num);
 534         smu[num] = 1;
 535
 536         /* discrepancy set to S1 */
 537         dmu[1] = si[1];
 538
 539         /* polynom order set to 0 */
 540         lmu[1] = 0;
 541
 542         delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
 543
 544         /* Init the Sigma(x) last row */
 545         memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
 546
 547         for (i = 1; i <= strength; i++) {
 548                 mu[i + 1] = i << 1;
 549                 /* Begin Computing Sigma (Mu+1) and L(mu) */
 550                 /* check if discrepancy is set to 0 */
 551                 if (dmu[i] == 0) {
 552                         dmu_0_count++;
 553
 554                         tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
 555                         if ((strength - (lmu[i] >> 1) - 1) & 0x1)
 556                                 tmp += 2;
 557                         else
 558                                 tmp += 1;
 559
 560                         if (dmu_0_count == tmp) {
 561                                 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
 562                                         smu[(strength + 1) * num + j] =
 563                                                         smu[i * num + j];
 564
 565                                 lmu[strength + 1] = lmu[i];
 566                                 return;
 567                         }
 568
 569                         /* copy polynom */
 570                         for (j = 0; j <= lmu[i] >> 1; j++)
 571                                 smu[(i + 1) * num + j] = smu[i * num + j];
 572
 573                         /* copy previous polynom order to the next */
 574                         lmu[i + 1] = lmu[i];
 575                 } else {
 576                         ro = 0;
 577                         largest = -1;
 578                         /* find largest delta with dmu != 0 */
 579                         for (j = 0; j < i; j++) {
 580                                 if ((dmu[j]) && (delta[j] > largest)) {
 581                                         largest = delta[j];
 582                                         ro = j;
 583                                 }
 584                         }
 585
 586                         /* compute difference */
 587                         diff = (mu[i] - mu[ro]);
 588
 589                         /* Compute degree of the new smu polynomial */
 590                         if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
 591                                 lmu[i + 1] = lmu[i];
 592                         else
 593                                 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
 594
 595                         /* Init smu[i+1] with 0 */
 596                         for (k = 0; k < num; k++)
 597                                 smu[(i + 1) * num + k] = 0;
 598
 599                         /* Compute smu[i+1] */
 600                         for (k = 0; k <= lmu[ro] >> 1; k++) {
 601                                 s16 a, b, c;
 602
 603                                 if (!(smu[ro * num + k] && dmu[i]))
 604                                         continue;
 605
 606                                 a = index_of[dmu[i]];
 607                                 b = index_of[dmu[ro]];
 608                                 c = index_of[smu[ro * num + k]];
 609                                 tmp = a + (cw_len - b) + c;
 610                                 a = alpha_to[tmp % cw_len];
 611                                 smu[(i + 1) * num + (k + diff)] = a;
 612                         }
 613
 614                         for (k = 0; k <= lmu[i] >> 1; k++)
 615                                 smu[(i + 1) * num + k] ^= smu[i * num + k];
 616                 }
 617
 618                 /* End Computing Sigma (Mu+1) and L(mu) */
 619                 /* In either case compute delta */
 620                 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
 621
 622                 /* Do not compute discrepancy for the last iteration */
 623                 if (i >= strength)
 624                         continue;
 625
 626                 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
 627                         tmp = 2 * (i - 1);
 628                         if (k == 0) {
 629                                 dmu[i + 1] = si[tmp + 3];
 630                         } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
 631                                 s16 a, b, c;
 632
 633                                 a = index_of[smu[(i + 1) * num + k]];
 634                                 b = si[2 * (i - 1) + 3 - k];
 635                                 c = index_of[b];
 636                                 tmp = a + c;
 637                                 tmp %= cw_len;
 638                                 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
 639                         }
 640                 }
 641         }
 642 }
 643
 644 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
 645 {
 646         int sector_size = get_sectorsize(user);
 647         int degree = sector_size == 512 ? 13 : 14;
 648         struct atmel_pmecc *pmecc = user->pmecc;
 649         int strength = get_strength(user);
 650         int ret, roots_nbr, i, err_nbr = 0;
 651         int num = (2 * strength) + 1;
 652         s16 *smu = user->smu;
 653         u32 val;
 654
 655         writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
 656
 657         for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
 658                 writel_relaxed(smu[(strength + 1) * num + i],
 659                                pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
 660                 err_nbr++;
 661         }
 662
 663         val = (err_nbr - 1) << 16;
 664         if (sector_size == 1024)
 665                 val |= 1;
 666
 667         writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
 668         writel((sector_size * 8) + (degree * strength),
 669                pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
 670
 671         ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
 672                                          ATMEL_PMERRLOC_ELISR,
 673                                          val, val & PMERRLOC_CALC_DONE, 0,
 674                                          PMECC_MAX_TIMEOUT_MS * 1000);
 675         if (ret) {
 676                 dev_err(pmecc->dev,
 677                         "PMECC: Timeout to calculate error location.\n");
 678                 return ret;
 679         }
 680
 681         roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
 682         /* Number of roots == degree of smu hence <= cap */
 683         if (roots_nbr == user->lmu[strength + 1] >> 1)
 684                 return err_nbr - 1;
 685
 686         /*
 687          * Number of roots does not match the degree of smu
 688          * unable to correct error.
 689          */
 690         return -EBADMSG;
 691 }
 692
 693 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
 694                                void *data, void *ecc)
 695 {
 696         struct atmel_pmecc *pmecc = user->pmecc;
 697         int sectorsize = get_sectorsize(user);
 698         int eccbytes = user->eccbytes;
 699         int i, nerrors;
 700
 701         if (!(user->isr & BIT(sector)))
 702                 return 0;
 703
 704         atmel_pmecc_gen_syndrome(user, sector);
 705         atmel_pmecc_substitute(user);
 706         atmel_pmecc_get_sigma(user);
 707
 708         nerrors = atmel_pmecc_err_location(user);
 709         if (nerrors < 0)
 710                 return nerrors;
 711
 712         for (i = 0; i < nerrors; i++) {
 713                 const char *area;
 714                 int byte, bit;
 715                 u32 errpos;
 716                 u8 *ptr;
 717
 718                 errpos = readl_relaxed(pmecc->regs.errloc +
 719                                 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
 720                 errpos--;
 721
 722                 byte = errpos / 8;
 723                 bit = errpos % 8;
 724
 725                 if (byte < sectorsize) {
 726                         ptr = data + byte;
 727                         area = "data";
 728                 } else if (byte < sectorsize + eccbytes) {
 729                         ptr = ecc + byte - sectorsize;
 730                         area = "ECC";
 731                 } else {
 732                         dev_dbg(pmecc->dev,
 733                                 "Invalid errpos value (%d, max is %d)\n",
 734                                 errpos, (sectorsize + eccbytes) * 8);
 735                         return -EINVAL;
 736                 }
 737
 738                 dev_dbg(pmecc->dev,
 739                         "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
 740                         area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
 741
 742                 *ptr ^= BIT(bit);
 743         }
 744
 745         return nerrors;
 746 }
 747 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
 748
 749 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
 750 {
 751         return user->pmecc->caps->correct_erased_chunks;
 752 }
 753 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
 754
 755 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
 756                                         int sector, void *ecc)
 757 {
 758         struct atmel_pmecc *pmecc = user->pmecc;
 759         u8 *ptr = ecc;
 760         int i;
 761
 762         for (i = 0; i < user->eccbytes; i++)
 763                 ptr[i] = readb_relaxed(pmecc->regs.base +
 764                                        ATMEL_PMECC_ECC(sector, i));
 765 }
 766 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
 767
 768 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
 769 {
 770         struct atmel_pmecc *pmecc = user->pmecc;
 771         u32 cfg;
 772
 773         if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
 774                 dev_err(pmecc->dev, "Bad ECC operation!");
 775                 return -EINVAL;
 776         }
 777
 778         mutex_lock(&user->pmecc->lock);
 779
 780         cfg = user->cache.cfg;
 781         if (op == NAND_ECC_WRITE)
 782                 cfg |= PMECC_CFG_WRITE_OP;
 783         else
 784                 cfg |= PMECC_CFG_AUTO_ENABLE;
 785
 786         writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
 787         writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
 788         writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
 789         writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
 790
 791         writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 792         writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
 793
 794         return 0;
 795 }
 796 EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
 797
 798 void atmel_pmecc_disable(struct atmel_pmecc_user *user)
 799 {
 800         struct atmel_pmecc *pmecc = user->pmecc;
 801
 802         writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
 803         writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 804         mutex_unlock(&user->pmecc->lock);
 805 }
 806 EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
 807
 808 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
 809 {
 810         struct atmel_pmecc *pmecc = user->pmecc;
 811         u32 status;
 812         int ret;
 813
 814         ret = readl_relaxed_poll_timeout(pmecc->regs.base +
 815                                          ATMEL_PMECC_SR,
 816                                          status, !(status & PMECC_SR_BUSY), 0,
 817                                          PMECC_MAX_TIMEOUT_MS * 1000);
 818         if (ret) {
 819                 dev_err(pmecc->dev,
 820                         "Timeout while waiting for PMECC ready.\n");
 821                 return ret;
 822         }
 823
 824         user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
 825
 826         return 0;
 827 }
 828 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
 829
 830 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,
 831                                         const struct atmel_pmecc_caps *caps,
 832                                         int pmecc_res_idx, int errloc_res_idx)
 833 {
 834         struct device *dev = &pdev->dev;
 835         struct atmel_pmecc *pmecc;
 836         struct resource *res;
 837
 838         pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
 839         if (!pmecc)
 840                 return ERR_PTR(-ENOMEM);
 841
 842         pmecc->caps = caps;
 843         pmecc->dev = dev;
 844         mutex_init(&pmecc->lock);
 845
 846         res = platform_get_resource(pdev, IORESOURCE_MEM, pmecc_res_idx);
 847         pmecc->regs.base = devm_ioremap_resource(dev, res);
 848         if (IS_ERR(pmecc->regs.base))
 849                 return ERR_CAST(pmecc->regs.base);
 850
 851         res = platform_get_resource(pdev, IORESOURCE_MEM, errloc_res_idx);
 852         pmecc->regs.errloc = devm_ioremap_resource(dev, res);
 853         if (IS_ERR(pmecc->regs.errloc))
 854                 return ERR_CAST(pmecc->regs.errloc);
 855
 856         /* Disable all interrupts before registering the PMECC handler. */
 857         writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
 858
 859         /* Reset the ECC engine */
 860         writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
 861         writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 862
 863         return pmecc;
 864 }
 865
 866 static void devm_atmel_pmecc_put(struct device *dev, void *res)
 867 {
 868         struct atmel_pmecc **pmecc = res;
 869
 870         put_device((*pmecc)->dev);
 871 }
 872
 873 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,
 874                                                    struct device_node *np)
 875 {
 876         struct platform_device *pdev;
 877         struct atmel_pmecc *pmecc, **ptr;
 878
 879         pdev = of_find_device_by_node(np);
 880         if (!pdev || !platform_get_drvdata(pdev))
 881                 return ERR_PTR(-EPROBE_DEFER);
 882
 883         ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
 884         if (!ptr)
 885                 return ERR_PTR(-ENOMEM);
 886
 887         get_device(&pdev->dev);
 888         pmecc = platform_get_drvdata(pdev);
 889
 890         *ptr = pmecc;
 891
 892         devres_add(userdev, ptr);
 893
 894         return pmecc;
 895 }
 896
 897 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
 898
 899 static struct atmel_pmecc_caps at91sam9g45_caps = {
 900         .strengths = atmel_pmecc_strengths,
 901         .nstrengths = 5,
 902         .el_offset = 0x8c,
 903 };
 904
 905 static struct atmel_pmecc_caps sama5d4_caps = {
 906         .strengths = atmel_pmecc_strengths,
 907         .nstrengths = 5,
 908         .el_offset = 0x8c,
 909         .correct_erased_chunks = true,
 910 };
 911
 912 static struct atmel_pmecc_caps sama5d2_caps = {
 913         .strengths = atmel_pmecc_strengths,
 914         .nstrengths = 6,
 915         .el_offset = 0xac,
 916         .correct_erased_chunks = true,
 917 };
 918
 919 static const struct of_device_id atmel_pmecc_legacy_match[] = {
 920         { .compatible = "atmel,sama5d4-nand", &sama5d4_caps },
 921         { .compatible = "atmel,sama5d2-nand", &sama5d2_caps },
 922         { /* sentinel */ }
 923 };
 924
 925 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)
 926 {
 927         struct atmel_pmecc *pmecc;
 928         struct device_node *np;
 929
 930         if (!userdev)
 931                 return ERR_PTR(-EINVAL);
 932
 933         if (!userdev->of_node)
 934                 return NULL;
 935
 936         np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);
 937         if (np) {
 938                 pmecc = atmel_pmecc_get_by_node(userdev, np);
 939                 of_node_put(np);
 940         } else {
 941                 /*
 942                  * Support old DT bindings: in this case the PMECC iomem
 943                  * resources are directly defined in the user pdev at position
 944                  * 1 and 2. Extract all relevant information from there.
 945                  */
 946                 struct platform_device *pdev = to_platform_device(userdev);
 947                 const struct atmel_pmecc_caps *caps;
 948                 const struct of_device_id *match;
 949
 950                 /* No PMECC engine available. */
 951                 if (!of_property_read_bool(userdev->of_node,
 952                                            "atmel,has-pmecc"))
 953                         return NULL;
 954
 955                 caps = &at91sam9g45_caps;
 956
 957                 /* Find the caps associated to the NAND dev node. */
 958                 match = of_match_node(atmel_pmecc_legacy_match,
 959                                       userdev->of_node);
 960                 if (match && match->data)
 961                         caps = match->data;
 962
 963                 pmecc = atmel_pmecc_create(pdev, caps, 1, 2);
 964         }
 965
 966         return pmecc;
 967 }
 968 EXPORT_SYMBOL(devm_atmel_pmecc_get);
 969
 970 static const struct of_device_id atmel_pmecc_match[] = {
 971         { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },
 972         { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },
 973         { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },
 974         { /* sentinel */ }
 975 };
 976 MODULE_DEVICE_TABLE(of, atmel_pmecc_match);
 977
 978 static int atmel_pmecc_probe(struct platform_device *pdev)
 979 {
 980         struct device *dev = &pdev->dev;
 981         const struct atmel_pmecc_caps *caps;
 982         struct atmel_pmecc *pmecc;
 983
 984         caps = of_device_get_match_data(&pdev->dev);
 985         if (!caps) {
 986                 dev_err(dev, "Invalid caps\n");
 987                 return -EINVAL;
 988         }
 989
 990         pmecc = atmel_pmecc_create(pdev, caps, 0, 1);
 991         if (IS_ERR(pmecc))
 992                 return PTR_ERR(pmecc);
 993
 994         platform_set_drvdata(pdev, pmecc);
 995
 996         return 0;
 997 }
 998
 999 static struct platform_driver atmel_pmecc_driver = {
1000         .driver = {
1001                 .name = "atmel-pmecc",
1002                 .of_match_table = of_match_ptr(atmel_pmecc_match),
1003         },
1004         .probe = atmel_pmecc_probe,
1005 };
1006 module_platform_driver(atmel_pmecc_driver);
1007
1008 MODULE_LICENSE("GPL");
1009 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1010 MODULE_DESCRIPTION("PMECC engine driver");
1011 MODULE_ALIAS("platform:atmel_pmecc");