1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2017 ATMEL 4 * Copyright 2017 Free Electrons 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 * 8 * Derived from the atmel_nand.c driver which contained the following 9 * copyrights: 10 * 11 * Copyright 2003 Rick Bronson 12 * 13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8) 14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) 15 * 16 * Derived from drivers/mtd/spia.c (removed in v3.8) 17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com) 18 * 19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 20 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 21 * 22 * Derived from Das U-Boot source code 23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) 24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas 25 * 26 * Add Programmable Multibit ECC support for various AT91 SoC 27 * Copyright 2012 ATMEL, Hong Xu 28 * 29 * Add Nand Flash Controller support for SAMA5 SoC 30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) 31 * 32 * The PMECC is an hardware assisted BCH engine, which means part of the 33 * ECC algorithm is left to the software. The hardware/software repartition 34 * is explained in the "PMECC Controller Functional Description" chapter in 35 * Atmel datasheets, and some of the functions in this file are directly 36 * implementing the algorithms described in the "Software Implementation" 37 * sub-section. 38 * 39 * TODO: it seems that the software BCH implementation in lib/bch.c is already 40 * providing some of the logic we are implementing here. It would be smart 41 * to expose the needed lib/bch.c helpers/functions and re-use them here. 42 */ 43 44 #include <linux/genalloc.h> 45 #include <linux/iopoll.h> 46 #include <linux/module.h> 47 #include <linux/mtd/rawnand.h> 48 #include <linux/of_irq.h> 49 #include <linux/of_platform.h> 50 #include <linux/platform_device.h> 51 #include <linux/slab.h> 52 53 #include "pmecc.h" 54 55 /* Galois field dimension */ 56 #define PMECC_GF_DIMENSION_13 13 57 #define PMECC_GF_DIMENSION_14 14 58 59 /* Primitive Polynomial used by PMECC */ 60 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b 61 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443 62 63 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000 64 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000 65 66 /* Time out value for reading PMECC status register */ 67 #define PMECC_MAX_TIMEOUT_MS 100 68 69 /* PMECC Register Definitions */ 70 #define ATMEL_PMECC_CFG 0x0 71 #define PMECC_CFG_BCH_STRENGTH(x) (x) 72 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0) 73 #define PMECC_CFG_SECTOR512 (0 << 4) 74 #define PMECC_CFG_SECTOR1024 (1 << 4) 75 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8) 76 #define PMECC_CFG_READ_OP (0 << 12) 77 #define PMECC_CFG_WRITE_OP (1 << 12) 78 #define PMECC_CFG_SPARE_ENABLE BIT(16) 79 #define PMECC_CFG_AUTO_ENABLE BIT(20) 80 81 #define ATMEL_PMECC_SAREA 0x4 82 #define ATMEL_PMECC_SADDR 0x8 83 #define ATMEL_PMECC_EADDR 0xc 84 85 #define ATMEL_PMECC_CLK 0x10 86 #define PMECC_CLK_133MHZ (2 << 0) 87 88 #define ATMEL_PMECC_CTRL 0x14 89 #define PMECC_CTRL_RST BIT(0) 90 #define PMECC_CTRL_DATA BIT(1) 91 #define PMECC_CTRL_USER BIT(2) 92 #define PMECC_CTRL_ENABLE BIT(4) 93 #define PMECC_CTRL_DISABLE BIT(5) 94 95 #define ATMEL_PMECC_SR 0x18 96 #define PMECC_SR_BUSY BIT(0) 97 #define PMECC_SR_ENABLE BIT(4) 98 99 #define ATMEL_PMECC_IER 0x1c 100 #define ATMEL_PMECC_IDR 0x20 101 #define ATMEL_PMECC_IMR 0x24 102 #define ATMEL_PMECC_ISR 0x28 103 #define PMECC_ERROR_INT BIT(0) 104 105 #define ATMEL_PMECC_ECC(sector, n) \ 106 ((((sector) + 1) * 0x40) + (n)) 107 108 #define ATMEL_PMECC_REM(sector, n) \ 109 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200) 110 111 /* PMERRLOC Register Definitions */ 112 #define ATMEL_PMERRLOC_ELCFG 0x0 113 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0) 114 #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0) 115 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16) 116 117 #define ATMEL_PMERRLOC_ELPRIM 0x4 118 #define ATMEL_PMERRLOC_ELEN 0x8 119 #define ATMEL_PMERRLOC_ELDIS 0xc 120 #define PMERRLOC_DISABLE BIT(0) 121 122 #define ATMEL_PMERRLOC_ELSR 0x10 123 #define PMERRLOC_ELSR_BUSY BIT(0) 124 125 #define ATMEL_PMERRLOC_ELIER 0x14 126 #define ATMEL_PMERRLOC_ELIDR 0x18 127 #define ATMEL_PMERRLOC_ELIMR 0x1c 128 #define ATMEL_PMERRLOC_ELISR 0x20 129 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8) 130 #define PMERRLOC_CALC_DONE BIT(0) 131 132 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28) 133 134 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs)) 135 136 struct atmel_pmecc_gf_tables { 137 u16 *alpha_to; 138 u16 *index_of; 139 }; 140 141 struct atmel_pmecc_caps { 142 const int *strengths; 143 int nstrengths; 144 int el_offset; 145 bool correct_erased_chunks; 146 }; 147 148 struct atmel_pmecc { 149 struct device *dev; 150 const struct atmel_pmecc_caps *caps; 151 152 struct { 153 void __iomem *base; 154 void __iomem *errloc; 155 } regs; 156 157 struct mutex lock; 158 }; 159 160 struct atmel_pmecc_user_conf_cache { 161 u32 cfg; 162 u32 sarea; 163 u32 saddr; 164 u32 eaddr; 165 }; 166 167 struct atmel_pmecc_user { 168 struct atmel_pmecc_user_conf_cache cache; 169 struct atmel_pmecc *pmecc; 170 const struct atmel_pmecc_gf_tables *gf_tables; 171 int eccbytes; 172 s16 *partial_syn; 173 s16 *si; 174 s16 *lmu; 175 s16 *smu; 176 s32 *mu; 177 s32 *dmu; 178 s32 *delta; 179 u32 isr; 180 }; 181 182 static DEFINE_MUTEX(pmecc_gf_tables_lock); 183 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512; 184 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024; 185 186 static inline int deg(unsigned int poly) 187 { 188 /* polynomial degree is the most-significant bit index */ 189 return fls(poly) - 1; 190 } 191 192 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly, 193 struct atmel_pmecc_gf_tables *gf_tables) 194 { 195 unsigned int i, x = 1; 196 const unsigned int k = BIT(deg(poly)); 197 unsigned int nn = BIT(mm) - 1; 198 199 /* primitive polynomial must be of degree m */ 200 if (k != (1u << mm)) 201 return -EINVAL; 202 203 for (i = 0; i < nn; i++) { 204 gf_tables->alpha_to[i] = x; 205 gf_tables->index_of[x] = i; 206 if (i && (x == 1)) 207 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */ 208 return -EINVAL; 209 x <<= 1; 210 if (x & k) 211 x ^= poly; 212 } 213 gf_tables->alpha_to[nn] = 1; 214 gf_tables->index_of[0] = 0; 215 216 return 0; 217 } 218 219 static const struct atmel_pmecc_gf_tables * 220 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req) 221 { 222 struct atmel_pmecc_gf_tables *gf_tables; 223 unsigned int poly, degree, table_size; 224 int ret; 225 226 if (req->ecc.sectorsize == 512) { 227 degree = PMECC_GF_DIMENSION_13; 228 poly = PMECC_GF_13_PRIMITIVE_POLY; 229 table_size = PMECC_LOOKUP_TABLE_SIZE_512; 230 } else { 231 degree = PMECC_GF_DIMENSION_14; 232 poly = PMECC_GF_14_PRIMITIVE_POLY; 233 table_size = PMECC_LOOKUP_TABLE_SIZE_1024; 234 } 235 236 gf_tables = kzalloc(sizeof(*gf_tables) + 237 (2 * table_size * sizeof(u16)), 238 GFP_KERNEL); 239 if (!gf_tables) 240 return ERR_PTR(-ENOMEM); 241 242 gf_tables->alpha_to = (void *)(gf_tables + 1); 243 gf_tables->index_of = gf_tables->alpha_to + table_size; 244 245 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables); 246 if (ret) { 247 kfree(gf_tables); 248 return ERR_PTR(ret); 249 } 250 251 return gf_tables; 252 } 253 254 static const struct atmel_pmecc_gf_tables * 255 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req) 256 { 257 const struct atmel_pmecc_gf_tables **gf_tables, *ret; 258 259 mutex_lock(&pmecc_gf_tables_lock); 260 if (req->ecc.sectorsize == 512) 261 gf_tables = &pmecc_gf_tables_512; 262 else 263 gf_tables = &pmecc_gf_tables_1024; 264 265 ret = *gf_tables; 266 267 if (!ret) { 268 ret = atmel_pmecc_create_gf_tables(req); 269 if (!IS_ERR(ret)) 270 *gf_tables = ret; 271 } 272 mutex_unlock(&pmecc_gf_tables_lock); 273 274 return ret; 275 } 276 277 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc, 278 struct atmel_pmecc_user_req *req) 279 { 280 int i, max_eccbytes, eccbytes = 0, eccstrength = 0; 281 282 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0) 283 return -EINVAL; 284 285 if (req->ecc.ooboffset >= 0 && 286 req->ecc.ooboffset + req->ecc.bytes > req->oobsize) 287 return -EINVAL; 288 289 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) { 290 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 291 return -EINVAL; 292 293 if (req->pagesize > 512) 294 req->ecc.sectorsize = 1024; 295 else 296 req->ecc.sectorsize = 512; 297 } 298 299 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024) 300 return -EINVAL; 301 302 if (req->pagesize % req->ecc.sectorsize) 303 return -EINVAL; 304 305 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize; 306 307 max_eccbytes = req->ecc.bytes; 308 309 for (i = 0; i < pmecc->caps->nstrengths; i++) { 310 int nbytes, strength = pmecc->caps->strengths[i]; 311 312 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH && 313 strength < req->ecc.strength) 314 continue; 315 316 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize), 317 8); 318 nbytes *= req->ecc.nsectors; 319 320 if (nbytes > max_eccbytes) 321 break; 322 323 eccstrength = strength; 324 eccbytes = nbytes; 325 326 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 327 break; 328 } 329 330 if (!eccstrength) 331 return -EINVAL; 332 333 req->ecc.bytes = eccbytes; 334 req->ecc.strength = eccstrength; 335 336 if (req->ecc.ooboffset < 0) 337 req->ecc.ooboffset = req->oobsize - eccbytes; 338 339 return 0; 340 } 341 342 struct atmel_pmecc_user * 343 atmel_pmecc_create_user(struct atmel_pmecc *pmecc, 344 struct atmel_pmecc_user_req *req) 345 { 346 struct atmel_pmecc_user *user; 347 const struct atmel_pmecc_gf_tables *gf_tables; 348 int strength, size, ret; 349 350 ret = atmel_pmecc_prepare_user_req(pmecc, req); 351 if (ret) 352 return ERR_PTR(ret); 353 354 size = sizeof(*user); 355 size = ALIGN(size, sizeof(u16)); 356 /* Reserve space for partial_syn, si and smu */ 357 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) * 358 (2 + req->ecc.strength + 2); 359 /* Reserve space for lmu. */ 360 size += (req->ecc.strength + 1) * sizeof(u16); 361 /* Reserve space for mu, dmu and delta. */ 362 size = ALIGN(size, sizeof(s32)); 363 size += (req->ecc.strength + 1) * sizeof(s32) * 3; 364 365 user = devm_kzalloc(pmecc->dev, size, GFP_KERNEL); 366 if (!user) 367 return ERR_PTR(-ENOMEM); 368 369 user->pmecc = pmecc; 370 371 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16)); 372 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1); 373 user->lmu = user->si + ((2 * req->ecc.strength) + 1); 374 user->smu = user->lmu + (req->ecc.strength + 1); 375 user->mu = (s32 *)PTR_ALIGN(user->smu + 376 (((2 * req->ecc.strength) + 1) * 377 (req->ecc.strength + 2)), 378 sizeof(s32)); 379 user->dmu = user->mu + req->ecc.strength + 1; 380 user->delta = user->dmu + req->ecc.strength + 1; 381 382 gf_tables = atmel_pmecc_get_gf_tables(req); 383 if (IS_ERR(gf_tables)) 384 return ERR_CAST(gf_tables); 385 386 user->gf_tables = gf_tables; 387 388 user->eccbytes = req->ecc.bytes / req->ecc.nsectors; 389 390 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) { 391 if (pmecc->caps->strengths[strength] == req->ecc.strength) 392 break; 393 } 394 395 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) | 396 PMECC_CFG_NSECTORS(req->ecc.nsectors); 397 398 if (req->ecc.sectorsize == 1024) 399 user->cache.cfg |= PMECC_CFG_SECTOR1024; 400 401 user->cache.sarea = req->oobsize - 1; 402 user->cache.saddr = req->ecc.ooboffset; 403 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1; 404 405 return user; 406 } 407 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user); 408 409 static int get_strength(struct atmel_pmecc_user *user) 410 { 411 const int *strengths = user->pmecc->caps->strengths; 412 413 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK]; 414 } 415 416 static int get_sectorsize(struct atmel_pmecc_user *user) 417 { 418 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512; 419 } 420 421 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector) 422 { 423 int strength = get_strength(user); 424 u32 value; 425 int i; 426 427 /* Fill odd syndromes */ 428 for (i = 0; i < strength; i++) { 429 value = readl_relaxed(user->pmecc->regs.base + 430 ATMEL_PMECC_REM(sector, i / 2)); 431 if (i & 1) 432 value >>= 16; 433 434 user->partial_syn[(2 * i) + 1] = value; 435 } 436 } 437 438 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user) 439 { 440 int degree = get_sectorsize(user) == 512 ? 13 : 14; 441 int cw_len = BIT(degree) - 1; 442 int strength = get_strength(user); 443 s16 *alpha_to = user->gf_tables->alpha_to; 444 s16 *index_of = user->gf_tables->index_of; 445 s16 *partial_syn = user->partial_syn; 446 s16 *si; 447 int i, j; 448 449 /* 450 * si[] is a table that holds the current syndrome value, 451 * an element of that table belongs to the field 452 */ 453 si = user->si; 454 455 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1)); 456 457 /* Computation 2t syndromes based on S(x) */ 458 /* Odd syndromes */ 459 for (i = 1; i < 2 * strength; i += 2) { 460 for (j = 0; j < degree; j++) { 461 if (partial_syn[i] & BIT(j)) 462 si[i] = alpha_to[i * j] ^ si[i]; 463 } 464 } 465 /* Even syndrome = (Odd syndrome) ** 2 */ 466 for (i = 2, j = 1; j <= strength; i = ++j << 1) { 467 if (si[j] == 0) { 468 si[i] = 0; 469 } else { 470 s16 tmp; 471 472 tmp = index_of[si[j]]; 473 tmp = (tmp * 2) % cw_len; 474 si[i] = alpha_to[tmp]; 475 } 476 } 477 } 478 479 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user) 480 { 481 s16 *lmu = user->lmu; 482 s16 *si = user->si; 483 s32 *mu = user->mu; 484 s32 *dmu = user->dmu; 485 s32 *delta = user->delta; 486 int degree = get_sectorsize(user) == 512 ? 13 : 14; 487 int cw_len = BIT(degree) - 1; 488 int strength = get_strength(user); 489 int num = 2 * strength + 1; 490 s16 *index_of = user->gf_tables->index_of; 491 s16 *alpha_to = user->gf_tables->alpha_to; 492 int i, j, k; 493 u32 dmu_0_count, tmp; 494 s16 *smu = user->smu; 495 496 /* index of largest delta */ 497 int ro; 498 int largest; 499 int diff; 500 501 dmu_0_count = 0; 502 503 /* First Row */ 504 505 /* Mu */ 506 mu[0] = -1; 507 508 memset(smu, 0, sizeof(s16) * num); 509 smu[0] = 1; 510 511 /* discrepancy set to 1 */ 512 dmu[0] = 1; 513 /* polynom order set to 0 */ 514 lmu[0] = 0; 515 delta[0] = (mu[0] * 2 - lmu[0]) >> 1; 516 517 /* Second Row */ 518 519 /* Mu */ 520 mu[1] = 0; 521 /* Sigma(x) set to 1 */ 522 memset(&smu[num], 0, sizeof(s16) * num); 523 smu[num] = 1; 524 525 /* discrepancy set to S1 */ 526 dmu[1] = si[1]; 527 528 /* polynom order set to 0 */ 529 lmu[1] = 0; 530 531 delta[1] = (mu[1] * 2 - lmu[1]) >> 1; 532 533 /* Init the Sigma(x) last row */ 534 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num); 535 536 for (i = 1; i <= strength; i++) { 537 mu[i + 1] = i << 1; 538 /* Begin Computing Sigma (Mu+1) and L(mu) */ 539 /* check if discrepancy is set to 0 */ 540 if (dmu[i] == 0) { 541 dmu_0_count++; 542 543 tmp = ((strength - (lmu[i] >> 1) - 1) / 2); 544 if ((strength - (lmu[i] >> 1) - 1) & 0x1) 545 tmp += 2; 546 else 547 tmp += 1; 548 549 if (dmu_0_count == tmp) { 550 for (j = 0; j <= (lmu[i] >> 1) + 1; j++) 551 smu[(strength + 1) * num + j] = 552 smu[i * num + j]; 553 554 lmu[strength + 1] = lmu[i]; 555 return; 556 } 557 558 /* copy polynom */ 559 for (j = 0; j <= lmu[i] >> 1; j++) 560 smu[(i + 1) * num + j] = smu[i * num + j]; 561 562 /* copy previous polynom order to the next */ 563 lmu[i + 1] = lmu[i]; 564 } else { 565 ro = 0; 566 largest = -1; 567 /* find largest delta with dmu != 0 */ 568 for (j = 0; j < i; j++) { 569 if ((dmu[j]) && (delta[j] > largest)) { 570 largest = delta[j]; 571 ro = j; 572 } 573 } 574 575 /* compute difference */ 576 diff = (mu[i] - mu[ro]); 577 578 /* Compute degree of the new smu polynomial */ 579 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff)) 580 lmu[i + 1] = lmu[i]; 581 else 582 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2; 583 584 /* Init smu[i+1] with 0 */ 585 for (k = 0; k < num; k++) 586 smu[(i + 1) * num + k] = 0; 587 588 /* Compute smu[i+1] */ 589 for (k = 0; k <= lmu[ro] >> 1; k++) { 590 s16 a, b, c; 591 592 if (!(smu[ro * num + k] && dmu[i])) 593 continue; 594 595 a = index_of[dmu[i]]; 596 b = index_of[dmu[ro]]; 597 c = index_of[smu[ro * num + k]]; 598 tmp = a + (cw_len - b) + c; 599 a = alpha_to[tmp % cw_len]; 600 smu[(i + 1) * num + (k + diff)] = a; 601 } 602 603 for (k = 0; k <= lmu[i] >> 1; k++) 604 smu[(i + 1) * num + k] ^= smu[i * num + k]; 605 } 606 607 /* End Computing Sigma (Mu+1) and L(mu) */ 608 /* In either case compute delta */ 609 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1; 610 611 /* Do not compute discrepancy for the last iteration */ 612 if (i >= strength) 613 continue; 614 615 for (k = 0; k <= (lmu[i + 1] >> 1); k++) { 616 tmp = 2 * (i - 1); 617 if (k == 0) { 618 dmu[i + 1] = si[tmp + 3]; 619 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) { 620 s16 a, b, c; 621 622 a = index_of[smu[(i + 1) * num + k]]; 623 b = si[2 * (i - 1) + 3 - k]; 624 c = index_of[b]; 625 tmp = a + c; 626 tmp %= cw_len; 627 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1]; 628 } 629 } 630 } 631 } 632 633 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user) 634 { 635 int sector_size = get_sectorsize(user); 636 int degree = sector_size == 512 ? 13 : 14; 637 struct atmel_pmecc *pmecc = user->pmecc; 638 int strength = get_strength(user); 639 int ret, roots_nbr, i, err_nbr = 0; 640 int num = (2 * strength) + 1; 641 s16 *smu = user->smu; 642 u32 val; 643 644 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS); 645 646 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) { 647 writel_relaxed(smu[(strength + 1) * num + i], 648 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i)); 649 err_nbr++; 650 } 651 652 val = (err_nbr - 1) << 16; 653 if (sector_size == 1024) 654 val |= 1; 655 656 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG); 657 writel((sector_size * 8) + (degree * strength), 658 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN); 659 660 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc + 661 ATMEL_PMERRLOC_ELISR, 662 val, val & PMERRLOC_CALC_DONE, 0, 663 PMECC_MAX_TIMEOUT_MS * 1000); 664 if (ret) { 665 dev_err(pmecc->dev, 666 "PMECC: Timeout to calculate error location.\n"); 667 return ret; 668 } 669 670 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8; 671 /* Number of roots == degree of smu hence <= cap */ 672 if (roots_nbr == user->lmu[strength + 1] >> 1) 673 return err_nbr - 1; 674 675 /* 676 * Number of roots does not match the degree of smu 677 * unable to correct error. 678 */ 679 return -EBADMSG; 680 } 681 682 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector, 683 void *data, void *ecc) 684 { 685 struct atmel_pmecc *pmecc = user->pmecc; 686 int sectorsize = get_sectorsize(user); 687 int eccbytes = user->eccbytes; 688 int i, nerrors; 689 690 if (!(user->isr & BIT(sector))) 691 return 0; 692 693 atmel_pmecc_gen_syndrome(user, sector); 694 atmel_pmecc_substitute(user); 695 atmel_pmecc_get_sigma(user); 696 697 nerrors = atmel_pmecc_err_location(user); 698 if (nerrors < 0) 699 return nerrors; 700 701 for (i = 0; i < nerrors; i++) { 702 const char *area; 703 int byte, bit; 704 u32 errpos; 705 u8 *ptr; 706 707 errpos = readl_relaxed(pmecc->regs.errloc + 708 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i)); 709 errpos--; 710 711 byte = errpos / 8; 712 bit = errpos % 8; 713 714 if (byte < sectorsize) { 715 ptr = data + byte; 716 area = "data"; 717 } else if (byte < sectorsize + eccbytes) { 718 ptr = ecc + byte - sectorsize; 719 area = "ECC"; 720 } else { 721 dev_dbg(pmecc->dev, 722 "Invalid errpos value (%d, max is %d)\n", 723 errpos, (sectorsize + eccbytes) * 8); 724 return -EINVAL; 725 } 726 727 dev_dbg(pmecc->dev, 728 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n", 729 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit))); 730 731 *ptr ^= BIT(bit); 732 } 733 734 return nerrors; 735 } 736 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector); 737 738 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user) 739 { 740 return user->pmecc->caps->correct_erased_chunks; 741 } 742 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks); 743 744 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user, 745 int sector, void *ecc) 746 { 747 struct atmel_pmecc *pmecc = user->pmecc; 748 u8 *ptr = ecc; 749 int i; 750 751 for (i = 0; i < user->eccbytes; i++) 752 ptr[i] = readb_relaxed(pmecc->regs.base + 753 ATMEL_PMECC_ECC(sector, i)); 754 } 755 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes); 756 757 void atmel_pmecc_reset(struct atmel_pmecc *pmecc) 758 { 759 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); 760 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 761 } 762 EXPORT_SYMBOL_GPL(atmel_pmecc_reset); 763 764 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op) 765 { 766 struct atmel_pmecc *pmecc = user->pmecc; 767 u32 cfg; 768 769 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) { 770 dev_err(pmecc->dev, "Bad ECC operation!"); 771 return -EINVAL; 772 } 773 774 mutex_lock(&user->pmecc->lock); 775 776 cfg = user->cache.cfg; 777 if (op == NAND_ECC_WRITE) 778 cfg |= PMECC_CFG_WRITE_OP; 779 else 780 cfg |= PMECC_CFG_AUTO_ENABLE; 781 782 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG); 783 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA); 784 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR); 785 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR); 786 787 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 788 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL); 789 790 return 0; 791 } 792 EXPORT_SYMBOL_GPL(atmel_pmecc_enable); 793 794 void atmel_pmecc_disable(struct atmel_pmecc_user *user) 795 { 796 atmel_pmecc_reset(user->pmecc); 797 mutex_unlock(&user->pmecc->lock); 798 } 799 EXPORT_SYMBOL_GPL(atmel_pmecc_disable); 800 801 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user) 802 { 803 struct atmel_pmecc *pmecc = user->pmecc; 804 u32 status; 805 int ret; 806 807 ret = readl_relaxed_poll_timeout(pmecc->regs.base + 808 ATMEL_PMECC_SR, 809 status, !(status & PMECC_SR_BUSY), 0, 810 PMECC_MAX_TIMEOUT_MS * 1000); 811 if (ret) { 812 dev_err(pmecc->dev, 813 "Timeout while waiting for PMECC ready.\n"); 814 return ret; 815 } 816 817 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR); 818 819 return 0; 820 } 821 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy); 822 823 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, 824 const struct atmel_pmecc_caps *caps, 825 int pmecc_res_idx, int errloc_res_idx) 826 { 827 struct device *dev = &pdev->dev; 828 struct atmel_pmecc *pmecc; 829 830 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL); 831 if (!pmecc) 832 return ERR_PTR(-ENOMEM); 833 834 pmecc->caps = caps; 835 pmecc->dev = dev; 836 mutex_init(&pmecc->lock); 837 838 pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx); 839 if (IS_ERR(pmecc->regs.base)) 840 return ERR_CAST(pmecc->regs.base); 841 842 pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx); 843 if (IS_ERR(pmecc->regs.errloc)) 844 return ERR_CAST(pmecc->regs.errloc); 845 846 /* Disable all interrupts before registering the PMECC handler. */ 847 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); 848 atmel_pmecc_reset(pmecc); 849 850 return pmecc; 851 } 852 853 static void devm_atmel_pmecc_put(struct device *dev, void *res) 854 { 855 struct atmel_pmecc **pmecc = res; 856 857 put_device((*pmecc)->dev); 858 } 859 860 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, 861 struct device_node *np) 862 { 863 struct platform_device *pdev; 864 struct atmel_pmecc *pmecc, **ptr; 865 int ret; 866 867 pdev = of_find_device_by_node(np); 868 if (!pdev) 869 return ERR_PTR(-EPROBE_DEFER); 870 pmecc = platform_get_drvdata(pdev); 871 if (!pmecc) { 872 ret = -EPROBE_DEFER; 873 goto err_put_device; 874 } 875 876 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); 877 if (!ptr) { 878 ret = -ENOMEM; 879 goto err_put_device; 880 } 881 882 *ptr = pmecc; 883 884 devres_add(userdev, ptr); 885 886 return pmecc; 887 888 err_put_device: 889 put_device(&pdev->dev); 890 return ERR_PTR(ret); 891 } 892 893 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; 894 895 static struct atmel_pmecc_caps at91sam9g45_caps = { 896 .strengths = atmel_pmecc_strengths, 897 .nstrengths = 5, 898 .el_offset = 0x8c, 899 }; 900 901 static struct atmel_pmecc_caps sama5d4_caps = { 902 .strengths = atmel_pmecc_strengths, 903 .nstrengths = 5, 904 .el_offset = 0x8c, 905 .correct_erased_chunks = true, 906 }; 907 908 static struct atmel_pmecc_caps sama5d2_caps = { 909 .strengths = atmel_pmecc_strengths, 910 .nstrengths = 6, 911 .el_offset = 0xac, 912 .correct_erased_chunks = true, 913 }; 914 915 static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = { 916 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps }, 917 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps }, 918 { /* sentinel */ } 919 }; 920 921 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev) 922 { 923 struct atmel_pmecc *pmecc; 924 struct device_node *np; 925 926 if (!userdev) 927 return ERR_PTR(-EINVAL); 928 929 if (!userdev->of_node) 930 return NULL; 931 932 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0); 933 if (np) { 934 pmecc = atmel_pmecc_get_by_node(userdev, np); 935 of_node_put(np); 936 } else { 937 /* 938 * Support old DT bindings: in this case the PMECC iomem 939 * resources are directly defined in the user pdev at position 940 * 1 and 2. Extract all relevant information from there. 941 */ 942 struct platform_device *pdev = to_platform_device(userdev); 943 const struct atmel_pmecc_caps *caps; 944 const struct of_device_id *match; 945 946 /* No PMECC engine available. */ 947 if (!of_property_read_bool(userdev->of_node, 948 "atmel,has-pmecc")) 949 return NULL; 950 951 caps = &at91sam9g45_caps; 952 953 /* Find the caps associated to the NAND dev node. */ 954 match = of_match_node(atmel_pmecc_legacy_match, 955 userdev->of_node); 956 if (match && match->data) 957 caps = match->data; 958 959 pmecc = atmel_pmecc_create(pdev, caps, 1, 2); 960 } 961 962 return pmecc; 963 } 964 EXPORT_SYMBOL(devm_atmel_pmecc_get); 965 966 static const struct of_device_id atmel_pmecc_match[] = { 967 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps }, 968 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps }, 969 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps }, 970 { /* sentinel */ } 971 }; 972 MODULE_DEVICE_TABLE(of, atmel_pmecc_match); 973 974 static int atmel_pmecc_probe(struct platform_device *pdev) 975 { 976 struct device *dev = &pdev->dev; 977 const struct atmel_pmecc_caps *caps; 978 struct atmel_pmecc *pmecc; 979 980 caps = of_device_get_match_data(&pdev->dev); 981 if (!caps) { 982 dev_err(dev, "Invalid caps\n"); 983 return -EINVAL; 984 } 985 986 pmecc = atmel_pmecc_create(pdev, caps, 0, 1); 987 if (IS_ERR(pmecc)) 988 return PTR_ERR(pmecc); 989 990 platform_set_drvdata(pdev, pmecc); 991 992 return 0; 993 } 994 995 static struct platform_driver atmel_pmecc_driver = { 996 .driver = { 997 .name = "atmel-pmecc", 998 .of_match_table = atmel_pmecc_match, 999 }, 1000 .probe = atmel_pmecc_probe, 1001 }; 1002 module_platform_driver(atmel_pmecc_driver); 1003 1004 MODULE_LICENSE("GPL"); 1005 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); 1006 MODULE_DESCRIPTION("PMECC engine driver"); 1007 MODULE_ALIAS("platform:atmel_pmecc"); 1008