1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2017 ATMEL 4 * Copyright 2017 Free Electrons 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 * 8 * Derived from the atmel_nand.c driver which contained the following 9 * copyrights: 10 * 11 * Copyright 2003 Rick Bronson 12 * 13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8) 14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) 15 * 16 * Derived from drivers/mtd/spia.c (removed in v3.8) 17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com) 18 * 19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 20 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 21 * 22 * Derived from Das U-Boot source code 23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) 24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas 25 * 26 * Add Programmable Multibit ECC support for various AT91 SoC 27 * Copyright 2012 ATMEL, Hong Xu 28 * 29 * Add Nand Flash Controller support for SAMA5 SoC 30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) 31 * 32 * The PMECC is an hardware assisted BCH engine, which means part of the 33 * ECC algorithm is left to the software. The hardware/software repartition 34 * is explained in the "PMECC Controller Functional Description" chapter in 35 * Atmel datasheets, and some of the functions in this file are directly 36 * implementing the algorithms described in the "Software Implementation" 37 * sub-section. 38 * 39 * TODO: it seems that the software BCH implementation in lib/bch.c is already 40 * providing some of the logic we are implementing here. It would be smart 41 * to expose the needed lib/bch.c helpers/functions and re-use them here. 42 */ 43 44 #include <linux/genalloc.h> 45 #include <linux/iopoll.h> 46 #include <linux/module.h> 47 #include <linux/mtd/rawnand.h> 48 #include <linux/of_irq.h> 49 #include <linux/of_platform.h> 50 #include <linux/platform_device.h> 51 #include <linux/slab.h> 52 53 #include "pmecc.h" 54 55 /* Galois field dimension */ 56 #define PMECC_GF_DIMENSION_13 13 57 #define PMECC_GF_DIMENSION_14 14 58 59 /* Primitive Polynomial used by PMECC */ 60 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b 61 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443 62 63 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000 64 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000 65 66 /* Time out value for reading PMECC status register */ 67 #define PMECC_MAX_TIMEOUT_MS 100 68 69 /* PMECC Register Definitions */ 70 #define ATMEL_PMECC_CFG 0x0 71 #define PMECC_CFG_BCH_STRENGTH(x) (x) 72 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0) 73 #define PMECC_CFG_SECTOR512 (0 << 4) 74 #define PMECC_CFG_SECTOR1024 (1 << 4) 75 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8) 76 #define PMECC_CFG_READ_OP (0 << 12) 77 #define PMECC_CFG_WRITE_OP (1 << 12) 78 #define PMECC_CFG_SPARE_ENABLE BIT(16) 79 #define PMECC_CFG_AUTO_ENABLE BIT(20) 80 81 #define ATMEL_PMECC_SAREA 0x4 82 #define ATMEL_PMECC_SADDR 0x8 83 #define ATMEL_PMECC_EADDR 0xc 84 85 #define ATMEL_PMECC_CLK 0x10 86 #define PMECC_CLK_133MHZ (2 << 0) 87 88 #define ATMEL_PMECC_CTRL 0x14 89 #define PMECC_CTRL_RST BIT(0) 90 #define PMECC_CTRL_DATA BIT(1) 91 #define PMECC_CTRL_USER BIT(2) 92 #define PMECC_CTRL_ENABLE BIT(4) 93 #define PMECC_CTRL_DISABLE BIT(5) 94 95 #define ATMEL_PMECC_SR 0x18 96 #define PMECC_SR_BUSY BIT(0) 97 #define PMECC_SR_ENABLE BIT(4) 98 99 #define ATMEL_PMECC_IER 0x1c 100 #define ATMEL_PMECC_IDR 0x20 101 #define ATMEL_PMECC_IMR 0x24 102 #define ATMEL_PMECC_ISR 0x28 103 #define PMECC_ERROR_INT BIT(0) 104 105 #define ATMEL_PMECC_ECC(sector, n) \ 106 ((((sector) + 1) * 0x40) + (n)) 107 108 #define ATMEL_PMECC_REM(sector, n) \ 109 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200) 110 111 /* PMERRLOC Register Definitions */ 112 #define ATMEL_PMERRLOC_ELCFG 0x0 113 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0) 114 #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0) 115 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16) 116 117 #define ATMEL_PMERRLOC_ELPRIM 0x4 118 #define ATMEL_PMERRLOC_ELEN 0x8 119 #define ATMEL_PMERRLOC_ELDIS 0xc 120 #define PMERRLOC_DISABLE BIT(0) 121 122 #define ATMEL_PMERRLOC_ELSR 0x10 123 #define PMERRLOC_ELSR_BUSY BIT(0) 124 125 #define ATMEL_PMERRLOC_ELIER 0x14 126 #define ATMEL_PMERRLOC_ELIDR 0x18 127 #define ATMEL_PMERRLOC_ELIMR 0x1c 128 #define ATMEL_PMERRLOC_ELISR 0x20 129 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8) 130 #define PMERRLOC_CALC_DONE BIT(0) 131 132 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28) 133 134 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs)) 135 136 struct atmel_pmecc_gf_tables { 137 u16 *alpha_to; 138 u16 *index_of; 139 }; 140 141 struct atmel_pmecc_caps { 142 const int *strengths; 143 int nstrengths; 144 int el_offset; 145 bool correct_erased_chunks; 146 bool clk_ctrl; 147 }; 148 149 struct atmel_pmecc { 150 struct device *dev; 151 const struct atmel_pmecc_caps *caps; 152 153 struct { 154 void __iomem *base; 155 void __iomem *errloc; 156 } regs; 157 158 struct mutex lock; 159 }; 160 161 struct atmel_pmecc_user_conf_cache { 162 u32 cfg; 163 u32 sarea; 164 u32 saddr; 165 u32 eaddr; 166 }; 167 168 struct atmel_pmecc_user { 169 struct atmel_pmecc_user_conf_cache cache; 170 struct atmel_pmecc *pmecc; 171 const struct atmel_pmecc_gf_tables *gf_tables; 172 int eccbytes; 173 s16 *partial_syn; 174 s16 *si; 175 s16 *lmu; 176 s16 *smu; 177 s32 *mu; 178 s32 *dmu; 179 s32 *delta; 180 u32 isr; 181 }; 182 183 static DEFINE_MUTEX(pmecc_gf_tables_lock); 184 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512; 185 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024; 186 187 static inline int deg(unsigned int poly) 188 { 189 /* polynomial degree is the most-significant bit index */ 190 return fls(poly) - 1; 191 } 192 193 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly, 194 struct atmel_pmecc_gf_tables *gf_tables) 195 { 196 unsigned int i, x = 1; 197 const unsigned int k = BIT(deg(poly)); 198 unsigned int nn = BIT(mm) - 1; 199 200 /* primitive polynomial must be of degree m */ 201 if (k != (1u << mm)) 202 return -EINVAL; 203 204 for (i = 0; i < nn; i++) { 205 gf_tables->alpha_to[i] = x; 206 gf_tables->index_of[x] = i; 207 if (i && (x == 1)) 208 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */ 209 return -EINVAL; 210 x <<= 1; 211 if (x & k) 212 x ^= poly; 213 } 214 gf_tables->alpha_to[nn] = 1; 215 gf_tables->index_of[0] = 0; 216 217 return 0; 218 } 219 220 static const struct atmel_pmecc_gf_tables * 221 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req) 222 { 223 struct atmel_pmecc_gf_tables *gf_tables; 224 unsigned int poly, degree, table_size; 225 int ret; 226 227 if (req->ecc.sectorsize == 512) { 228 degree = PMECC_GF_DIMENSION_13; 229 poly = PMECC_GF_13_PRIMITIVE_POLY; 230 table_size = PMECC_LOOKUP_TABLE_SIZE_512; 231 } else { 232 degree = PMECC_GF_DIMENSION_14; 233 poly = PMECC_GF_14_PRIMITIVE_POLY; 234 table_size = PMECC_LOOKUP_TABLE_SIZE_1024; 235 } 236 237 gf_tables = kzalloc(sizeof(*gf_tables) + 238 (2 * table_size * sizeof(u16)), 239 GFP_KERNEL); 240 if (!gf_tables) 241 return ERR_PTR(-ENOMEM); 242 243 gf_tables->alpha_to = (void *)(gf_tables + 1); 244 gf_tables->index_of = gf_tables->alpha_to + table_size; 245 246 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables); 247 if (ret) { 248 kfree(gf_tables); 249 return ERR_PTR(ret); 250 } 251 252 return gf_tables; 253 } 254 255 static const struct atmel_pmecc_gf_tables * 256 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req) 257 { 258 const struct atmel_pmecc_gf_tables **gf_tables, *ret; 259 260 mutex_lock(&pmecc_gf_tables_lock); 261 if (req->ecc.sectorsize == 512) 262 gf_tables = &pmecc_gf_tables_512; 263 else 264 gf_tables = &pmecc_gf_tables_1024; 265 266 ret = *gf_tables; 267 268 if (!ret) { 269 ret = atmel_pmecc_create_gf_tables(req); 270 if (!IS_ERR(ret)) 271 *gf_tables = ret; 272 } 273 mutex_unlock(&pmecc_gf_tables_lock); 274 275 return ret; 276 } 277 278 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc, 279 struct atmel_pmecc_user_req *req) 280 { 281 int i, max_eccbytes, eccbytes = 0, eccstrength = 0; 282 283 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0) 284 return -EINVAL; 285 286 if (req->ecc.ooboffset >= 0 && 287 req->ecc.ooboffset + req->ecc.bytes > req->oobsize) 288 return -EINVAL; 289 290 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) { 291 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 292 return -EINVAL; 293 294 if (req->pagesize > 512) 295 req->ecc.sectorsize = 1024; 296 else 297 req->ecc.sectorsize = 512; 298 } 299 300 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024) 301 return -EINVAL; 302 303 if (req->pagesize % req->ecc.sectorsize) 304 return -EINVAL; 305 306 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize; 307 308 max_eccbytes = req->ecc.bytes; 309 310 for (i = 0; i < pmecc->caps->nstrengths; i++) { 311 int nbytes, strength = pmecc->caps->strengths[i]; 312 313 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH && 314 strength < req->ecc.strength) 315 continue; 316 317 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize), 318 8); 319 nbytes *= req->ecc.nsectors; 320 321 if (nbytes > max_eccbytes) 322 break; 323 324 eccstrength = strength; 325 eccbytes = nbytes; 326 327 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 328 break; 329 } 330 331 if (!eccstrength) 332 return -EINVAL; 333 334 req->ecc.bytes = eccbytes; 335 req->ecc.strength = eccstrength; 336 337 if (req->ecc.ooboffset < 0) 338 req->ecc.ooboffset = req->oobsize - eccbytes; 339 340 return 0; 341 } 342 343 struct atmel_pmecc_user * 344 atmel_pmecc_create_user(struct atmel_pmecc *pmecc, 345 struct atmel_pmecc_user_req *req) 346 { 347 struct atmel_pmecc_user *user; 348 const struct atmel_pmecc_gf_tables *gf_tables; 349 int strength, size, ret; 350 351 ret = atmel_pmecc_prepare_user_req(pmecc, req); 352 if (ret) 353 return ERR_PTR(ret); 354 355 size = sizeof(*user); 356 size = ALIGN(size, sizeof(u16)); 357 /* Reserve space for partial_syn, si and smu */ 358 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) * 359 (2 + req->ecc.strength + 2); 360 /* Reserve space for lmu. */ 361 size += (req->ecc.strength + 1) * sizeof(u16); 362 /* Reserve space for mu, dmu and delta. */ 363 size = ALIGN(size, sizeof(s32)); 364 size += (req->ecc.strength + 1) * sizeof(s32) * 3; 365 366 user = devm_kzalloc(pmecc->dev, size, GFP_KERNEL); 367 if (!user) 368 return ERR_PTR(-ENOMEM); 369 370 user->pmecc = pmecc; 371 372 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16)); 373 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1); 374 user->lmu = user->si + ((2 * req->ecc.strength) + 1); 375 user->smu = user->lmu + (req->ecc.strength + 1); 376 user->mu = (s32 *)PTR_ALIGN(user->smu + 377 (((2 * req->ecc.strength) + 1) * 378 (req->ecc.strength + 2)), 379 sizeof(s32)); 380 user->dmu = user->mu + req->ecc.strength + 1; 381 user->delta = user->dmu + req->ecc.strength + 1; 382 383 gf_tables = atmel_pmecc_get_gf_tables(req); 384 if (IS_ERR(gf_tables)) 385 return ERR_CAST(gf_tables); 386 387 user->gf_tables = gf_tables; 388 389 user->eccbytes = req->ecc.bytes / req->ecc.nsectors; 390 391 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) { 392 if (pmecc->caps->strengths[strength] == req->ecc.strength) 393 break; 394 } 395 396 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) | 397 PMECC_CFG_NSECTORS(req->ecc.nsectors); 398 399 if (req->ecc.sectorsize == 1024) 400 user->cache.cfg |= PMECC_CFG_SECTOR1024; 401 402 user->cache.sarea = req->oobsize - 1; 403 user->cache.saddr = req->ecc.ooboffset; 404 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1; 405 406 return user; 407 } 408 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user); 409 410 static int get_strength(struct atmel_pmecc_user *user) 411 { 412 const int *strengths = user->pmecc->caps->strengths; 413 414 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK]; 415 } 416 417 static int get_sectorsize(struct atmel_pmecc_user *user) 418 { 419 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512; 420 } 421 422 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector) 423 { 424 int strength = get_strength(user); 425 u32 value; 426 int i; 427 428 /* Fill odd syndromes */ 429 for (i = 0; i < strength; i++) { 430 value = readl_relaxed(user->pmecc->regs.base + 431 ATMEL_PMECC_REM(sector, i / 2)); 432 if (i & 1) 433 value >>= 16; 434 435 user->partial_syn[(2 * i) + 1] = value; 436 } 437 } 438 439 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user) 440 { 441 int degree = get_sectorsize(user) == 512 ? 13 : 14; 442 int cw_len = BIT(degree) - 1; 443 int strength = get_strength(user); 444 s16 *alpha_to = user->gf_tables->alpha_to; 445 s16 *index_of = user->gf_tables->index_of; 446 s16 *partial_syn = user->partial_syn; 447 s16 *si; 448 int i, j; 449 450 /* 451 * si[] is a table that holds the current syndrome value, 452 * an element of that table belongs to the field 453 */ 454 si = user->si; 455 456 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1)); 457 458 /* Computation 2t syndromes based on S(x) */ 459 /* Odd syndromes */ 460 for (i = 1; i < 2 * strength; i += 2) { 461 for (j = 0; j < degree; j++) { 462 if (partial_syn[i] & BIT(j)) 463 si[i] = alpha_to[i * j] ^ si[i]; 464 } 465 } 466 /* Even syndrome = (Odd syndrome) ** 2 */ 467 for (i = 2, j = 1; j <= strength; i = ++j << 1) { 468 if (si[j] == 0) { 469 si[i] = 0; 470 } else { 471 s16 tmp; 472 473 tmp = index_of[si[j]]; 474 tmp = (tmp * 2) % cw_len; 475 si[i] = alpha_to[tmp]; 476 } 477 } 478 } 479 480 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user) 481 { 482 s16 *lmu = user->lmu; 483 s16 *si = user->si; 484 s32 *mu = user->mu; 485 s32 *dmu = user->dmu; 486 s32 *delta = user->delta; 487 int degree = get_sectorsize(user) == 512 ? 13 : 14; 488 int cw_len = BIT(degree) - 1; 489 int strength = get_strength(user); 490 int num = 2 * strength + 1; 491 s16 *index_of = user->gf_tables->index_of; 492 s16 *alpha_to = user->gf_tables->alpha_to; 493 int i, j, k; 494 u32 dmu_0_count, tmp; 495 s16 *smu = user->smu; 496 497 /* index of largest delta */ 498 int ro; 499 int largest; 500 int diff; 501 502 dmu_0_count = 0; 503 504 /* First Row */ 505 506 /* Mu */ 507 mu[0] = -1; 508 509 memset(smu, 0, sizeof(s16) * num); 510 smu[0] = 1; 511 512 /* discrepancy set to 1 */ 513 dmu[0] = 1; 514 /* polynom order set to 0 */ 515 lmu[0] = 0; 516 delta[0] = (mu[0] * 2 - lmu[0]) >> 1; 517 518 /* Second Row */ 519 520 /* Mu */ 521 mu[1] = 0; 522 /* Sigma(x) set to 1 */ 523 memset(&smu[num], 0, sizeof(s16) * num); 524 smu[num] = 1; 525 526 /* discrepancy set to S1 */ 527 dmu[1] = si[1]; 528 529 /* polynom order set to 0 */ 530 lmu[1] = 0; 531 532 delta[1] = (mu[1] * 2 - lmu[1]) >> 1; 533 534 /* Init the Sigma(x) last row */ 535 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num); 536 537 for (i = 1; i <= strength; i++) { 538 mu[i + 1] = i << 1; 539 /* Begin Computing Sigma (Mu+1) and L(mu) */ 540 /* check if discrepancy is set to 0 */ 541 if (dmu[i] == 0) { 542 dmu_0_count++; 543 544 tmp = ((strength - (lmu[i] >> 1) - 1) / 2); 545 if ((strength - (lmu[i] >> 1) - 1) & 0x1) 546 tmp += 2; 547 else 548 tmp += 1; 549 550 if (dmu_0_count == tmp) { 551 for (j = 0; j <= (lmu[i] >> 1) + 1; j++) 552 smu[(strength + 1) * num + j] = 553 smu[i * num + j]; 554 555 lmu[strength + 1] = lmu[i]; 556 return; 557 } 558 559 /* copy polynom */ 560 for (j = 0; j <= lmu[i] >> 1; j++) 561 smu[(i + 1) * num + j] = smu[i * num + j]; 562 563 /* copy previous polynom order to the next */ 564 lmu[i + 1] = lmu[i]; 565 } else { 566 ro = 0; 567 largest = -1; 568 /* find largest delta with dmu != 0 */ 569 for (j = 0; j < i; j++) { 570 if ((dmu[j]) && (delta[j] > largest)) { 571 largest = delta[j]; 572 ro = j; 573 } 574 } 575 576 /* compute difference */ 577 diff = (mu[i] - mu[ro]); 578 579 /* Compute degree of the new smu polynomial */ 580 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff)) 581 lmu[i + 1] = lmu[i]; 582 else 583 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2; 584 585 /* Init smu[i+1] with 0 */ 586 for (k = 0; k < num; k++) 587 smu[(i + 1) * num + k] = 0; 588 589 /* Compute smu[i+1] */ 590 for (k = 0; k <= lmu[ro] >> 1; k++) { 591 s16 a, b, c; 592 593 if (!(smu[ro * num + k] && dmu[i])) 594 continue; 595 596 a = index_of[dmu[i]]; 597 b = index_of[dmu[ro]]; 598 c = index_of[smu[ro * num + k]]; 599 tmp = a + (cw_len - b) + c; 600 a = alpha_to[tmp % cw_len]; 601 smu[(i + 1) * num + (k + diff)] = a; 602 } 603 604 for (k = 0; k <= lmu[i] >> 1; k++) 605 smu[(i + 1) * num + k] ^= smu[i * num + k]; 606 } 607 608 /* End Computing Sigma (Mu+1) and L(mu) */ 609 /* In either case compute delta */ 610 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1; 611 612 /* Do not compute discrepancy for the last iteration */ 613 if (i >= strength) 614 continue; 615 616 for (k = 0; k <= (lmu[i + 1] >> 1); k++) { 617 tmp = 2 * (i - 1); 618 if (k == 0) { 619 dmu[i + 1] = si[tmp + 3]; 620 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) { 621 s16 a, b, c; 622 623 a = index_of[smu[(i + 1) * num + k]]; 624 b = si[2 * (i - 1) + 3 - k]; 625 c = index_of[b]; 626 tmp = a + c; 627 tmp %= cw_len; 628 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1]; 629 } 630 } 631 } 632 } 633 634 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user) 635 { 636 int sector_size = get_sectorsize(user); 637 int degree = sector_size == 512 ? 13 : 14; 638 struct atmel_pmecc *pmecc = user->pmecc; 639 int strength = get_strength(user); 640 int ret, roots_nbr, i, err_nbr = 0; 641 int num = (2 * strength) + 1; 642 s16 *smu = user->smu; 643 u32 val; 644 645 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS); 646 647 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) { 648 writel_relaxed(smu[(strength + 1) * num + i], 649 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i)); 650 err_nbr++; 651 } 652 653 val = (err_nbr - 1) << 16; 654 if (sector_size == 1024) 655 val |= 1; 656 657 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG); 658 writel((sector_size * 8) + (degree * strength), 659 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN); 660 661 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc + 662 ATMEL_PMERRLOC_ELISR, 663 val, val & PMERRLOC_CALC_DONE, 0, 664 PMECC_MAX_TIMEOUT_MS * 1000); 665 if (ret) { 666 dev_err(pmecc->dev, 667 "PMECC: Timeout to calculate error location.\n"); 668 return ret; 669 } 670 671 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8; 672 /* Number of roots == degree of smu hence <= cap */ 673 if (roots_nbr == user->lmu[strength + 1] >> 1) 674 return err_nbr - 1; 675 676 /* 677 * Number of roots does not match the degree of smu 678 * unable to correct error. 679 */ 680 return -EBADMSG; 681 } 682 683 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector, 684 void *data, void *ecc) 685 { 686 struct atmel_pmecc *pmecc = user->pmecc; 687 int sectorsize = get_sectorsize(user); 688 int eccbytes = user->eccbytes; 689 int i, nerrors; 690 691 if (!(user->isr & BIT(sector))) 692 return 0; 693 694 atmel_pmecc_gen_syndrome(user, sector); 695 atmel_pmecc_substitute(user); 696 atmel_pmecc_get_sigma(user); 697 698 nerrors = atmel_pmecc_err_location(user); 699 if (nerrors < 0) 700 return nerrors; 701 702 for (i = 0; i < nerrors; i++) { 703 const char *area; 704 int byte, bit; 705 u32 errpos; 706 u8 *ptr; 707 708 errpos = readl_relaxed(pmecc->regs.errloc + 709 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i)); 710 errpos--; 711 712 byte = errpos / 8; 713 bit = errpos % 8; 714 715 if (byte < sectorsize) { 716 ptr = data + byte; 717 area = "data"; 718 } else if (byte < sectorsize + eccbytes) { 719 ptr = ecc + byte - sectorsize; 720 area = "ECC"; 721 } else { 722 dev_dbg(pmecc->dev, 723 "Invalid errpos value (%d, max is %d)\n", 724 errpos, (sectorsize + eccbytes) * 8); 725 return -EINVAL; 726 } 727 728 dev_dbg(pmecc->dev, 729 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n", 730 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit))); 731 732 *ptr ^= BIT(bit); 733 } 734 735 return nerrors; 736 } 737 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector); 738 739 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user) 740 { 741 return user->pmecc->caps->correct_erased_chunks; 742 } 743 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks); 744 745 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user, 746 int sector, void *ecc) 747 { 748 struct atmel_pmecc *pmecc = user->pmecc; 749 u8 *ptr = ecc; 750 int i; 751 752 for (i = 0; i < user->eccbytes; i++) 753 ptr[i] = readb_relaxed(pmecc->regs.base + 754 ATMEL_PMECC_ECC(sector, i)); 755 } 756 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes); 757 758 void atmel_pmecc_reset(struct atmel_pmecc *pmecc) 759 { 760 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); 761 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 762 } 763 EXPORT_SYMBOL_GPL(atmel_pmecc_reset); 764 765 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op) 766 { 767 struct atmel_pmecc *pmecc = user->pmecc; 768 u32 cfg; 769 770 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) { 771 dev_err(pmecc->dev, "Bad ECC operation!"); 772 return -EINVAL; 773 } 774 775 mutex_lock(&user->pmecc->lock); 776 777 cfg = user->cache.cfg; 778 if (op == NAND_ECC_WRITE) 779 cfg |= PMECC_CFG_WRITE_OP; 780 else 781 cfg |= PMECC_CFG_AUTO_ENABLE; 782 783 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG); 784 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA); 785 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR); 786 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR); 787 788 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 789 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL); 790 791 return 0; 792 } 793 EXPORT_SYMBOL_GPL(atmel_pmecc_enable); 794 795 void atmel_pmecc_disable(struct atmel_pmecc_user *user) 796 { 797 atmel_pmecc_reset(user->pmecc); 798 mutex_unlock(&user->pmecc->lock); 799 } 800 EXPORT_SYMBOL_GPL(atmel_pmecc_disable); 801 802 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user) 803 { 804 struct atmel_pmecc *pmecc = user->pmecc; 805 u32 status; 806 int ret; 807 808 ret = readl_relaxed_poll_timeout(pmecc->regs.base + 809 ATMEL_PMECC_SR, 810 status, !(status & PMECC_SR_BUSY), 0, 811 PMECC_MAX_TIMEOUT_MS * 1000); 812 if (ret) { 813 dev_err(pmecc->dev, 814 "Timeout while waiting for PMECC ready.\n"); 815 return ret; 816 } 817 818 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR); 819 820 return 0; 821 } 822 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy); 823 824 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, 825 const struct atmel_pmecc_caps *caps, 826 int pmecc_res_idx, int errloc_res_idx) 827 { 828 struct device *dev = &pdev->dev; 829 struct atmel_pmecc *pmecc; 830 831 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL); 832 if (!pmecc) 833 return ERR_PTR(-ENOMEM); 834 835 pmecc->caps = caps; 836 pmecc->dev = dev; 837 mutex_init(&pmecc->lock); 838 839 pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx); 840 if (IS_ERR(pmecc->regs.base)) 841 return ERR_CAST(pmecc->regs.base); 842 843 pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx); 844 if (IS_ERR(pmecc->regs.errloc)) 845 return ERR_CAST(pmecc->regs.errloc); 846 847 /* pmecc data setup time */ 848 if (caps->clk_ctrl) 849 writel(PMECC_CLK_133MHZ, pmecc->regs.base + ATMEL_PMECC_CLK); 850 851 /* Disable all interrupts before registering the PMECC handler. */ 852 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); 853 atmel_pmecc_reset(pmecc); 854 855 return pmecc; 856 } 857 858 static void devm_atmel_pmecc_put(struct device *dev, void *res) 859 { 860 struct atmel_pmecc **pmecc = res; 861 862 put_device((*pmecc)->dev); 863 } 864 865 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, 866 struct device_node *np) 867 { 868 struct platform_device *pdev; 869 struct atmel_pmecc *pmecc, **ptr; 870 int ret; 871 872 pdev = of_find_device_by_node(np); 873 if (!pdev) 874 return ERR_PTR(-EPROBE_DEFER); 875 pmecc = platform_get_drvdata(pdev); 876 if (!pmecc) { 877 ret = -EPROBE_DEFER; 878 goto err_put_device; 879 } 880 881 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); 882 if (!ptr) { 883 ret = -ENOMEM; 884 goto err_put_device; 885 } 886 887 *ptr = pmecc; 888 889 devres_add(userdev, ptr); 890 891 return pmecc; 892 893 err_put_device: 894 put_device(&pdev->dev); 895 return ERR_PTR(ret); 896 } 897 898 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; 899 900 static struct atmel_pmecc_caps at91sam9g45_caps = { 901 .strengths = atmel_pmecc_strengths, 902 .nstrengths = 5, 903 .el_offset = 0x8c, 904 .clk_ctrl = true, 905 }; 906 907 static struct atmel_pmecc_caps sama5d4_caps = { 908 .strengths = atmel_pmecc_strengths, 909 .nstrengths = 5, 910 .el_offset = 0x8c, 911 .correct_erased_chunks = true, 912 }; 913 914 static struct atmel_pmecc_caps sama5d2_caps = { 915 .strengths = atmel_pmecc_strengths, 916 .nstrengths = 6, 917 .el_offset = 0xac, 918 .correct_erased_chunks = true, 919 }; 920 921 static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = { 922 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps }, 923 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps }, 924 { /* sentinel */ } 925 }; 926 927 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev) 928 { 929 struct atmel_pmecc *pmecc; 930 struct device_node *np; 931 932 if (!userdev) 933 return ERR_PTR(-EINVAL); 934 935 if (!userdev->of_node) 936 return NULL; 937 938 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0); 939 if (np) { 940 pmecc = atmel_pmecc_get_by_node(userdev, np); 941 of_node_put(np); 942 } else { 943 /* 944 * Support old DT bindings: in this case the PMECC iomem 945 * resources are directly defined in the user pdev at position 946 * 1 and 2. Extract all relevant information from there. 947 */ 948 struct platform_device *pdev = to_platform_device(userdev); 949 const struct atmel_pmecc_caps *caps; 950 const struct of_device_id *match; 951 952 /* No PMECC engine available. */ 953 if (!of_property_read_bool(userdev->of_node, 954 "atmel,has-pmecc")) 955 return NULL; 956 957 caps = &at91sam9g45_caps; 958 959 /* Find the caps associated to the NAND dev node. */ 960 match = of_match_node(atmel_pmecc_legacy_match, 961 userdev->of_node); 962 if (match && match->data) 963 caps = match->data; 964 965 pmecc = atmel_pmecc_create(pdev, caps, 1, 2); 966 } 967 968 return pmecc; 969 } 970 EXPORT_SYMBOL(devm_atmel_pmecc_get); 971 972 static const struct of_device_id atmel_pmecc_match[] = { 973 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps }, 974 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps }, 975 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps }, 976 { /* sentinel */ } 977 }; 978 MODULE_DEVICE_TABLE(of, atmel_pmecc_match); 979 980 static int atmel_pmecc_probe(struct platform_device *pdev) 981 { 982 struct device *dev = &pdev->dev; 983 const struct atmel_pmecc_caps *caps; 984 struct atmel_pmecc *pmecc; 985 986 caps = of_device_get_match_data(&pdev->dev); 987 if (!caps) { 988 dev_err(dev, "Invalid caps\n"); 989 return -EINVAL; 990 } 991 992 pmecc = atmel_pmecc_create(pdev, caps, 0, 1); 993 if (IS_ERR(pmecc)) 994 return PTR_ERR(pmecc); 995 996 platform_set_drvdata(pdev, pmecc); 997 998 return 0; 999 } 1000 1001 static struct platform_driver atmel_pmecc_driver = { 1002 .driver = { 1003 .name = "atmel-pmecc", 1004 .of_match_table = atmel_pmecc_match, 1005 }, 1006 .probe = atmel_pmecc_probe, 1007 }; 1008 module_platform_driver(atmel_pmecc_driver); 1009 1010 MODULE_LICENSE("GPL"); 1011 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); 1012 MODULE_DESCRIPTION("PMECC engine driver"); 1013 MODULE_ALIAS("platform:atmel_pmecc"); 1014