1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale GPMI NAND Flash Driver 4 * 5 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc. 6 * Copyright (C) 2008 Embedded Alley Solutions, Inc. 7 */ 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/slab.h> 11 #include <linux/sched/task_stack.h> 12 #include <linux/interrupt.h> 13 #include <linux/module.h> 14 #include <linux/mtd/partitions.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_runtime.h> 18 #include <linux/pinctrl/consumer.h> 19 #include <linux/dma/mxs-dma.h> 20 #include <linux/string_choices.h> 21 #include "gpmi-nand.h" 22 #include "gpmi-regs.h" 23 #include "bch-regs.h" 24 25 /* Resource names for the GPMI NAND driver. */ 26 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME "gpmi-nand" 27 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME "bch" 28 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME "bch" 29 30 /* Converts time to clock cycles */ 31 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period) 32 33 #define MXS_SET_ADDR 0x4 34 #define MXS_CLR_ADDR 0x8 35 /* 36 * Clear the bit and poll it cleared. This is usually called with 37 * a reset address and mask being either SFTRST(bit 31) or CLKGATE 38 * (bit 30). 39 */ 40 static int clear_poll_bit(void __iomem *addr, u32 mask) 41 { 42 int timeout = 0x400; 43 44 /* clear the bit */ 45 writel(mask, addr + MXS_CLR_ADDR); 46 47 /* 48 * SFTRST needs 3 GPMI clocks to settle, the reference manual 49 * recommends to wait 1us. 50 */ 51 udelay(1); 52 53 /* poll the bit becoming clear */ 54 while ((readl(addr) & mask) && --timeout) 55 /* nothing */; 56 57 return !timeout; 58 } 59 60 #define MODULE_CLKGATE (1 << 30) 61 #define MODULE_SFTRST (1 << 31) 62 /* 63 * The current mxs_reset_block() will do two things: 64 * [1] enable the module. 65 * [2] reset the module. 66 * 67 * In most of the cases, it's ok. 68 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847). 69 * If you try to soft reset the BCH block, it becomes unusable until 70 * the next hard reset. This case occurs in the NAND boot mode. When the board 71 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself. 72 * So If the driver tries to reset the BCH again, the BCH will not work anymore. 73 * You will see a DMA timeout in this case. The bug has been fixed 74 * in the following chips, such as MX28. 75 * 76 * To avoid this bug, just add a new parameter `just_enable` for 77 * the mxs_reset_block(), and rewrite it here. 78 */ 79 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable) 80 { 81 int ret; 82 int timeout = 0x400; 83 84 /* clear and poll SFTRST */ 85 ret = clear_poll_bit(reset_addr, MODULE_SFTRST); 86 if (unlikely(ret)) 87 goto error; 88 89 /* clear CLKGATE */ 90 writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR); 91 92 if (!just_enable) { 93 /* set SFTRST to reset the block */ 94 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR); 95 udelay(1); 96 97 /* poll CLKGATE becoming set */ 98 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout) 99 /* nothing */; 100 if (unlikely(!timeout)) 101 goto error; 102 } 103 104 /* clear and poll SFTRST */ 105 ret = clear_poll_bit(reset_addr, MODULE_SFTRST); 106 if (unlikely(ret)) 107 goto error; 108 109 /* clear and poll CLKGATE */ 110 ret = clear_poll_bit(reset_addr, MODULE_CLKGATE); 111 if (unlikely(ret)) 112 goto error; 113 114 return 0; 115 116 error: 117 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr); 118 return -ETIMEDOUT; 119 } 120 121 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v) 122 { 123 struct clk *clk; 124 int ret; 125 int i; 126 127 for (i = 0; i < GPMI_CLK_MAX; i++) { 128 clk = this->resources.clock[i]; 129 if (!clk) 130 break; 131 132 if (v) { 133 ret = clk_prepare_enable(clk); 134 if (ret) 135 goto err_clk; 136 } else { 137 clk_disable_unprepare(clk); 138 } 139 } 140 return 0; 141 142 err_clk: 143 for (; i > 0; i--) 144 clk_disable_unprepare(this->resources.clock[i - 1]); 145 return ret; 146 } 147 148 static int gpmi_init(struct gpmi_nand_data *this) 149 { 150 struct resources *r = &this->resources; 151 int ret; 152 153 ret = pm_runtime_resume_and_get(this->dev); 154 if (ret < 0) 155 return ret; 156 157 ret = gpmi_reset_block(r->gpmi_regs, false); 158 if (ret) 159 goto err_out; 160 161 /* 162 * Reset BCH here, too. We got failures otherwise :( 163 * See later BCH reset for explanation of MX23 and MX28 handling 164 */ 165 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this)); 166 if (ret) 167 goto err_out; 168 169 /* Choose NAND mode. */ 170 writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR); 171 172 /* Set the IRQ polarity. */ 173 writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY, 174 r->gpmi_regs + HW_GPMI_CTRL1_SET); 175 176 /* Disable Write-Protection. */ 177 writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET); 178 179 /* Select BCH ECC. */ 180 writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET); 181 182 /* 183 * Decouple the chip select from dma channel. We use dma0 for all 184 * the chips, force all NAND RDY_BUSY inputs to be sourced from 185 * RDY_BUSY0. 186 */ 187 writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY, 188 r->gpmi_regs + HW_GPMI_CTRL1_SET); 189 190 err_out: 191 pm_runtime_mark_last_busy(this->dev); 192 pm_runtime_put_autosuspend(this->dev); 193 return ret; 194 } 195 196 /* This function is very useful. It is called only when the bug occur. */ 197 static void gpmi_dump_info(struct gpmi_nand_data *this) 198 { 199 struct resources *r = &this->resources; 200 struct bch_geometry *geo = &this->bch_geometry; 201 u32 reg; 202 int i; 203 204 dev_err(this->dev, "Show GPMI registers :\n"); 205 for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) { 206 reg = readl(r->gpmi_regs + i * 0x10); 207 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg); 208 } 209 210 /* start to print out the BCH info */ 211 dev_err(this->dev, "Show BCH registers :\n"); 212 for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) { 213 reg = readl(r->bch_regs + i * 0x10); 214 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg); 215 } 216 dev_err(this->dev, "BCH Geometry :\n" 217 "GF length : %u\n" 218 "ECC Strength : %u\n" 219 "Page Size in Bytes : %u\n" 220 "Metadata Size in Bytes : %u\n" 221 "ECC0 Chunk Size in Bytes: %u\n" 222 "ECCn Chunk Size in Bytes: %u\n" 223 "ECC Chunk Count : %u\n" 224 "Payload Size in Bytes : %u\n" 225 "Auxiliary Size in Bytes: %u\n" 226 "Auxiliary Status Offset: %u\n" 227 "Block Mark Byte Offset : %u\n" 228 "Block Mark Bit Offset : %u\n", 229 geo->gf_len, 230 geo->ecc_strength, 231 geo->page_size, 232 geo->metadata_size, 233 geo->ecc0_chunk_size, 234 geo->eccn_chunk_size, 235 geo->ecc_chunk_count, 236 geo->payload_size, 237 geo->auxiliary_size, 238 geo->auxiliary_status_offset, 239 geo->block_mark_byte_offset, 240 geo->block_mark_bit_offset); 241 } 242 243 static bool gpmi_check_ecc(struct gpmi_nand_data *this) 244 { 245 struct nand_chip *chip = &this->nand; 246 struct bch_geometry *geo = &this->bch_geometry; 247 struct nand_device *nand = &chip->base; 248 struct nand_ecc_props *conf = &nand->ecc.ctx.conf; 249 250 conf->step_size = geo->eccn_chunk_size; 251 conf->strength = geo->ecc_strength; 252 253 /* Do the sanity check. */ 254 if (GPMI_IS_MXS(this)) { 255 /* The mx23/mx28 only support the GF13. */ 256 if (geo->gf_len == 14) 257 return false; 258 } 259 260 if (geo->ecc_strength > this->devdata->bch_max_ecc_strength) 261 return false; 262 263 if (!nand_ecc_is_strong_enough(nand)) 264 return false; 265 266 return true; 267 } 268 269 /* check if bbm locates in data chunk rather than ecc chunk */ 270 static bool bbm_in_data_chunk(struct gpmi_nand_data *this, 271 unsigned int *chunk_num) 272 { 273 struct bch_geometry *geo = &this->bch_geometry; 274 struct nand_chip *chip = &this->nand; 275 struct mtd_info *mtd = nand_to_mtd(chip); 276 unsigned int i, j; 277 278 if (geo->ecc0_chunk_size != geo->eccn_chunk_size) { 279 dev_err(this->dev, 280 "The size of ecc0_chunk must equal to eccn_chunk\n"); 281 return false; 282 } 283 284 i = (mtd->writesize * 8 - geo->metadata_size * 8) / 285 (geo->gf_len * geo->ecc_strength + 286 geo->eccn_chunk_size * 8); 287 288 j = (mtd->writesize * 8 - geo->metadata_size * 8) - 289 (geo->gf_len * geo->ecc_strength + 290 geo->eccn_chunk_size * 8) * i; 291 292 if (j < geo->eccn_chunk_size * 8) { 293 *chunk_num = i+1; 294 dev_dbg(this->dev, "Set ecc to %d and bbm in chunk %d\n", 295 geo->ecc_strength, *chunk_num); 296 return true; 297 } 298 299 return false; 300 } 301 302 /* 303 * If we can get the ECC information from the nand chip, we do not 304 * need to calculate them ourselves. 305 * 306 * We may have available oob space in this case. 307 */ 308 static int set_geometry_by_ecc_info(struct gpmi_nand_data *this, 309 unsigned int ecc_strength, 310 unsigned int ecc_step) 311 { 312 struct bch_geometry *geo = &this->bch_geometry; 313 struct nand_chip *chip = &this->nand; 314 struct mtd_info *mtd = nand_to_mtd(chip); 315 unsigned int block_mark_bit_offset; 316 317 switch (ecc_step) { 318 case SZ_512: 319 geo->gf_len = 13; 320 break; 321 case SZ_1K: 322 geo->gf_len = 14; 323 break; 324 default: 325 dev_err(this->dev, 326 "unsupported nand chip. ecc bits : %d, ecc size : %d\n", 327 nanddev_get_ecc_requirements(&chip->base)->strength, 328 nanddev_get_ecc_requirements(&chip->base)->step_size); 329 return -EINVAL; 330 } 331 geo->ecc0_chunk_size = ecc_step; 332 geo->eccn_chunk_size = ecc_step; 333 geo->ecc_strength = round_up(ecc_strength, 2); 334 if (!gpmi_check_ecc(this)) 335 return -EINVAL; 336 337 /* Keep the C >= O */ 338 if (geo->eccn_chunk_size < mtd->oobsize) { 339 dev_err(this->dev, 340 "unsupported nand chip. ecc size: %d, oob size : %d\n", 341 ecc_step, mtd->oobsize); 342 return -EINVAL; 343 } 344 345 /* The default value, see comment in the legacy_set_geometry(). */ 346 geo->metadata_size = 10; 347 348 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 349 350 /* 351 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below: 352 * 353 * | P | 354 * |<----------------------------------------------------->| 355 * | | 356 * | (Block Mark) | 357 * | P' | | | | 358 * |<-------------------------------------------->| D | | O' | 359 * | |<---->| |<--->| 360 * V V V V V 361 * +---+----------+-+----------+-+----------+-+----------+-+-----+ 362 * | M | data |E| data |E| data |E| data |E| | 363 * +---+----------+-+----------+-+----------+-+----------+-+-----+ 364 * ^ ^ 365 * | O | 366 * |<------------>| 367 * | | 368 * 369 * P : the page size for BCH module. 370 * E : The ECC strength. 371 * G : the length of Galois Field. 372 * N : The chunk count of per page. 373 * M : the metasize of per page. 374 * C : the ecc chunk size, aka the "data" above. 375 * P': the nand chip's page size. 376 * O : the nand chip's oob size. 377 * O': the free oob. 378 * 379 * The formula for P is : 380 * 381 * E * G * N 382 * P = ------------ + P' + M 383 * 8 384 * 385 * The position of block mark moves forward in the ECC-based view 386 * of page, and the delta is: 387 * 388 * E * G * (N - 1) 389 * D = (---------------- + M) 390 * 8 391 * 392 * Please see the comment in legacy_set_geometry(). 393 * With the condition C >= O , we still can get same result. 394 * So the bit position of the physical block mark within the ECC-based 395 * view of the page is : 396 * (P' - D) * 8 397 */ 398 geo->page_size = mtd->writesize + geo->metadata_size + 399 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 400 401 geo->payload_size = mtd->writesize; 402 403 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4); 404 geo->auxiliary_size = ALIGN(geo->metadata_size, 4) 405 + ALIGN(geo->ecc_chunk_count, 4); 406 407 if (!this->swap_block_mark) 408 return 0; 409 410 /* For bit swap. */ 411 block_mark_bit_offset = mtd->writesize * 8 - 412 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1) 413 + geo->metadata_size * 8); 414 415 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 416 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 417 return 0; 418 } 419 420 /* 421 * Calculate the ECC strength by hand: 422 * E : The ECC strength. 423 * G : the length of Galois Field. 424 * N : The chunk count of per page. 425 * O : the oobsize of the NAND chip. 426 * M : the metasize of per page. 427 * 428 * The formula is : 429 * E * G * N 430 * ------------ <= (O - M) 431 * 8 432 * 433 * So, we get E by: 434 * (O - M) * 8 435 * E <= ------------- 436 * G * N 437 */ 438 static inline int get_ecc_strength(struct gpmi_nand_data *this) 439 { 440 struct bch_geometry *geo = &this->bch_geometry; 441 struct mtd_info *mtd = nand_to_mtd(&this->nand); 442 int ecc_strength; 443 444 ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8) 445 / (geo->gf_len * geo->ecc_chunk_count); 446 447 /* We need the minor even number. */ 448 return round_down(ecc_strength, 2); 449 } 450 451 static int set_geometry_for_large_oob(struct gpmi_nand_data *this) 452 { 453 struct bch_geometry *geo = &this->bch_geometry; 454 struct nand_chip *chip = &this->nand; 455 struct mtd_info *mtd = nand_to_mtd(chip); 456 const struct nand_ecc_props *requirements = 457 nanddev_get_ecc_requirements(&chip->base); 458 unsigned int block_mark_bit_offset; 459 unsigned int max_ecc; 460 unsigned int bbm_chunk; 461 unsigned int i; 462 463 /* sanity check for the minimum ecc nand required */ 464 if (!(requirements->strength > 0 && 465 requirements->step_size > 0)) 466 return -EINVAL; 467 geo->ecc_strength = requirements->strength; 468 469 /* check if platform can support this nand */ 470 if (!gpmi_check_ecc(this)) { 471 dev_err(this->dev, 472 "unsupported NAND chip, minimum ecc required %d\n", 473 geo->ecc_strength); 474 return -EINVAL; 475 } 476 477 /* calculate the maximum ecc platform can support*/ 478 geo->metadata_size = 10; 479 geo->gf_len = 14; 480 geo->ecc0_chunk_size = 1024; 481 geo->eccn_chunk_size = 1024; 482 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 483 max_ecc = min(get_ecc_strength(this), 484 this->devdata->bch_max_ecc_strength); 485 486 /* 487 * search a supported ecc strength that makes bbm 488 * located in data chunk 489 */ 490 geo->ecc_strength = max_ecc; 491 while (!(geo->ecc_strength < requirements->strength)) { 492 if (bbm_in_data_chunk(this, &bbm_chunk)) 493 goto geo_setting; 494 geo->ecc_strength -= 2; 495 } 496 497 /* if none of them works, keep using the minimum ecc */ 498 /* nand required but changing ecc page layout */ 499 geo->ecc_strength = requirements->strength; 500 /* add extra ecc for meta data */ 501 geo->ecc0_chunk_size = 0; 502 geo->ecc_chunk_count = (mtd->writesize / geo->eccn_chunk_size) + 1; 503 geo->ecc_for_meta = 1; 504 /* check if oob can afford this extra ecc chunk */ 505 if (mtd->oobsize * 8 < geo->metadata_size * 8 + 506 geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) { 507 dev_err(this->dev, "unsupported NAND chip with new layout\n"); 508 return -EINVAL; 509 } 510 511 /* calculate in which chunk bbm located */ 512 bbm_chunk = (mtd->writesize * 8 - geo->metadata_size * 8 - 513 geo->gf_len * geo->ecc_strength) / 514 (geo->gf_len * geo->ecc_strength + 515 geo->eccn_chunk_size * 8) + 1; 516 517 geo_setting: 518 519 geo->page_size = mtd->writesize + geo->metadata_size + 520 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 521 geo->payload_size = mtd->writesize; 522 523 /* 524 * The auxiliary buffer contains the metadata and the ECC status. The 525 * metadata is padded to the nearest 32-bit boundary. The ECC status 526 * contains one byte for every ECC chunk, and is also padded to the 527 * nearest 32-bit boundary. 528 */ 529 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4); 530 geo->auxiliary_size = ALIGN(geo->metadata_size, 4) 531 + ALIGN(geo->ecc_chunk_count, 4); 532 533 if (!this->swap_block_mark) 534 return 0; 535 536 /* calculate the number of ecc chunk behind the bbm */ 537 i = (mtd->writesize / geo->eccn_chunk_size) - bbm_chunk + 1; 538 539 block_mark_bit_offset = mtd->writesize * 8 - 540 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - i) 541 + geo->metadata_size * 8); 542 543 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 544 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 545 546 dev_dbg(this->dev, "BCH Geometry :\n" 547 "GF length : %u\n" 548 "ECC Strength : %u\n" 549 "Page Size in Bytes : %u\n" 550 "Metadata Size in Bytes : %u\n" 551 "ECC0 Chunk Size in Bytes: %u\n" 552 "ECCn Chunk Size in Bytes: %u\n" 553 "ECC Chunk Count : %u\n" 554 "Payload Size in Bytes : %u\n" 555 "Auxiliary Size in Bytes: %u\n" 556 "Auxiliary Status Offset: %u\n" 557 "Block Mark Byte Offset : %u\n" 558 "Block Mark Bit Offset : %u\n" 559 "Block Mark in chunk : %u\n" 560 "Ecc for Meta data : %u\n", 561 geo->gf_len, 562 geo->ecc_strength, 563 geo->page_size, 564 geo->metadata_size, 565 geo->ecc0_chunk_size, 566 geo->eccn_chunk_size, 567 geo->ecc_chunk_count, 568 geo->payload_size, 569 geo->auxiliary_size, 570 geo->auxiliary_status_offset, 571 geo->block_mark_byte_offset, 572 geo->block_mark_bit_offset, 573 bbm_chunk, 574 geo->ecc_for_meta); 575 576 return 0; 577 } 578 579 static int legacy_set_geometry(struct gpmi_nand_data *this) 580 { 581 struct bch_geometry *geo = &this->bch_geometry; 582 struct mtd_info *mtd = nand_to_mtd(&this->nand); 583 unsigned int metadata_size; 584 unsigned int status_size; 585 unsigned int block_mark_bit_offset; 586 587 /* 588 * The size of the metadata can be changed, though we set it to 10 589 * bytes now. But it can't be too large, because we have to save 590 * enough space for BCH. 591 */ 592 geo->metadata_size = 10; 593 594 /* The default for the length of Galois Field. */ 595 geo->gf_len = 13; 596 597 /* The default for chunk size. */ 598 geo->ecc0_chunk_size = 512; 599 geo->eccn_chunk_size = 512; 600 while (geo->eccn_chunk_size < mtd->oobsize) { 601 geo->ecc0_chunk_size *= 2; /* keep C >= O */ 602 geo->eccn_chunk_size *= 2; /* keep C >= O */ 603 geo->gf_len = 14; 604 } 605 606 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 607 608 /* We use the same ECC strength for all chunks. */ 609 geo->ecc_strength = get_ecc_strength(this); 610 if (!gpmi_check_ecc(this)) { 611 dev_err(this->dev, 612 "ecc strength: %d cannot be supported by the controller (%d)\n" 613 "try to use minimum ecc strength that NAND chip required\n", 614 geo->ecc_strength, 615 this->devdata->bch_max_ecc_strength); 616 return -EINVAL; 617 } 618 619 geo->page_size = mtd->writesize + geo->metadata_size + 620 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 621 geo->payload_size = mtd->writesize; 622 623 /* 624 * The auxiliary buffer contains the metadata and the ECC status. The 625 * metadata is padded to the nearest 32-bit boundary. The ECC status 626 * contains one byte for every ECC chunk, and is also padded to the 627 * nearest 32-bit boundary. 628 */ 629 metadata_size = ALIGN(geo->metadata_size, 4); 630 status_size = ALIGN(geo->ecc_chunk_count, 4); 631 632 geo->auxiliary_size = metadata_size + status_size; 633 geo->auxiliary_status_offset = metadata_size; 634 635 if (!this->swap_block_mark) 636 return 0; 637 638 /* 639 * We need to compute the byte and bit offsets of 640 * the physical block mark within the ECC-based view of the page. 641 * 642 * NAND chip with 2K page shows below: 643 * (Block Mark) 644 * | | 645 * | D | 646 * |<---->| 647 * V V 648 * +---+----------+-+----------+-+----------+-+----------+-+ 649 * | M | data |E| data |E| data |E| data |E| 650 * +---+----------+-+----------+-+----------+-+----------+-+ 651 * 652 * The position of block mark moves forward in the ECC-based view 653 * of page, and the delta is: 654 * 655 * E * G * (N - 1) 656 * D = (---------------- + M) 657 * 8 658 * 659 * With the formula to compute the ECC strength, and the condition 660 * : C >= O (C is the ecc chunk size) 661 * 662 * It's easy to deduce to the following result: 663 * 664 * E * G (O - M) C - M C - M 665 * ----------- <= ------- <= -------- < --------- 666 * 8 N N (N - 1) 667 * 668 * So, we get: 669 * 670 * E * G * (N - 1) 671 * D = (---------------- + M) < C 672 * 8 673 * 674 * The above inequality means the position of block mark 675 * within the ECC-based view of the page is still in the data chunk, 676 * and it's NOT in the ECC bits of the chunk. 677 * 678 * Use the following to compute the bit position of the 679 * physical block mark within the ECC-based view of the page: 680 * (page_size - D) * 8 681 * 682 * --Huang Shijie 683 */ 684 block_mark_bit_offset = mtd->writesize * 8 - 685 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1) 686 + geo->metadata_size * 8); 687 688 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 689 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 690 return 0; 691 } 692 693 static int common_nfc_set_geometry(struct gpmi_nand_data *this) 694 { 695 struct nand_chip *chip = &this->nand; 696 struct mtd_info *mtd = nand_to_mtd(&this->nand); 697 const struct nand_ecc_props *requirements = 698 nanddev_get_ecc_requirements(&chip->base); 699 bool use_minimun_ecc; 700 int err; 701 702 use_minimun_ecc = of_property_read_bool(this->dev->of_node, 703 "fsl,use-minimum-ecc"); 704 705 /* use legacy bch geometry settings by default*/ 706 if ((!use_minimun_ecc && mtd->oobsize < 1024) || 707 !(requirements->strength > 0 && requirements->step_size > 0)) { 708 dev_dbg(this->dev, "use legacy bch geometry\n"); 709 err = legacy_set_geometry(this); 710 if (!err) 711 return 0; 712 } 713 714 /* for large oob nand */ 715 if (mtd->oobsize > 1024) { 716 dev_dbg(this->dev, "use large oob bch geometry\n"); 717 err = set_geometry_for_large_oob(this); 718 if (!err) 719 return 0; 720 } 721 722 /* otherwise use the minimum ecc nand chip required */ 723 dev_dbg(this->dev, "use minimum ecc bch geometry\n"); 724 err = set_geometry_by_ecc_info(this, requirements->strength, 725 requirements->step_size); 726 if (err) 727 dev_err(this->dev, "none of the bch geometry setting works\n"); 728 729 return err; 730 } 731 732 /* Configures the geometry for BCH. */ 733 static int bch_set_geometry(struct gpmi_nand_data *this) 734 { 735 struct resources *r = &this->resources; 736 int ret; 737 738 ret = common_nfc_set_geometry(this); 739 if (ret) 740 return ret; 741 742 ret = pm_runtime_resume_and_get(this->dev); 743 if (ret < 0) { 744 return ret; 745 } 746 747 /* 748 * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this 749 * chip, otherwise it will lock up. So we skip resetting BCH on the MX23. 750 * and MX28. 751 */ 752 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this)); 753 if (ret) 754 goto err_out; 755 756 /* Set *all* chip selects to use layout 0. */ 757 writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT); 758 759 ret = 0; 760 err_out: 761 pm_runtime_mark_last_busy(this->dev); 762 pm_runtime_put_autosuspend(this->dev); 763 764 return ret; 765 } 766 767 /* 768 * <1> Firstly, we should know what's the GPMI-clock means. 769 * The GPMI-clock is the internal clock in the gpmi nand controller. 770 * If you set 100MHz to gpmi nand controller, the GPMI-clock's period 771 * is 10ns. Mark the GPMI-clock's period as GPMI-clock-period. 772 * 773 * <2> Secondly, we should know what's the frequency on the nand chip pins. 774 * The frequency on the nand chip pins is derived from the GPMI-clock. 775 * We can get it from the following equation: 776 * 777 * F = G / (DS + DH) 778 * 779 * F : the frequency on the nand chip pins. 780 * G : the GPMI clock, such as 100MHz. 781 * DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP 782 * DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD 783 * 784 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz, 785 * the nand EDO(extended Data Out) timing could be applied. 786 * The GPMI implements a feedback read strobe to sample the read data. 787 * The feedback read strobe can be delayed to support the nand EDO timing 788 * where the read strobe may deasserts before the read data is valid, and 789 * read data is valid for some time after read strobe. 790 * 791 * The following figure illustrates some aspects of a NAND Flash read: 792 * 793 * |<---tREA---->| 794 * | | 795 * | | | 796 * |<--tRP-->| | 797 * | | | 798 * __ ___|__________________________________ 799 * RDN \________/ | 800 * | 801 * /---------\ 802 * Read Data --------------< >--------- 803 * \---------/ 804 * | | 805 * |<-D->| 806 * FeedbackRDN ________ ____________ 807 * \___________/ 808 * 809 * D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY. 810 * 811 * 812 * <4> Now, we begin to describe how to compute the right RDN_DELAY. 813 * 814 * 4.1) From the aspect of the nand chip pins: 815 * Delay = (tREA + C - tRP) {1} 816 * 817 * tREA : the maximum read access time. 818 * C : a constant to adjust the delay. default is 4000ps. 819 * tRP : the read pulse width, which is exactly: 820 * tRP = (GPMI-clock-period) * DATA_SETUP 821 * 822 * 4.2) From the aspect of the GPMI nand controller: 823 * Delay = RDN_DELAY * 0.125 * RP {2} 824 * 825 * RP : the DLL reference period. 826 * if (GPMI-clock-period > DLL_THRETHOLD) 827 * RP = GPMI-clock-period / 2; 828 * else 829 * RP = GPMI-clock-period; 830 * 831 * Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period 832 * is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD 833 * is 16000ps, but in mx6q, we use 12000ps. 834 * 835 * 4.3) since {1} equals {2}, we get: 836 * 837 * (tREA + 4000 - tRP) * 8 838 * RDN_DELAY = ----------------------- {3} 839 * RP 840 */ 841 static int gpmi_nfc_compute_timings(struct gpmi_nand_data *this, 842 const struct nand_sdr_timings *sdr) 843 { 844 struct gpmi_nfc_hardware_timing *hw = &this->hw; 845 struct resources *r = &this->resources; 846 unsigned int dll_threshold_ps = this->devdata->max_chain_delay; 847 unsigned int period_ps, reference_period_ps; 848 unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles; 849 unsigned int tRP_ps; 850 bool use_half_period; 851 int sample_delay_ps, sample_delay_factor; 852 unsigned int busy_timeout_cycles; 853 u8 wrn_dly_sel; 854 unsigned long clk_rate, min_rate; 855 u64 busy_timeout_ps; 856 857 if (sdr->tRC_min >= 30000) { 858 /* ONFI non-EDO modes [0-3] */ 859 hw->clk_rate = 22000000; 860 min_rate = 0; 861 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS; 862 } else if (sdr->tRC_min >= 25000) { 863 /* ONFI EDO mode 4 */ 864 hw->clk_rate = 80000000; 865 min_rate = 22000000; 866 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY; 867 } else { 868 /* ONFI EDO mode 5 */ 869 hw->clk_rate = 100000000; 870 min_rate = 80000000; 871 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY; 872 } 873 874 clk_rate = clk_round_rate(r->clock[0], hw->clk_rate); 875 if (clk_rate <= min_rate) { 876 dev_err(this->dev, "clock setting: expected %ld, got %ld\n", 877 hw->clk_rate, clk_rate); 878 return -ENOTSUPP; 879 } 880 881 hw->clk_rate = clk_rate; 882 /* SDR core timings are given in picoseconds */ 883 period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate); 884 885 addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps); 886 data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps); 887 data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps); 888 busy_timeout_ps = max(sdr->tBERS_max, sdr->tPROG_max); 889 busy_timeout_cycles = TO_CYCLES(busy_timeout_ps, period_ps); 890 891 hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) | 892 BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) | 893 BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles); 894 hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(DIV_ROUND_UP(busy_timeout_cycles, 4096)); 895 896 /* 897 * Derive NFC ideal delay from {3}: 898 * 899 * (tREA + 4000 - tRP) * 8 900 * RDN_DELAY = ----------------------- 901 * RP 902 */ 903 if (period_ps > dll_threshold_ps) { 904 use_half_period = true; 905 reference_period_ps = period_ps / 2; 906 } else { 907 use_half_period = false; 908 reference_period_ps = period_ps; 909 } 910 911 tRP_ps = data_setup_cycles * period_ps; 912 sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8; 913 if (sample_delay_ps > 0) 914 sample_delay_factor = sample_delay_ps / reference_period_ps; 915 else 916 sample_delay_factor = 0; 917 918 hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel); 919 if (sample_delay_factor) 920 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) | 921 BM_GPMI_CTRL1_DLL_ENABLE | 922 (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0); 923 return 0; 924 } 925 926 static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this) 927 { 928 struct gpmi_nfc_hardware_timing *hw = &this->hw; 929 struct resources *r = &this->resources; 930 void __iomem *gpmi_regs = r->gpmi_regs; 931 unsigned int dll_wait_time_us; 932 int ret; 933 934 /* Clock dividers do NOT guarantee a clean clock signal on its output 935 * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8, 936 * all clock dividers provide these guarantee. 937 */ 938 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) 939 clk_disable_unprepare(r->clock[0]); 940 941 ret = clk_set_rate(r->clock[0], hw->clk_rate); 942 if (ret) { 943 dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret); 944 return ret; 945 } 946 947 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) { 948 ret = clk_prepare_enable(r->clock[0]); 949 if (ret) 950 return ret; 951 } 952 953 writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0); 954 writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1); 955 956 /* 957 * Clear several CTRL1 fields, DLL must be disabled when setting 958 * RDN_DELAY or HALF_PERIOD. 959 */ 960 writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR); 961 writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET); 962 963 /* Wait 64 clock cycles before using the GPMI after enabling the DLL */ 964 dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64; 965 if (!dll_wait_time_us) 966 dll_wait_time_us = 1; 967 968 /* Wait for the DLL to settle. */ 969 udelay(dll_wait_time_us); 970 971 return 0; 972 } 973 974 static int gpmi_setup_interface(struct nand_chip *chip, int chipnr, 975 const struct nand_interface_config *conf) 976 { 977 struct gpmi_nand_data *this = nand_get_controller_data(chip); 978 const struct nand_sdr_timings *sdr; 979 int ret; 980 981 /* Retrieve required NAND timings */ 982 sdr = nand_get_sdr_timings(conf); 983 if (IS_ERR(sdr)) 984 return PTR_ERR(sdr); 985 986 /* Only MX28/MX6 GPMI controller can reach EDO timings */ 987 if (sdr->tRC_min <= 25000 && !this->devdata->support_edo_timing) 988 return -ENOTSUPP; 989 990 /* Stop here if this call was just a check */ 991 if (chipnr < 0) 992 return 0; 993 994 /* Do the actual derivation of the controller timings */ 995 ret = gpmi_nfc_compute_timings(this, sdr); 996 if (ret) 997 return ret; 998 999 this->hw.must_apply_timings = true; 1000 1001 return 0; 1002 } 1003 1004 /* Clears a BCH interrupt. */ 1005 static void gpmi_clear_bch(struct gpmi_nand_data *this) 1006 { 1007 struct resources *r = &this->resources; 1008 writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR); 1009 } 1010 1011 static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this) 1012 { 1013 /* We use the DMA channel 0 to access all the nand chips. */ 1014 return this->dma_chans[0]; 1015 } 1016 1017 /* This will be called after the DMA operation is finished. */ 1018 static void dma_irq_callback(void *param) 1019 { 1020 struct gpmi_nand_data *this = param; 1021 struct completion *dma_c = &this->dma_done; 1022 1023 complete(dma_c); 1024 } 1025 1026 static irqreturn_t bch_irq(int irq, void *cookie) 1027 { 1028 struct gpmi_nand_data *this = cookie; 1029 1030 gpmi_clear_bch(this); 1031 complete(&this->bch_done); 1032 return IRQ_HANDLED; 1033 } 1034 1035 static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len) 1036 { 1037 /* 1038 * raw_len is the length to read/write including bch data which 1039 * we are passed in exec_op. Calculate the data length from it. 1040 */ 1041 if (this->bch) 1042 return ALIGN_DOWN(raw_len, this->bch_geometry.eccn_chunk_size); 1043 else 1044 return raw_len; 1045 } 1046 1047 /* Can we use the upper's buffer directly for DMA? */ 1048 static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf, 1049 int raw_len, struct scatterlist *sgl, 1050 enum dma_data_direction dr) 1051 { 1052 int ret; 1053 int len = gpmi_raw_len_to_len(this, raw_len); 1054 1055 /* first try to map the upper buffer directly */ 1056 if (virt_addr_valid(buf) && !object_is_on_stack(buf)) { 1057 sg_init_one(sgl, buf, len); 1058 ret = dma_map_sg(this->dev, sgl, 1, dr); 1059 if (ret == 0) 1060 goto map_fail; 1061 1062 return true; 1063 } 1064 1065 map_fail: 1066 /* We have to use our own DMA buffer. */ 1067 sg_init_one(sgl, this->data_buffer_dma, len); 1068 1069 if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma) 1070 memcpy(this->data_buffer_dma, buf, len); 1071 1072 dma_map_sg(this->dev, sgl, 1, dr); 1073 1074 return false; 1075 } 1076 1077 /* add our owner bbt descriptor */ 1078 static uint8_t scan_ff_pattern[] = { 0xff }; 1079 static struct nand_bbt_descr gpmi_bbt_descr = { 1080 .options = 0, 1081 .offs = 0, 1082 .len = 1, 1083 .pattern = scan_ff_pattern 1084 }; 1085 1086 /* 1087 * We may change the layout if we can get the ECC info from the datasheet, 1088 * else we will use all the (page + OOB). 1089 */ 1090 static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section, 1091 struct mtd_oob_region *oobregion) 1092 { 1093 struct nand_chip *chip = mtd_to_nand(mtd); 1094 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1095 struct bch_geometry *geo = &this->bch_geometry; 1096 1097 if (section) 1098 return -ERANGE; 1099 1100 oobregion->offset = 0; 1101 oobregion->length = geo->page_size - mtd->writesize; 1102 1103 return 0; 1104 } 1105 1106 static int gpmi_ooblayout_free(struct mtd_info *mtd, int section, 1107 struct mtd_oob_region *oobregion) 1108 { 1109 struct nand_chip *chip = mtd_to_nand(mtd); 1110 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1111 struct bch_geometry *geo = &this->bch_geometry; 1112 1113 if (section) 1114 return -ERANGE; 1115 1116 /* The available oob size we have. */ 1117 if (geo->page_size < mtd->writesize + mtd->oobsize) { 1118 oobregion->offset = geo->page_size - mtd->writesize; 1119 oobregion->length = mtd->oobsize - oobregion->offset; 1120 } 1121 1122 return 0; 1123 } 1124 1125 static const char * const gpmi_clks_for_mx2x[] = { 1126 "gpmi_io", 1127 }; 1128 1129 static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = { 1130 .ecc = gpmi_ooblayout_ecc, 1131 .free = gpmi_ooblayout_free, 1132 }; 1133 1134 static const struct gpmi_devdata gpmi_devdata_imx23 = { 1135 .type = IS_MX23, 1136 .bch_max_ecc_strength = 20, 1137 .max_chain_delay = 16000, 1138 .clks = gpmi_clks_for_mx2x, 1139 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x), 1140 }; 1141 1142 static const struct gpmi_devdata gpmi_devdata_imx28 = { 1143 .type = IS_MX28, 1144 .bch_max_ecc_strength = 20, 1145 .max_chain_delay = 16000, 1146 .support_edo_timing = true, 1147 .clks = gpmi_clks_for_mx2x, 1148 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x), 1149 }; 1150 1151 static const char * const gpmi_clks_for_mx6[] = { 1152 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch", 1153 }; 1154 1155 static const struct gpmi_devdata gpmi_devdata_imx6q = { 1156 .type = IS_MX6Q, 1157 .bch_max_ecc_strength = 40, 1158 .max_chain_delay = 12000, 1159 .support_edo_timing = true, 1160 .clks = gpmi_clks_for_mx6, 1161 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6), 1162 }; 1163 1164 static const struct gpmi_devdata gpmi_devdata_imx6sx = { 1165 .type = IS_MX6SX, 1166 .bch_max_ecc_strength = 62, 1167 .max_chain_delay = 12000, 1168 .support_edo_timing = true, 1169 .clks = gpmi_clks_for_mx6, 1170 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6), 1171 }; 1172 1173 static const char * const gpmi_clks_for_mx7d[] = { 1174 "gpmi_io", "gpmi_bch_apb", 1175 }; 1176 1177 static const struct gpmi_devdata gpmi_devdata_imx7d = { 1178 .type = IS_MX7D, 1179 .bch_max_ecc_strength = 62, 1180 .max_chain_delay = 12000, 1181 .support_edo_timing = true, 1182 .clks = gpmi_clks_for_mx7d, 1183 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d), 1184 }; 1185 1186 static const char *gpmi_clks_for_mx8qxp[GPMI_CLK_MAX] = { 1187 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", 1188 }; 1189 1190 static const struct gpmi_devdata gpmi_devdata_imx8qxp = { 1191 .type = IS_MX8QXP, 1192 .bch_max_ecc_strength = 62, 1193 .max_chain_delay = 12000, 1194 .support_edo_timing = true, 1195 .clks = gpmi_clks_for_mx8qxp, 1196 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx8qxp), 1197 }; 1198 1199 static int acquire_register_block(struct gpmi_nand_data *this, 1200 const char *res_name) 1201 { 1202 struct platform_device *pdev = this->pdev; 1203 struct resources *res = &this->resources; 1204 void __iomem *p; 1205 1206 p = devm_platform_ioremap_resource_byname(pdev, res_name); 1207 if (IS_ERR(p)) 1208 return PTR_ERR(p); 1209 1210 if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME)) 1211 res->gpmi_regs = p; 1212 else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME)) 1213 res->bch_regs = p; 1214 else 1215 dev_err(this->dev, "unknown resource name : %s\n", res_name); 1216 1217 return 0; 1218 } 1219 1220 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h) 1221 { 1222 struct platform_device *pdev = this->pdev; 1223 const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME; 1224 int err; 1225 1226 err = platform_get_irq_byname(pdev, res_name); 1227 if (err < 0) 1228 return err; 1229 1230 err = devm_request_irq(this->dev, err, irq_h, 0, res_name, this); 1231 if (err) 1232 dev_err(this->dev, "error requesting BCH IRQ\n"); 1233 1234 return err; 1235 } 1236 1237 static void release_dma_channels(struct gpmi_nand_data *this) 1238 { 1239 unsigned int i; 1240 for (i = 0; i < DMA_CHANS; i++) 1241 if (this->dma_chans[i]) { 1242 dma_release_channel(this->dma_chans[i]); 1243 this->dma_chans[i] = NULL; 1244 } 1245 } 1246 1247 static int acquire_dma_channels(struct gpmi_nand_data *this) 1248 { 1249 struct platform_device *pdev = this->pdev; 1250 struct dma_chan *dma_chan; 1251 int ret = 0; 1252 1253 /* request dma channel */ 1254 dma_chan = dma_request_chan(&pdev->dev, "rx-tx"); 1255 if (IS_ERR(dma_chan)) { 1256 ret = dev_err_probe(this->dev, PTR_ERR(dma_chan), 1257 "DMA channel request failed\n"); 1258 release_dma_channels(this); 1259 } else { 1260 this->dma_chans[0] = dma_chan; 1261 } 1262 1263 return ret; 1264 } 1265 1266 static int gpmi_get_clks(struct gpmi_nand_data *this) 1267 { 1268 struct resources *r = &this->resources; 1269 struct clk *clk; 1270 int err, i; 1271 1272 for (i = 0; i < this->devdata->clks_count; i++) { 1273 clk = devm_clk_get(this->dev, this->devdata->clks[i]); 1274 if (IS_ERR(clk)) { 1275 err = PTR_ERR(clk); 1276 goto err_clock; 1277 } 1278 1279 r->clock[i] = clk; 1280 } 1281 1282 return 0; 1283 1284 err_clock: 1285 dev_dbg(this->dev, "failed in finding the clocks.\n"); 1286 return err; 1287 } 1288 1289 static int acquire_resources(struct gpmi_nand_data *this) 1290 { 1291 int ret; 1292 1293 ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME); 1294 if (ret) 1295 goto exit_regs; 1296 1297 ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME); 1298 if (ret) 1299 goto exit_regs; 1300 1301 ret = acquire_bch_irq(this, bch_irq); 1302 if (ret) 1303 goto exit_regs; 1304 1305 ret = acquire_dma_channels(this); 1306 if (ret) 1307 goto exit_regs; 1308 1309 ret = gpmi_get_clks(this); 1310 if (ret) 1311 goto exit_clock; 1312 return 0; 1313 1314 exit_clock: 1315 release_dma_channels(this); 1316 exit_regs: 1317 return ret; 1318 } 1319 1320 static void release_resources(struct gpmi_nand_data *this) 1321 { 1322 release_dma_channels(this); 1323 } 1324 1325 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this) 1326 { 1327 struct device *dev = this->dev; 1328 struct bch_geometry *geo = &this->bch_geometry; 1329 1330 if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt)) 1331 dma_free_coherent(dev, geo->auxiliary_size, 1332 this->auxiliary_virt, 1333 this->auxiliary_phys); 1334 kfree(this->data_buffer_dma); 1335 kfree(this->raw_buffer); 1336 1337 this->data_buffer_dma = NULL; 1338 this->raw_buffer = NULL; 1339 } 1340 1341 /* Allocate the DMA buffers */ 1342 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this) 1343 { 1344 struct bch_geometry *geo = &this->bch_geometry; 1345 struct device *dev = this->dev; 1346 struct mtd_info *mtd = nand_to_mtd(&this->nand); 1347 1348 /* 1349 * [2] Allocate a read/write data buffer. 1350 * The gpmi_alloc_dma_buffer can be called twice. 1351 * We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer 1352 * is called before the NAND identification; and we allocate a 1353 * buffer of the real NAND page size when the gpmi_alloc_dma_buffer 1354 * is called after. 1355 */ 1356 this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE, 1357 GFP_DMA | GFP_KERNEL); 1358 if (this->data_buffer_dma == NULL) 1359 goto error_alloc; 1360 1361 this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size, 1362 &this->auxiliary_phys, GFP_DMA); 1363 if (!this->auxiliary_virt) 1364 goto error_alloc; 1365 1366 this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL); 1367 if (!this->raw_buffer) 1368 goto error_alloc; 1369 1370 return 0; 1371 1372 error_alloc: 1373 gpmi_free_dma_buffer(this); 1374 return -ENOMEM; 1375 } 1376 1377 /* 1378 * Handles block mark swapping. 1379 * It can be called in swapping the block mark, or swapping it back, 1380 * because the operations are the same. 1381 */ 1382 static void block_mark_swapping(struct gpmi_nand_data *this, 1383 void *payload, void *auxiliary) 1384 { 1385 struct bch_geometry *nfc_geo = &this->bch_geometry; 1386 unsigned char *p; 1387 unsigned char *a; 1388 unsigned int bit; 1389 unsigned char mask; 1390 unsigned char from_data; 1391 unsigned char from_oob; 1392 1393 if (!this->swap_block_mark) 1394 return; 1395 1396 /* 1397 * If control arrives here, we're swapping. Make some convenience 1398 * variables. 1399 */ 1400 bit = nfc_geo->block_mark_bit_offset; 1401 p = payload + nfc_geo->block_mark_byte_offset; 1402 a = auxiliary; 1403 1404 /* 1405 * Get the byte from the data area that overlays the block mark. Since 1406 * the ECC engine applies its own view to the bits in the page, the 1407 * physical block mark won't (in general) appear on a byte boundary in 1408 * the data. 1409 */ 1410 from_data = (p[0] >> bit) | (p[1] << (8 - bit)); 1411 1412 /* Get the byte from the OOB. */ 1413 from_oob = a[0]; 1414 1415 /* Swap them. */ 1416 a[0] = from_data; 1417 1418 mask = (0x1 << bit) - 1; 1419 p[0] = (p[0] & mask) | (from_oob << bit); 1420 1421 mask = ~0 << bit; 1422 p[1] = (p[1] & mask) | (from_oob >> (8 - bit)); 1423 } 1424 1425 static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first, 1426 int last, int meta) 1427 { 1428 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1429 struct bch_geometry *nfc_geo = &this->bch_geometry; 1430 struct mtd_info *mtd = nand_to_mtd(chip); 1431 int i; 1432 unsigned char *status; 1433 unsigned int max_bitflips = 0; 1434 1435 /* Loop over status bytes, accumulating ECC status. */ 1436 status = this->auxiliary_virt + ALIGN(meta, 4); 1437 1438 for (i = first; i < last; i++, status++) { 1439 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED)) 1440 continue; 1441 1442 if (*status == STATUS_UNCORRECTABLE) { 1443 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1444 u8 *eccbuf = this->raw_buffer; 1445 int offset, bitoffset; 1446 int eccbytes; 1447 int flips; 1448 1449 /* Read ECC bytes into our internal raw_buffer */ 1450 offset = nfc_geo->metadata_size * 8; 1451 offset += ((8 * nfc_geo->eccn_chunk_size) + eccbits) * (i + 1); 1452 offset -= eccbits; 1453 bitoffset = offset % 8; 1454 eccbytes = DIV_ROUND_UP(offset + eccbits, 8); 1455 offset /= 8; 1456 eccbytes -= offset; 1457 nand_change_read_column_op(chip, offset, eccbuf, 1458 eccbytes, false); 1459 1460 /* 1461 * ECC data are not byte aligned and we may have 1462 * in-band data in the first and last byte of 1463 * eccbuf. Set non-eccbits to one so that 1464 * nand_check_erased_ecc_chunk() does not count them 1465 * as bitflips. 1466 */ 1467 if (bitoffset) 1468 eccbuf[0] |= GENMASK(bitoffset - 1, 0); 1469 1470 bitoffset = (bitoffset + eccbits) % 8; 1471 if (bitoffset) 1472 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset); 1473 1474 /* 1475 * The ECC hardware has an uncorrectable ECC status 1476 * code in case we have bitflips in an erased page. As 1477 * nothing was written into this subpage the ECC is 1478 * obviously wrong and we can not trust it. We assume 1479 * at this point that we are reading an erased page and 1480 * try to correct the bitflips in buffer up to 1481 * ecc_strength bitflips. If this is a page with random 1482 * data, we exceed this number of bitflips and have a 1483 * ECC failure. Otherwise we use the corrected buffer. 1484 */ 1485 if (i == 0) { 1486 /* The first block includes metadata */ 1487 flips = nand_check_erased_ecc_chunk( 1488 buf + i * nfc_geo->eccn_chunk_size, 1489 nfc_geo->eccn_chunk_size, 1490 eccbuf, eccbytes, 1491 this->auxiliary_virt, 1492 nfc_geo->metadata_size, 1493 nfc_geo->ecc_strength); 1494 } else { 1495 flips = nand_check_erased_ecc_chunk( 1496 buf + i * nfc_geo->eccn_chunk_size, 1497 nfc_geo->eccn_chunk_size, 1498 eccbuf, eccbytes, 1499 NULL, 0, 1500 nfc_geo->ecc_strength); 1501 } 1502 1503 if (flips > 0) { 1504 max_bitflips = max_t(unsigned int, max_bitflips, 1505 flips); 1506 mtd->ecc_stats.corrected += flips; 1507 continue; 1508 } 1509 1510 mtd->ecc_stats.failed++; 1511 continue; 1512 } 1513 1514 mtd->ecc_stats.corrected += *status; 1515 max_bitflips = max_t(unsigned int, max_bitflips, *status); 1516 } 1517 1518 return max_bitflips; 1519 } 1520 1521 static void gpmi_bch_layout_std(struct gpmi_nand_data *this) 1522 { 1523 struct bch_geometry *geo = &this->bch_geometry; 1524 unsigned int ecc_strength = geo->ecc_strength >> 1; 1525 unsigned int gf_len = geo->gf_len; 1526 unsigned int block0_size = geo->ecc0_chunk_size; 1527 unsigned int blockn_size = geo->eccn_chunk_size; 1528 1529 this->bch_flashlayout0 = 1530 BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) | 1531 BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) | 1532 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) | 1533 BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) | 1534 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block0_size, this); 1535 1536 this->bch_flashlayout1 = 1537 BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) | 1538 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) | 1539 BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) | 1540 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(blockn_size, this); 1541 } 1542 1543 static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf, 1544 int oob_required, int page) 1545 { 1546 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1547 struct mtd_info *mtd = nand_to_mtd(chip); 1548 struct bch_geometry *geo = &this->bch_geometry; 1549 unsigned int max_bitflips; 1550 int ret; 1551 1552 gpmi_bch_layout_std(this); 1553 this->bch = true; 1554 1555 ret = nand_read_page_op(chip, page, 0, buf, geo->page_size); 1556 if (ret) 1557 return ret; 1558 1559 max_bitflips = gpmi_count_bitflips(chip, buf, 0, 1560 geo->ecc_chunk_count, 1561 geo->auxiliary_status_offset); 1562 1563 /* handle the block mark swapping */ 1564 block_mark_swapping(this, buf, this->auxiliary_virt); 1565 1566 if (oob_required) { 1567 /* 1568 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob() 1569 * for details about our policy for delivering the OOB. 1570 * 1571 * We fill the caller's buffer with set bits, and then copy the 1572 * block mark to th caller's buffer. Note that, if block mark 1573 * swapping was necessary, it has already been done, so we can 1574 * rely on the first byte of the auxiliary buffer to contain 1575 * the block mark. 1576 */ 1577 memset(chip->oob_poi, ~0, mtd->oobsize); 1578 chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0]; 1579 } 1580 1581 return max_bitflips; 1582 } 1583 1584 /* Fake a virtual small page for the subpage read */ 1585 static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs, 1586 uint32_t len, uint8_t *buf, int page) 1587 { 1588 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1589 struct bch_geometry *geo = &this->bch_geometry; 1590 int size = chip->ecc.size; /* ECC chunk size */ 1591 int meta, n, page_size; 1592 unsigned int max_bitflips; 1593 unsigned int ecc_strength; 1594 int first, last, marker_pos; 1595 int ecc_parity_size; 1596 int col = 0; 1597 int ret; 1598 1599 /* The size of ECC parity */ 1600 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8; 1601 1602 /* Align it with the chunk size */ 1603 first = offs / size; 1604 last = (offs + len - 1) / size; 1605 1606 if (this->swap_block_mark) { 1607 /* 1608 * Find the chunk which contains the Block Marker. 1609 * If this chunk is in the range of [first, last], 1610 * we have to read out the whole page. 1611 * Why? since we had swapped the data at the position of Block 1612 * Marker to the metadata which is bound with the chunk 0. 1613 */ 1614 marker_pos = geo->block_mark_byte_offset / size; 1615 if (last >= marker_pos && first <= marker_pos) { 1616 dev_dbg(this->dev, 1617 "page:%d, first:%d, last:%d, marker at:%d\n", 1618 page, first, last, marker_pos); 1619 return gpmi_ecc_read_page(chip, buf, 0, page); 1620 } 1621 } 1622 1623 /* 1624 * if there is an ECC dedicate for meta: 1625 * - need to add an extra ECC size when calculating col and page_size, 1626 * if the meta size is NOT zero. 1627 * - ecc0_chunk size need to set to the same size as other chunks, 1628 * if the meta size is zero. 1629 */ 1630 1631 meta = geo->metadata_size; 1632 if (first) { 1633 if (geo->ecc_for_meta) 1634 col = meta + ecc_parity_size 1635 + (size + ecc_parity_size) * first; 1636 else 1637 col = meta + (size + ecc_parity_size) * first; 1638 1639 meta = 0; 1640 buf = buf + first * size; 1641 } 1642 1643 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8; 1644 n = last - first + 1; 1645 1646 if (geo->ecc_for_meta && meta) 1647 page_size = meta + ecc_parity_size 1648 + (size + ecc_parity_size) * n; 1649 else 1650 page_size = meta + (size + ecc_parity_size) * n; 1651 1652 ecc_strength = geo->ecc_strength >> 1; 1653 1654 this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS( 1655 (geo->ecc_for_meta ? n : n - 1)) | 1656 BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) | 1657 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) | 1658 BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) | 1659 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE((geo->ecc_for_meta ? 1660 0 : geo->ecc0_chunk_size), this); 1661 1662 this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) | 1663 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) | 1664 BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) | 1665 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->eccn_chunk_size, this); 1666 1667 this->bch = true; 1668 1669 ret = nand_read_page_op(chip, page, col, buf, page_size); 1670 if (ret) 1671 return ret; 1672 1673 dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n", 1674 page, offs, len, col, first, n, page_size); 1675 1676 max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta); 1677 1678 return max_bitflips; 1679 } 1680 1681 static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf, 1682 int oob_required, int page) 1683 { 1684 struct mtd_info *mtd = nand_to_mtd(chip); 1685 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1686 struct bch_geometry *nfc_geo = &this->bch_geometry; 1687 1688 dev_dbg(this->dev, "ecc write page.\n"); 1689 1690 gpmi_bch_layout_std(this); 1691 this->bch = true; 1692 1693 memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size); 1694 1695 if (this->swap_block_mark) { 1696 /* 1697 * When doing bad block marker swapping we must always copy the 1698 * input buffer as we can't modify the const buffer. 1699 */ 1700 memcpy(this->data_buffer_dma, buf, mtd->writesize); 1701 buf = this->data_buffer_dma; 1702 block_mark_swapping(this, this->data_buffer_dma, 1703 this->auxiliary_virt); 1704 } 1705 1706 return nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size); 1707 } 1708 1709 /* 1710 * There are several places in this driver where we have to handle the OOB and 1711 * block marks. This is the function where things are the most complicated, so 1712 * this is where we try to explain it all. All the other places refer back to 1713 * here. 1714 * 1715 * These are the rules, in order of decreasing importance: 1716 * 1717 * 1) Nothing the caller does can be allowed to imperil the block mark. 1718 * 1719 * 2) In read operations, the first byte of the OOB we return must reflect the 1720 * true state of the block mark, no matter where that block mark appears in 1721 * the physical page. 1722 * 1723 * 3) ECC-based read operations return an OOB full of set bits (since we never 1724 * allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads 1725 * return). 1726 * 1727 * 4) "Raw" read operations return a direct view of the physical bytes in the 1728 * page, using the conventional definition of which bytes are data and which 1729 * are OOB. This gives the caller a way to see the actual, physical bytes 1730 * in the page, without the distortions applied by our ECC engine. 1731 * 1732 * 1733 * What we do for this specific read operation depends on two questions: 1734 * 1735 * 1) Are we doing a "raw" read, or an ECC-based read? 1736 * 1737 * 2) Are we using block mark swapping or transcription? 1738 * 1739 * There are four cases, illustrated by the following Karnaugh map: 1740 * 1741 * | Raw | ECC-based | 1742 * -------------+-------------------------+-------------------------+ 1743 * | Read the conventional | | 1744 * | OOB at the end of the | | 1745 * Swapping | page and return it. It | | 1746 * | contains exactly what | | 1747 * | we want. | Read the block mark and | 1748 * -------------+-------------------------+ return it in a buffer | 1749 * | Read the conventional | full of set bits. | 1750 * | OOB at the end of the | | 1751 * | page and also the block | | 1752 * Transcribing | mark in the metadata. | | 1753 * | Copy the block mark | | 1754 * | into the first byte of | | 1755 * | the OOB. | | 1756 * -------------+-------------------------+-------------------------+ 1757 * 1758 * Note that we break rule #4 in the Transcribing/Raw case because we're not 1759 * giving an accurate view of the actual, physical bytes in the page (we're 1760 * overwriting the block mark). That's OK because it's more important to follow 1761 * rule #2. 1762 * 1763 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not 1764 * easy. When reading a page, for example, the NAND Flash MTD code calls our 1765 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an 1766 * ECC-based or raw view of the page is implicit in which function it calls 1767 * (there is a similar pair of ECC-based/raw functions for writing). 1768 */ 1769 static int gpmi_ecc_read_oob(struct nand_chip *chip, int page) 1770 { 1771 struct mtd_info *mtd = nand_to_mtd(chip); 1772 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1773 int ret; 1774 1775 /* clear the OOB buffer */ 1776 memset(chip->oob_poi, ~0, mtd->oobsize); 1777 1778 /* Read out the conventional OOB. */ 1779 ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi, 1780 mtd->oobsize); 1781 if (ret) 1782 return ret; 1783 1784 /* 1785 * Now, we want to make sure the block mark is correct. In the 1786 * non-transcribing case (!GPMI_IS_MX23()), we already have it. 1787 * Otherwise, we need to explicitly read it. 1788 */ 1789 if (GPMI_IS_MX23(this)) { 1790 /* Read the block mark into the first byte of the OOB buffer. */ 1791 ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1); 1792 if (ret) 1793 return ret; 1794 } 1795 1796 return 0; 1797 } 1798 1799 static int gpmi_ecc_write_oob(struct nand_chip *chip, int page) 1800 { 1801 struct mtd_info *mtd = nand_to_mtd(chip); 1802 struct mtd_oob_region of = { }; 1803 1804 /* Do we have available oob area? */ 1805 mtd_ooblayout_free(mtd, 0, &of); 1806 if (!of.length) 1807 return -EPERM; 1808 1809 if (!nand_is_slc(chip)) 1810 return -EPERM; 1811 1812 return nand_prog_page_op(chip, page, mtd->writesize + of.offset, 1813 chip->oob_poi + of.offset, of.length); 1814 } 1815 1816 /* 1817 * This function reads a NAND page without involving the ECC engine (no HW 1818 * ECC correction). 1819 * The tricky part in the GPMI/BCH controller is that it stores ECC bits 1820 * inline (interleaved with payload DATA), and do not align data chunk on 1821 * byte boundaries. 1822 * We thus need to take care moving the payload data and ECC bits stored in the 1823 * page into the provided buffers, which is why we're using nand_extract_bits(). 1824 * 1825 * See set_geometry_by_ecc_info inline comments to have a full description 1826 * of the layout used by the GPMI controller. 1827 */ 1828 static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 1829 int oob_required, int page) 1830 { 1831 struct mtd_info *mtd = nand_to_mtd(chip); 1832 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1833 struct bch_geometry *nfc_geo = &this->bch_geometry; 1834 int eccsize = nfc_geo->eccn_chunk_size; 1835 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1836 u8 *tmp_buf = this->raw_buffer; 1837 size_t src_bit_off; 1838 size_t oob_bit_off; 1839 size_t oob_byte_off; 1840 uint8_t *oob = chip->oob_poi; 1841 int step; 1842 int ret; 1843 1844 ret = nand_read_page_op(chip, page, 0, tmp_buf, 1845 mtd->writesize + mtd->oobsize); 1846 if (ret) 1847 return ret; 1848 1849 /* 1850 * If required, swap the bad block marker and the data stored in the 1851 * metadata section, so that we don't wrongly consider a block as bad. 1852 * 1853 * See the layout description for a detailed explanation on why this 1854 * is needed. 1855 */ 1856 if (this->swap_block_mark) 1857 swap(tmp_buf[0], tmp_buf[mtd->writesize]); 1858 1859 /* 1860 * Copy the metadata section into the oob buffer (this section is 1861 * guaranteed to be aligned on a byte boundary). 1862 */ 1863 if (oob_required) 1864 memcpy(oob, tmp_buf, nfc_geo->metadata_size); 1865 1866 oob_bit_off = nfc_geo->metadata_size * 8; 1867 src_bit_off = oob_bit_off; 1868 1869 /* Extract interleaved payload data and ECC bits */ 1870 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) { 1871 if (buf) 1872 nand_extract_bits(buf, step * eccsize * 8, tmp_buf, 1873 src_bit_off, eccsize * 8); 1874 src_bit_off += eccsize * 8; 1875 1876 /* Align last ECC block to align a byte boundary */ 1877 if (step == nfc_geo->ecc_chunk_count - 1 && 1878 (oob_bit_off + eccbits) % 8) 1879 eccbits += 8 - ((oob_bit_off + eccbits) % 8); 1880 1881 if (oob_required) 1882 nand_extract_bits(oob, oob_bit_off, tmp_buf, 1883 src_bit_off, eccbits); 1884 1885 src_bit_off += eccbits; 1886 oob_bit_off += eccbits; 1887 } 1888 1889 if (oob_required) { 1890 oob_byte_off = oob_bit_off / 8; 1891 1892 if (oob_byte_off < mtd->oobsize) 1893 memcpy(oob + oob_byte_off, 1894 tmp_buf + mtd->writesize + oob_byte_off, 1895 mtd->oobsize - oob_byte_off); 1896 } 1897 1898 return 0; 1899 } 1900 1901 /* 1902 * This function writes a NAND page without involving the ECC engine (no HW 1903 * ECC generation). 1904 * The tricky part in the GPMI/BCH controller is that it stores ECC bits 1905 * inline (interleaved with payload DATA), and do not align data chunk on 1906 * byte boundaries. 1907 * We thus need to take care moving the OOB area at the right place in the 1908 * final page, which is why we're using nand_extract_bits(). 1909 * 1910 * See set_geometry_by_ecc_info inline comments to have a full description 1911 * of the layout used by the GPMI controller. 1912 */ 1913 static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf, 1914 int oob_required, int page) 1915 { 1916 struct mtd_info *mtd = nand_to_mtd(chip); 1917 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1918 struct bch_geometry *nfc_geo = &this->bch_geometry; 1919 int eccsize = nfc_geo->eccn_chunk_size; 1920 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1921 u8 *tmp_buf = this->raw_buffer; 1922 uint8_t *oob = chip->oob_poi; 1923 size_t dst_bit_off; 1924 size_t oob_bit_off; 1925 size_t oob_byte_off; 1926 int step; 1927 1928 /* 1929 * Initialize all bits to 1 in case we don't have a buffer for the 1930 * payload or oob data in order to leave unspecified bits of data 1931 * to their initial state. 1932 */ 1933 if (!buf || !oob_required) 1934 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize); 1935 1936 /* 1937 * First copy the metadata section (stored in oob buffer) at the 1938 * beginning of the page, as imposed by the GPMI layout. 1939 */ 1940 memcpy(tmp_buf, oob, nfc_geo->metadata_size); 1941 oob_bit_off = nfc_geo->metadata_size * 8; 1942 dst_bit_off = oob_bit_off; 1943 1944 /* Interleave payload data and ECC bits */ 1945 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) { 1946 if (buf) 1947 nand_extract_bits(tmp_buf, dst_bit_off, buf, 1948 step * eccsize * 8, eccsize * 8); 1949 dst_bit_off += eccsize * 8; 1950 1951 /* Align last ECC block to align a byte boundary */ 1952 if (step == nfc_geo->ecc_chunk_count - 1 && 1953 (oob_bit_off + eccbits) % 8) 1954 eccbits += 8 - ((oob_bit_off + eccbits) % 8); 1955 1956 if (oob_required) 1957 nand_extract_bits(tmp_buf, dst_bit_off, oob, 1958 oob_bit_off, eccbits); 1959 1960 dst_bit_off += eccbits; 1961 oob_bit_off += eccbits; 1962 } 1963 1964 oob_byte_off = oob_bit_off / 8; 1965 1966 if (oob_required && oob_byte_off < mtd->oobsize) 1967 memcpy(tmp_buf + mtd->writesize + oob_byte_off, 1968 oob + oob_byte_off, mtd->oobsize - oob_byte_off); 1969 1970 /* 1971 * If required, swap the bad block marker and the first byte of the 1972 * metadata section, so that we don't modify the bad block marker. 1973 * 1974 * See the layout description for a detailed explanation on why this 1975 * is needed. 1976 */ 1977 if (this->swap_block_mark) 1978 swap(tmp_buf[0], tmp_buf[mtd->writesize]); 1979 1980 return nand_prog_page_op(chip, page, 0, tmp_buf, 1981 mtd->writesize + mtd->oobsize); 1982 } 1983 1984 static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page) 1985 { 1986 return gpmi_ecc_read_page_raw(chip, NULL, 1, page); 1987 } 1988 1989 static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page) 1990 { 1991 return gpmi_ecc_write_page_raw(chip, NULL, 1, page); 1992 } 1993 1994 static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs) 1995 { 1996 struct mtd_info *mtd = nand_to_mtd(chip); 1997 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1998 int ret = 0; 1999 uint8_t *block_mark; 2000 int column, page, chipnr; 2001 2002 chipnr = (int)(ofs >> chip->chip_shift); 2003 nand_select_target(chip, chipnr); 2004 2005 column = !GPMI_IS_MX23(this) ? mtd->writesize : 0; 2006 2007 /* Write the block mark. */ 2008 block_mark = this->data_buffer_dma; 2009 block_mark[0] = 0; /* bad block marker */ 2010 2011 /* Shift to get page */ 2012 page = (int)(ofs >> chip->page_shift); 2013 2014 ret = nand_prog_page_op(chip, page, column, block_mark, 1); 2015 2016 nand_deselect_target(chip); 2017 2018 return ret; 2019 } 2020 2021 static int nand_boot_set_geometry(struct gpmi_nand_data *this) 2022 { 2023 struct boot_rom_geometry *geometry = &this->rom_geometry; 2024 2025 /* 2026 * Set the boot block stride size. 2027 * 2028 * In principle, we should be reading this from the OTP bits, since 2029 * that's where the ROM is going to get it. In fact, we don't have any 2030 * way to read the OTP bits, so we go with the default and hope for the 2031 * best. 2032 */ 2033 geometry->stride_size_in_pages = 64; 2034 2035 /* 2036 * Set the search area stride exponent. 2037 * 2038 * In principle, we should be reading this from the OTP bits, since 2039 * that's where the ROM is going to get it. In fact, we don't have any 2040 * way to read the OTP bits, so we go with the default and hope for the 2041 * best. 2042 */ 2043 geometry->search_area_stride_exponent = 2; 2044 return 0; 2045 } 2046 2047 static const char *fingerprint = "STMP"; 2048 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) 2049 { 2050 struct boot_rom_geometry *rom_geo = &this->rom_geometry; 2051 struct device *dev = this->dev; 2052 struct nand_chip *chip = &this->nand; 2053 unsigned int search_area_size_in_strides; 2054 unsigned int stride; 2055 unsigned int page; 2056 u8 *buffer = nand_get_data_buf(chip); 2057 int found_an_ncb_fingerprint = false; 2058 int ret; 2059 2060 /* Compute the number of strides in a search area. */ 2061 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; 2062 2063 nand_select_target(chip, 0); 2064 2065 /* 2066 * Loop through the first search area, looking for the NCB fingerprint. 2067 */ 2068 dev_dbg(dev, "Scanning for an NCB fingerprint...\n"); 2069 2070 for (stride = 0; stride < search_area_size_in_strides; stride++) { 2071 /* Compute the page addresses. */ 2072 page = stride * rom_geo->stride_size_in_pages; 2073 2074 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page); 2075 2076 /* 2077 * Read the NCB fingerprint. The fingerprint is four bytes long 2078 * and starts in the 12th byte of the page. 2079 */ 2080 ret = nand_read_page_op(chip, page, 12, buffer, 2081 strlen(fingerprint)); 2082 if (ret) 2083 continue; 2084 2085 /* Look for the fingerprint. */ 2086 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) { 2087 found_an_ncb_fingerprint = true; 2088 break; 2089 } 2090 2091 } 2092 2093 nand_deselect_target(chip); 2094 2095 if (found_an_ncb_fingerprint) 2096 dev_dbg(dev, "\tFound a fingerprint\n"); 2097 else 2098 dev_dbg(dev, "\tNo fingerprint found\n"); 2099 return found_an_ncb_fingerprint; 2100 } 2101 2102 /* Writes a transcription stamp. */ 2103 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this) 2104 { 2105 struct device *dev = this->dev; 2106 struct boot_rom_geometry *rom_geo = &this->rom_geometry; 2107 struct nand_chip *chip = &this->nand; 2108 struct mtd_info *mtd = nand_to_mtd(chip); 2109 unsigned int block_size_in_pages; 2110 unsigned int search_area_size_in_strides; 2111 unsigned int search_area_size_in_pages; 2112 unsigned int search_area_size_in_blocks; 2113 unsigned int block; 2114 unsigned int stride; 2115 unsigned int page; 2116 u8 *buffer = nand_get_data_buf(chip); 2117 int status; 2118 2119 /* Compute the search area geometry. */ 2120 block_size_in_pages = mtd->erasesize / mtd->writesize; 2121 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; 2122 search_area_size_in_pages = search_area_size_in_strides * 2123 rom_geo->stride_size_in_pages; 2124 search_area_size_in_blocks = 2125 (search_area_size_in_pages + (block_size_in_pages - 1)) / 2126 block_size_in_pages; 2127 2128 dev_dbg(dev, "Search Area Geometry :\n"); 2129 dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks); 2130 dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides); 2131 dev_dbg(dev, "\tin Pages : %u\n", search_area_size_in_pages); 2132 2133 nand_select_target(chip, 0); 2134 2135 /* Loop over blocks in the first search area, erasing them. */ 2136 dev_dbg(dev, "Erasing the search area...\n"); 2137 2138 for (block = 0; block < search_area_size_in_blocks; block++) { 2139 /* Erase this block. */ 2140 dev_dbg(dev, "\tErasing block 0x%x\n", block); 2141 status = nand_erase_op(chip, block); 2142 if (status) 2143 dev_err(dev, "[%s] Erase failed.\n", __func__); 2144 } 2145 2146 /* Write the NCB fingerprint into the page buffer. */ 2147 memset(buffer, ~0, mtd->writesize); 2148 memcpy(buffer + 12, fingerprint, strlen(fingerprint)); 2149 2150 /* Loop through the first search area, writing NCB fingerprints. */ 2151 dev_dbg(dev, "Writing NCB fingerprints...\n"); 2152 for (stride = 0; stride < search_area_size_in_strides; stride++) { 2153 /* Compute the page addresses. */ 2154 page = stride * rom_geo->stride_size_in_pages; 2155 2156 /* Write the first page of the current stride. */ 2157 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page); 2158 2159 status = chip->ecc.write_page_raw(chip, buffer, 0, page); 2160 if (status) 2161 dev_err(dev, "[%s] Write failed.\n", __func__); 2162 } 2163 2164 nand_deselect_target(chip); 2165 2166 return 0; 2167 } 2168 2169 static int mx23_boot_init(struct gpmi_nand_data *this) 2170 { 2171 struct device *dev = this->dev; 2172 struct nand_chip *chip = &this->nand; 2173 struct mtd_info *mtd = nand_to_mtd(chip); 2174 unsigned int block_count; 2175 unsigned int block; 2176 int chipnr; 2177 int page; 2178 loff_t byte; 2179 uint8_t block_mark; 2180 int ret = 0; 2181 2182 /* 2183 * If control arrives here, we can't use block mark swapping, which 2184 * means we're forced to use transcription. First, scan for the 2185 * transcription stamp. If we find it, then we don't have to do 2186 * anything -- the block marks are already transcribed. 2187 */ 2188 if (mx23_check_transcription_stamp(this)) 2189 return 0; 2190 2191 /* 2192 * If control arrives here, we couldn't find a transcription stamp, so 2193 * so we presume the block marks are in the conventional location. 2194 */ 2195 dev_dbg(dev, "Transcribing bad block marks...\n"); 2196 2197 /* Compute the number of blocks in the entire medium. */ 2198 block_count = nanddev_eraseblocks_per_target(&chip->base); 2199 2200 /* 2201 * Loop over all the blocks in the medium, transcribing block marks as 2202 * we go. 2203 */ 2204 for (block = 0; block < block_count; block++) { 2205 /* 2206 * Compute the chip, page and byte addresses for this block's 2207 * conventional mark. 2208 */ 2209 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift); 2210 page = block << (chip->phys_erase_shift - chip->page_shift); 2211 byte = block << chip->phys_erase_shift; 2212 2213 /* Send the command to read the conventional block mark. */ 2214 nand_select_target(chip, chipnr); 2215 ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark, 2216 1); 2217 nand_deselect_target(chip); 2218 2219 if (ret) 2220 continue; 2221 2222 /* 2223 * Check if the block is marked bad. If so, we need to mark it 2224 * again, but this time the result will be a mark in the 2225 * location where we transcribe block marks. 2226 */ 2227 if (block_mark != 0xff) { 2228 dev_dbg(dev, "Transcribing mark in block %u\n", block); 2229 ret = chip->legacy.block_markbad(chip, byte); 2230 if (ret) 2231 dev_err(dev, 2232 "Failed to mark block bad with ret %d\n", 2233 ret); 2234 } 2235 } 2236 2237 /* Write the stamp that indicates we've transcribed the block marks. */ 2238 mx23_write_transcription_stamp(this); 2239 return 0; 2240 } 2241 2242 static int nand_boot_init(struct gpmi_nand_data *this) 2243 { 2244 nand_boot_set_geometry(this); 2245 2246 /* This is ROM arch-specific initilization before the BBT scanning. */ 2247 if (GPMI_IS_MX23(this)) 2248 return mx23_boot_init(this); 2249 return 0; 2250 } 2251 2252 static int gpmi_set_geometry(struct gpmi_nand_data *this) 2253 { 2254 int ret; 2255 2256 /* Free the temporary DMA memory for reading ID. */ 2257 gpmi_free_dma_buffer(this); 2258 2259 /* Set up the NFC geometry which is used by BCH. */ 2260 ret = bch_set_geometry(this); 2261 if (ret) { 2262 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret); 2263 return ret; 2264 } 2265 2266 /* Alloc the new DMA buffers according to the pagesize and oobsize */ 2267 return gpmi_alloc_dma_buffer(this); 2268 } 2269 2270 static int gpmi_init_last(struct gpmi_nand_data *this) 2271 { 2272 struct nand_chip *chip = &this->nand; 2273 struct mtd_info *mtd = nand_to_mtd(chip); 2274 struct nand_ecc_ctrl *ecc = &chip->ecc; 2275 struct bch_geometry *bch_geo = &this->bch_geometry; 2276 int ret; 2277 2278 /* Set up the medium geometry */ 2279 ret = gpmi_set_geometry(this); 2280 if (ret) 2281 return ret; 2282 2283 /* Init the nand_ecc_ctrl{} */ 2284 ecc->read_page = gpmi_ecc_read_page; 2285 ecc->write_page = gpmi_ecc_write_page; 2286 ecc->read_oob = gpmi_ecc_read_oob; 2287 ecc->write_oob = gpmi_ecc_write_oob; 2288 ecc->read_page_raw = gpmi_ecc_read_page_raw; 2289 ecc->write_page_raw = gpmi_ecc_write_page_raw; 2290 ecc->read_oob_raw = gpmi_ecc_read_oob_raw; 2291 ecc->write_oob_raw = gpmi_ecc_write_oob_raw; 2292 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2293 ecc->size = bch_geo->eccn_chunk_size; 2294 ecc->strength = bch_geo->ecc_strength; 2295 mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops); 2296 2297 /* 2298 * We only enable the subpage read when: 2299 * (1) the chip is imx6, and 2300 * (2) the size of the ECC parity is byte aligned. 2301 */ 2302 if (GPMI_IS_MX6(this) && 2303 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) { 2304 ecc->read_subpage = gpmi_ecc_read_subpage; 2305 chip->options |= NAND_SUBPAGE_READ; 2306 } 2307 2308 return 0; 2309 } 2310 2311 static int gpmi_nand_attach_chip(struct nand_chip *chip) 2312 { 2313 struct gpmi_nand_data *this = nand_get_controller_data(chip); 2314 int ret; 2315 2316 if (chip->bbt_options & NAND_BBT_USE_FLASH) { 2317 chip->bbt_options |= NAND_BBT_NO_OOB; 2318 2319 if (of_property_read_bool(this->dev->of_node, 2320 "fsl,no-blockmark-swap")) 2321 this->swap_block_mark = false; 2322 } 2323 dev_dbg(this->dev, "Blockmark swapping %s\n", 2324 str_enabled_disabled(this->swap_block_mark)); 2325 2326 ret = gpmi_init_last(this); 2327 if (ret) 2328 return ret; 2329 2330 chip->options |= NAND_SKIP_BBTSCAN; 2331 2332 return 0; 2333 } 2334 2335 static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this) 2336 { 2337 struct gpmi_transfer *transfer = &this->transfers[this->ntransfers]; 2338 2339 this->ntransfers++; 2340 2341 if (this->ntransfers == GPMI_MAX_TRANSFERS) 2342 return NULL; 2343 2344 return transfer; 2345 } 2346 2347 static struct dma_async_tx_descriptor *gpmi_chain_command( 2348 struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr) 2349 { 2350 struct dma_chan *channel = get_dma_chan(this); 2351 struct dma_async_tx_descriptor *desc; 2352 struct gpmi_transfer *transfer; 2353 int chip = this->nand.cur_cs; 2354 u32 pio[3]; 2355 2356 /* [1] send out the PIO words */ 2357 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE) 2358 | BM_GPMI_CTRL0_WORD_LENGTH 2359 | BF_GPMI_CTRL0_CS(chip, this) 2360 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2361 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE) 2362 | BM_GPMI_CTRL0_ADDRESS_INCREMENT 2363 | BF_GPMI_CTRL0_XFER_COUNT(naddr + 1); 2364 pio[1] = 0; 2365 pio[2] = 0; 2366 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2367 DMA_TRANS_NONE, 0); 2368 if (!desc) 2369 return NULL; 2370 2371 transfer = get_next_transfer(this); 2372 if (!transfer) 2373 return NULL; 2374 2375 transfer->cmdbuf[0] = cmd; 2376 if (naddr) 2377 memcpy(&transfer->cmdbuf[1], addr, naddr); 2378 2379 sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1); 2380 dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE); 2381 2382 transfer->direction = DMA_TO_DEVICE; 2383 2384 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV, 2385 MXS_DMA_CTRL_WAIT4END); 2386 return desc; 2387 } 2388 2389 static struct dma_async_tx_descriptor *gpmi_chain_wait_ready( 2390 struct gpmi_nand_data *this) 2391 { 2392 struct dma_chan *channel = get_dma_chan(this); 2393 u32 pio[2]; 2394 2395 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY) 2396 | BM_GPMI_CTRL0_WORD_LENGTH 2397 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2398 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2399 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2400 | BF_GPMI_CTRL0_XFER_COUNT(0); 2401 pio[1] = 0; 2402 2403 return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE, 2404 MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY); 2405 } 2406 2407 static struct dma_async_tx_descriptor *gpmi_chain_data_read( 2408 struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct) 2409 { 2410 struct dma_async_tx_descriptor *desc; 2411 struct dma_chan *channel = get_dma_chan(this); 2412 struct gpmi_transfer *transfer; 2413 u32 pio[6] = {}; 2414 2415 transfer = get_next_transfer(this); 2416 if (!transfer) 2417 return NULL; 2418 2419 transfer->direction = DMA_FROM_DEVICE; 2420 2421 *direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl, 2422 DMA_FROM_DEVICE); 2423 2424 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ) 2425 | BM_GPMI_CTRL0_WORD_LENGTH 2426 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2427 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2428 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2429 | BF_GPMI_CTRL0_XFER_COUNT(raw_len); 2430 2431 if (this->bch) { 2432 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC 2433 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE) 2434 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE 2435 | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY); 2436 pio[3] = raw_len; 2437 pio[4] = transfer->sgl.dma_address; 2438 pio[5] = this->auxiliary_phys; 2439 } 2440 2441 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2442 DMA_TRANS_NONE, 0); 2443 if (!desc) 2444 return NULL; 2445 2446 if (!this->bch) 2447 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, 2448 DMA_DEV_TO_MEM, 2449 MXS_DMA_CTRL_WAIT4END); 2450 2451 return desc; 2452 } 2453 2454 static struct dma_async_tx_descriptor *gpmi_chain_data_write( 2455 struct gpmi_nand_data *this, const void *buf, int raw_len) 2456 { 2457 struct dma_chan *channel = get_dma_chan(this); 2458 struct dma_async_tx_descriptor *desc; 2459 struct gpmi_transfer *transfer; 2460 u32 pio[6] = {}; 2461 2462 transfer = get_next_transfer(this); 2463 if (!transfer) 2464 return NULL; 2465 2466 transfer->direction = DMA_TO_DEVICE; 2467 2468 prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE); 2469 2470 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE) 2471 | BM_GPMI_CTRL0_WORD_LENGTH 2472 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2473 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2474 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2475 | BF_GPMI_CTRL0_XFER_COUNT(raw_len); 2476 2477 if (this->bch) { 2478 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC 2479 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE) 2480 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE | 2481 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY); 2482 pio[3] = raw_len; 2483 pio[4] = transfer->sgl.dma_address; 2484 pio[5] = this->auxiliary_phys; 2485 } 2486 2487 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2488 DMA_TRANS_NONE, 2489 (this->bch ? MXS_DMA_CTRL_WAIT4END : 0)); 2490 if (!desc) 2491 return NULL; 2492 2493 if (!this->bch) 2494 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, 2495 DMA_MEM_TO_DEV, 2496 MXS_DMA_CTRL_WAIT4END); 2497 2498 return desc; 2499 } 2500 2501 static int gpmi_nfc_exec_op(struct nand_chip *chip, 2502 const struct nand_operation *op, 2503 bool check_only) 2504 { 2505 const struct nand_op_instr *instr; 2506 struct gpmi_nand_data *this = nand_get_controller_data(chip); 2507 struct dma_async_tx_descriptor *desc = NULL; 2508 int i, ret, buf_len = 0, nbufs = 0; 2509 u8 cmd = 0; 2510 void *buf_read = NULL; 2511 const void *buf_write = NULL; 2512 bool direct = false; 2513 struct completion *dma_completion, *bch_completion; 2514 unsigned long to; 2515 2516 if (check_only) 2517 return 0; 2518 2519 this->ntransfers = 0; 2520 for (i = 0; i < GPMI_MAX_TRANSFERS; i++) 2521 this->transfers[i].direction = DMA_NONE; 2522 2523 ret = pm_runtime_resume_and_get(this->dev); 2524 if (ret < 0) 2525 return ret; 2526 2527 /* 2528 * This driver currently supports only one NAND chip. Plus, dies share 2529 * the same configuration. So once timings have been applied on the 2530 * controller side, they will not change anymore. When the time will 2531 * come, the check on must_apply_timings will have to be dropped. 2532 */ 2533 if (this->hw.must_apply_timings) { 2534 this->hw.must_apply_timings = false; 2535 ret = gpmi_nfc_apply_timings(this); 2536 if (ret) 2537 goto out_pm; 2538 } 2539 2540 dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs); 2541 2542 for (i = 0; i < op->ninstrs; i++) { 2543 instr = &op->instrs[i]; 2544 2545 nand_op_trace(" ", instr); 2546 2547 switch (instr->type) { 2548 case NAND_OP_WAITRDY_INSTR: 2549 desc = gpmi_chain_wait_ready(this); 2550 break; 2551 case NAND_OP_CMD_INSTR: 2552 cmd = instr->ctx.cmd.opcode; 2553 2554 /* 2555 * When this command has an address cycle chain it 2556 * together with the address cycle 2557 */ 2558 if (i + 1 != op->ninstrs && 2559 op->instrs[i + 1].type == NAND_OP_ADDR_INSTR) 2560 continue; 2561 2562 desc = gpmi_chain_command(this, cmd, NULL, 0); 2563 2564 break; 2565 case NAND_OP_ADDR_INSTR: 2566 desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs, 2567 instr->ctx.addr.naddrs); 2568 break; 2569 case NAND_OP_DATA_OUT_INSTR: 2570 buf_write = instr->ctx.data.buf.out; 2571 buf_len = instr->ctx.data.len; 2572 nbufs++; 2573 2574 desc = gpmi_chain_data_write(this, buf_write, buf_len); 2575 2576 break; 2577 case NAND_OP_DATA_IN_INSTR: 2578 if (!instr->ctx.data.len) 2579 break; 2580 buf_read = instr->ctx.data.buf.in; 2581 buf_len = instr->ctx.data.len; 2582 nbufs++; 2583 2584 desc = gpmi_chain_data_read(this, buf_read, buf_len, 2585 &direct); 2586 break; 2587 } 2588 2589 if (!desc) { 2590 ret = -ENXIO; 2591 goto unmap; 2592 } 2593 } 2594 2595 dev_dbg(this->dev, "%s setup done\n", __func__); 2596 2597 if (nbufs > 1) { 2598 dev_err(this->dev, "Multiple data instructions not supported\n"); 2599 ret = -EINVAL; 2600 goto unmap; 2601 } 2602 2603 if (this->bch) { 2604 writel(this->bch_flashlayout0, 2605 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0); 2606 writel(this->bch_flashlayout1, 2607 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1); 2608 } 2609 2610 desc->callback = dma_irq_callback; 2611 desc->callback_param = this; 2612 dma_completion = &this->dma_done; 2613 bch_completion = NULL; 2614 2615 init_completion(dma_completion); 2616 2617 if (this->bch && buf_read) { 2618 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN, 2619 this->resources.bch_regs + HW_BCH_CTRL_SET); 2620 bch_completion = &this->bch_done; 2621 init_completion(bch_completion); 2622 } 2623 2624 dmaengine_submit(desc); 2625 dma_async_issue_pending(get_dma_chan(this)); 2626 2627 to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000)); 2628 if (!to) { 2629 dev_err(this->dev, "DMA timeout, last DMA\n"); 2630 gpmi_dump_info(this); 2631 ret = -ETIMEDOUT; 2632 goto unmap; 2633 } 2634 2635 if (this->bch && buf_read) { 2636 to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000)); 2637 if (!to) { 2638 dev_err(this->dev, "BCH timeout, last DMA\n"); 2639 gpmi_dump_info(this); 2640 ret = -ETIMEDOUT; 2641 goto unmap; 2642 } 2643 } 2644 2645 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN, 2646 this->resources.bch_regs + HW_BCH_CTRL_CLR); 2647 gpmi_clear_bch(this); 2648 2649 ret = 0; 2650 2651 unmap: 2652 for (i = 0; i < this->ntransfers; i++) { 2653 struct gpmi_transfer *transfer = &this->transfers[i]; 2654 2655 if (transfer->direction != DMA_NONE) 2656 dma_unmap_sg(this->dev, &transfer->sgl, 1, 2657 transfer->direction); 2658 } 2659 2660 if (!ret && buf_read && !direct) 2661 memcpy(buf_read, this->data_buffer_dma, 2662 gpmi_raw_len_to_len(this, buf_len)); 2663 2664 this->bch = false; 2665 2666 out_pm: 2667 pm_runtime_mark_last_busy(this->dev); 2668 pm_runtime_put_autosuspend(this->dev); 2669 2670 return ret; 2671 } 2672 2673 static const struct nand_controller_ops gpmi_nand_controller_ops = { 2674 .attach_chip = gpmi_nand_attach_chip, 2675 .setup_interface = gpmi_setup_interface, 2676 .exec_op = gpmi_nfc_exec_op, 2677 }; 2678 2679 static int gpmi_nand_init(struct gpmi_nand_data *this) 2680 { 2681 struct nand_chip *chip = &this->nand; 2682 struct mtd_info *mtd = nand_to_mtd(chip); 2683 int ret; 2684 2685 /* init the MTD data structures */ 2686 mtd->name = "gpmi-nand"; 2687 mtd->dev.parent = this->dev; 2688 2689 /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */ 2690 nand_set_controller_data(chip, this); 2691 nand_set_flash_node(chip, this->pdev->dev.of_node); 2692 chip->legacy.block_markbad = gpmi_block_markbad; 2693 chip->badblock_pattern = &gpmi_bbt_descr; 2694 chip->options |= NAND_NO_SUBPAGE_WRITE; 2695 2696 /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */ 2697 this->swap_block_mark = !GPMI_IS_MX23(this); 2698 2699 /* 2700 * Allocate a temporary DMA buffer for reading ID in the 2701 * nand_scan_ident(). 2702 */ 2703 this->bch_geometry.payload_size = 1024; 2704 this->bch_geometry.auxiliary_size = 128; 2705 ret = gpmi_alloc_dma_buffer(this); 2706 if (ret) 2707 return ret; 2708 2709 nand_controller_init(&this->base); 2710 this->base.ops = &gpmi_nand_controller_ops; 2711 chip->controller = &this->base; 2712 2713 ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1); 2714 if (ret) 2715 goto err_out; 2716 2717 ret = nand_boot_init(this); 2718 if (ret) 2719 goto err_nand_cleanup; 2720 ret = nand_create_bbt(chip); 2721 if (ret) 2722 goto err_nand_cleanup; 2723 2724 ret = mtd_device_register(mtd, NULL, 0); 2725 if (ret) 2726 goto err_nand_cleanup; 2727 return 0; 2728 2729 err_nand_cleanup: 2730 nand_cleanup(chip); 2731 err_out: 2732 gpmi_free_dma_buffer(this); 2733 return ret; 2734 } 2735 2736 static const struct of_device_id gpmi_nand_id_table[] = { 2737 { .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, }, 2738 { .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, }, 2739 { .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, }, 2740 { .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, }, 2741 { .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,}, 2742 { .compatible = "fsl,imx8qxp-gpmi-nand", .data = &gpmi_devdata_imx8qxp, }, 2743 {} 2744 }; 2745 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table); 2746 2747 static int gpmi_nand_probe(struct platform_device *pdev) 2748 { 2749 struct gpmi_nand_data *this; 2750 int ret; 2751 2752 this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL); 2753 if (!this) 2754 return -ENOMEM; 2755 2756 this->devdata = of_device_get_match_data(&pdev->dev); 2757 platform_set_drvdata(pdev, this); 2758 this->pdev = pdev; 2759 this->dev = &pdev->dev; 2760 2761 ret = acquire_resources(this); 2762 if (ret) 2763 goto exit_acquire_resources; 2764 2765 pm_runtime_enable(&pdev->dev); 2766 pm_runtime_set_autosuspend_delay(&pdev->dev, 500); 2767 pm_runtime_use_autosuspend(&pdev->dev); 2768 2769 ret = gpmi_init(this); 2770 if (ret) 2771 goto exit_nfc_init; 2772 2773 ret = gpmi_nand_init(this); 2774 if (ret) 2775 goto exit_nfc_init; 2776 2777 dev_info(this->dev, "driver registered.\n"); 2778 2779 return 0; 2780 2781 exit_nfc_init: 2782 pm_runtime_dont_use_autosuspend(&pdev->dev); 2783 pm_runtime_disable(&pdev->dev); 2784 release_resources(this); 2785 exit_acquire_resources: 2786 2787 return ret; 2788 } 2789 2790 static void gpmi_nand_remove(struct platform_device *pdev) 2791 { 2792 struct gpmi_nand_data *this = platform_get_drvdata(pdev); 2793 struct nand_chip *chip = &this->nand; 2794 int ret; 2795 2796 ret = mtd_device_unregister(nand_to_mtd(chip)); 2797 WARN_ON(ret); 2798 nand_cleanup(chip); 2799 gpmi_free_dma_buffer(this); 2800 release_resources(this); 2801 pm_runtime_dont_use_autosuspend(&pdev->dev); 2802 pm_runtime_disable(&pdev->dev); 2803 } 2804 2805 static int gpmi_pm_suspend(struct device *dev) 2806 { 2807 int ret; 2808 2809 pinctrl_pm_select_sleep_state(dev); 2810 ret = pm_runtime_force_suspend(dev); 2811 2812 return ret; 2813 } 2814 2815 static int gpmi_pm_resume(struct device *dev) 2816 { 2817 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2818 int ret; 2819 2820 ret = pm_runtime_force_resume(dev); 2821 if (ret) { 2822 dev_err(this->dev, "Error in resume %d\n", ret); 2823 return ret; 2824 } 2825 2826 pinctrl_pm_select_default_state(dev); 2827 2828 /* re-init the GPMI registers */ 2829 ret = gpmi_init(this); 2830 if (ret) { 2831 dev_err(this->dev, "Error setting GPMI : %d\n", ret); 2832 return ret; 2833 } 2834 2835 /* Set flag to get timing setup restored for next exec_op */ 2836 if (this->hw.clk_rate) 2837 this->hw.must_apply_timings = true; 2838 2839 /* re-init the BCH registers */ 2840 ret = bch_set_geometry(this); 2841 if (ret) { 2842 dev_err(this->dev, "Error setting BCH : %d\n", ret); 2843 return ret; 2844 } 2845 2846 return 0; 2847 } 2848 2849 #define gpmi_enable_clk(x) __gpmi_enable_clk(x, true) 2850 #define gpmi_disable_clk(x) __gpmi_enable_clk(x, false) 2851 2852 static int gpmi_runtime_suspend(struct device *dev) 2853 { 2854 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2855 2856 gpmi_disable_clk(this); 2857 2858 return 0; 2859 } 2860 2861 static int gpmi_runtime_resume(struct device *dev) 2862 { 2863 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2864 int ret; 2865 2866 ret = gpmi_enable_clk(this); 2867 if (ret) 2868 return ret; 2869 2870 return 0; 2871 2872 } 2873 2874 static const struct dev_pm_ops gpmi_pm_ops = { 2875 SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume) 2876 RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL) 2877 }; 2878 2879 static struct platform_driver gpmi_nand_driver = { 2880 .driver = { 2881 .name = "gpmi-nand", 2882 .pm = pm_ptr(&gpmi_pm_ops), 2883 .of_match_table = gpmi_nand_id_table, 2884 }, 2885 .probe = gpmi_nand_probe, 2886 .remove = gpmi_nand_remove, 2887 }; 2888 module_platform_driver(gpmi_nand_driver); 2889 2890 MODULE_AUTHOR("Freescale Semiconductor, Inc."); 2891 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver"); 2892 MODULE_LICENSE("GPL"); 2893