1 /* 2 * Copyright (C) 2017 Free Electrons 3 * Copyright (C) 2017 NextThing Co 4 * 5 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 */ 17 18 #include <linux/mtd/rawnand.h> 19 #include <linux/slab.h> 20 21 /* 22 * Special Micron status bit 3 indicates that the block has been 23 * corrected by on-die ECC and should be rewritten. 24 */ 25 #define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3) 26 27 /* 28 * On chips with 8-bit ECC and additional bit can be used to distinguish 29 * cases where a errors were corrected without needing a rewrite 30 * 31 * Bit 4 Bit 3 Bit 0 Description 32 * ----- ----- ----- ----------- 33 * 0 0 0 No Errors 34 * 0 0 1 Multiple uncorrected errors 35 * 0 1 0 4 - 6 errors corrected, recommend rewrite 36 * 0 1 1 Reserved 37 * 1 0 0 1 - 3 errors corrected 38 * 1 0 1 Reserved 39 * 1 1 0 7 - 8 errors corrected, recommend rewrite 40 */ 41 #define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0)) 42 #define NAND_ECC_STATUS_UNCORRECTABLE BIT(0) 43 #define NAND_ECC_STATUS_4_6_CORRECTED BIT(3) 44 #define NAND_ECC_STATUS_1_3_CORRECTED BIT(4) 45 #define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3)) 46 47 struct nand_onfi_vendor_micron { 48 u8 two_plane_read; 49 u8 read_cache; 50 u8 read_unique_id; 51 u8 dq_imped; 52 u8 dq_imped_num_settings; 53 u8 dq_imped_feat_addr; 54 u8 rb_pulldown_strength; 55 u8 rb_pulldown_strength_feat_addr; 56 u8 rb_pulldown_strength_num_settings; 57 u8 otp_mode; 58 u8 otp_page_start; 59 u8 otp_data_prot_addr; 60 u8 otp_num_pages; 61 u8 otp_feat_addr; 62 u8 read_retry_options; 63 u8 reserved[72]; 64 u8 param_revision; 65 } __packed; 66 67 struct micron_on_die_ecc { 68 bool forced; 69 bool enabled; 70 void *rawbuf; 71 }; 72 73 struct micron_nand { 74 struct micron_on_die_ecc ecc; 75 }; 76 77 static int micron_nand_setup_read_retry(struct mtd_info *mtd, int retry_mode) 78 { 79 struct nand_chip *chip = mtd_to_nand(mtd); 80 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; 81 82 return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); 83 } 84 85 /* 86 * Configure chip properties from Micron vendor-specific ONFI table 87 */ 88 static int micron_nand_onfi_init(struct nand_chip *chip) 89 { 90 struct nand_parameters *p = &chip->parameters; 91 92 if (p->onfi) { 93 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor; 94 95 chip->read_retries = micron->read_retry_options; 96 chip->setup_read_retry = micron_nand_setup_read_retry; 97 } 98 99 if (p->supports_set_get_features) { 100 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list); 101 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list); 102 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list); 103 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list); 104 } 105 106 return 0; 107 } 108 109 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd, 110 int section, 111 struct mtd_oob_region *oobregion) 112 { 113 if (section >= 4) 114 return -ERANGE; 115 116 oobregion->offset = (section * 16) + 8; 117 oobregion->length = 8; 118 119 return 0; 120 } 121 122 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd, 123 int section, 124 struct mtd_oob_region *oobregion) 125 { 126 if (section >= 4) 127 return -ERANGE; 128 129 oobregion->offset = (section * 16) + 2; 130 oobregion->length = 6; 131 132 return 0; 133 } 134 135 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = { 136 .ecc = micron_nand_on_die_4_ooblayout_ecc, 137 .free = micron_nand_on_die_4_ooblayout_free, 138 }; 139 140 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd, 141 int section, 142 struct mtd_oob_region *oobregion) 143 { 144 struct nand_chip *chip = mtd_to_nand(mtd); 145 146 if (section) 147 return -ERANGE; 148 149 oobregion->offset = mtd->oobsize - chip->ecc.total; 150 oobregion->length = chip->ecc.total; 151 152 return 0; 153 } 154 155 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd, 156 int section, 157 struct mtd_oob_region *oobregion) 158 { 159 struct nand_chip *chip = mtd_to_nand(mtd); 160 161 if (section) 162 return -ERANGE; 163 164 oobregion->offset = 2; 165 oobregion->length = mtd->oobsize - chip->ecc.total - 2; 166 167 return 0; 168 } 169 170 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = { 171 .ecc = micron_nand_on_die_8_ooblayout_ecc, 172 .free = micron_nand_on_die_8_ooblayout_free, 173 }; 174 175 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable) 176 { 177 struct micron_nand *micron = nand_get_manufacturer_data(chip); 178 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, }; 179 int ret; 180 181 if (micron->ecc.forced) 182 return 0; 183 184 if (micron->ecc.enabled == enable) 185 return 0; 186 187 if (enable) 188 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN; 189 190 ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature); 191 if (!ret) 192 micron->ecc.enabled = enable; 193 194 return ret; 195 } 196 197 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status, 198 void *buf, int page, 199 int oob_required) 200 { 201 struct micron_nand *micron = nand_get_manufacturer_data(chip); 202 struct mtd_info *mtd = nand_to_mtd(chip); 203 unsigned int step, max_bitflips = 0; 204 int ret; 205 206 if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) { 207 if (status & NAND_STATUS_FAIL) 208 mtd->ecc_stats.failed++; 209 210 return 0; 211 } 212 213 /* 214 * The internal ECC doesn't tell us the number of bitflips that have 215 * been corrected, but tells us if it recommends to rewrite the block. 216 * If it's the case, we need to read the page in raw mode and compare 217 * its content to the corrected version to extract the actual number of 218 * bitflips. 219 * But before we do that, we must make sure we have all OOB bytes read 220 * in non-raw mode, even if the user did not request those bytes. 221 */ 222 if (!oob_required) { 223 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, 224 false); 225 if (ret) 226 return ret; 227 } 228 229 micron_nand_on_die_ecc_setup(chip, false); 230 231 ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf, 232 mtd->writesize + mtd->oobsize); 233 if (ret) 234 return ret; 235 236 for (step = 0; step < chip->ecc.steps; step++) { 237 unsigned int offs, i, nbitflips = 0; 238 u8 *rawbuf, *corrbuf; 239 240 offs = step * chip->ecc.size; 241 rawbuf = micron->ecc.rawbuf + offs; 242 corrbuf = buf + offs; 243 244 for (i = 0; i < chip->ecc.size; i++) 245 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 246 247 offs = (step * 16) + 4; 248 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs; 249 corrbuf = chip->oob_poi + offs; 250 251 for (i = 0; i < chip->ecc.bytes + 4; i++) 252 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 253 254 if (WARN_ON(nbitflips > chip->ecc.strength)) 255 return -EINVAL; 256 257 max_bitflips = max(nbitflips, max_bitflips); 258 mtd->ecc_stats.corrected += nbitflips; 259 } 260 261 return max_bitflips; 262 } 263 264 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) 265 { 266 struct mtd_info *mtd = nand_to_mtd(chip); 267 268 /* 269 * With 8/512 we have more information but still don't know precisely 270 * how many bit-flips were seen. 271 */ 272 switch (status & NAND_ECC_STATUS_MASK) { 273 case NAND_ECC_STATUS_UNCORRECTABLE: 274 mtd->ecc_stats.failed++; 275 return 0; 276 case NAND_ECC_STATUS_1_3_CORRECTED: 277 mtd->ecc_stats.corrected += 3; 278 return 3; 279 case NAND_ECC_STATUS_4_6_CORRECTED: 280 mtd->ecc_stats.corrected += 6; 281 /* rewrite recommended */ 282 return 6; 283 case NAND_ECC_STATUS_7_8_CORRECTED: 284 mtd->ecc_stats.corrected += 8; 285 /* rewrite recommended */ 286 return 8; 287 default: 288 return 0; 289 } 290 } 291 292 static int 293 micron_nand_read_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip, 294 uint8_t *buf, int oob_required, 295 int page) 296 { 297 u8 status; 298 int ret, max_bitflips = 0; 299 300 ret = micron_nand_on_die_ecc_setup(chip, true); 301 if (ret) 302 return ret; 303 304 ret = nand_read_page_op(chip, page, 0, NULL, 0); 305 if (ret) 306 goto out; 307 308 ret = nand_status_op(chip, &status); 309 if (ret) 310 goto out; 311 312 ret = nand_exit_status_op(chip); 313 if (ret) 314 goto out; 315 316 ret = nand_read_data_op(chip, buf, mtd->writesize, false); 317 if (!ret && oob_required) 318 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, 319 false); 320 321 if (chip->ecc.strength == 4) 322 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status, 323 buf, page, 324 oob_required); 325 else 326 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status); 327 328 out: 329 micron_nand_on_die_ecc_setup(chip, false); 330 331 return ret ? ret : max_bitflips; 332 } 333 334 static int 335 micron_nand_write_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip, 336 const uint8_t *buf, int oob_required, 337 int page) 338 { 339 int ret; 340 341 ret = micron_nand_on_die_ecc_setup(chip, true); 342 if (ret) 343 return ret; 344 345 ret = nand_write_page_raw(mtd, chip, buf, oob_required, page); 346 micron_nand_on_die_ecc_setup(chip, false); 347 348 return ret; 349 } 350 351 enum { 352 /* The NAND flash doesn't support on-die ECC */ 353 MICRON_ON_DIE_UNSUPPORTED, 354 355 /* 356 * The NAND flash supports on-die ECC and it can be 357 * enabled/disabled by a set features command. 358 */ 359 MICRON_ON_DIE_SUPPORTED, 360 361 /* 362 * The NAND flash supports on-die ECC, and it cannot be 363 * disabled. 364 */ 365 MICRON_ON_DIE_MANDATORY, 366 }; 367 368 #define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0) 369 #define MICRON_ID_ECC_ENABLED BIT(7) 370 371 /* 372 * Try to detect if the NAND support on-die ECC. To do this, we enable 373 * the feature, and read back if it has been enabled as expected. We 374 * also check if it can be disabled, because some Micron NANDs do not 375 * allow disabling the on-die ECC and we don't support such NANDs for 376 * now. 377 * 378 * This function also has the side effect of disabling on-die ECC if 379 * it had been left enabled by the firmware/bootloader. 380 */ 381 static int micron_supports_on_die_ecc(struct nand_chip *chip) 382 { 383 u8 id[5]; 384 int ret; 385 386 if (!chip->parameters.onfi) 387 return MICRON_ON_DIE_UNSUPPORTED; 388 389 if (chip->bits_per_cell != 1) 390 return MICRON_ON_DIE_UNSUPPORTED; 391 392 /* 393 * We only support on-die ECC of 4/512 or 8/512 394 */ 395 if (chip->ecc_strength_ds != 4 && chip->ecc_strength_ds != 8) 396 return MICRON_ON_DIE_UNSUPPORTED; 397 398 /* 0x2 means on-die ECC is available. */ 399 if (chip->id.len != 5 || 400 (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2) 401 return MICRON_ON_DIE_UNSUPPORTED; 402 403 ret = micron_nand_on_die_ecc_setup(chip, true); 404 if (ret) 405 return MICRON_ON_DIE_UNSUPPORTED; 406 407 ret = nand_readid_op(chip, 0, id, sizeof(id)); 408 if (ret) 409 return MICRON_ON_DIE_UNSUPPORTED; 410 411 if (!(id[4] & MICRON_ID_ECC_ENABLED)) 412 return MICRON_ON_DIE_UNSUPPORTED; 413 414 ret = micron_nand_on_die_ecc_setup(chip, false); 415 if (ret) 416 return MICRON_ON_DIE_UNSUPPORTED; 417 418 ret = nand_readid_op(chip, 0, id, sizeof(id)); 419 if (ret) 420 return MICRON_ON_DIE_UNSUPPORTED; 421 422 if (id[4] & MICRON_ID_ECC_ENABLED) 423 return MICRON_ON_DIE_MANDATORY; 424 425 /* 426 * We only support on-die ECC of 4/512 or 8/512 427 */ 428 if (chip->ecc_strength_ds != 4 && chip->ecc_strength_ds != 8) 429 return MICRON_ON_DIE_UNSUPPORTED; 430 431 return MICRON_ON_DIE_SUPPORTED; 432 } 433 434 static int micron_nand_init(struct nand_chip *chip) 435 { 436 struct mtd_info *mtd = nand_to_mtd(chip); 437 struct micron_nand *micron; 438 int ondie; 439 int ret; 440 441 micron = kzalloc(sizeof(*micron), GFP_KERNEL); 442 if (!micron) 443 return -ENOMEM; 444 445 nand_set_manufacturer_data(chip, micron); 446 447 ret = micron_nand_onfi_init(chip); 448 if (ret) 449 goto err_free_manuf_data; 450 451 if (mtd->writesize == 2048) 452 chip->bbt_options |= NAND_BBT_SCAN2NDPAGE; 453 454 ondie = micron_supports_on_die_ecc(chip); 455 456 if (ondie == MICRON_ON_DIE_MANDATORY && 457 chip->ecc.mode != NAND_ECC_ON_DIE) { 458 pr_err("On-die ECC forcefully enabled, not supported\n"); 459 ret = -EINVAL; 460 goto err_free_manuf_data; 461 } 462 463 if (chip->ecc.mode == NAND_ECC_ON_DIE) { 464 if (ondie == MICRON_ON_DIE_UNSUPPORTED) { 465 pr_err("On-die ECC selected but not supported\n"); 466 ret = -EINVAL; 467 goto err_free_manuf_data; 468 } 469 470 if (ondie == MICRON_ON_DIE_MANDATORY) { 471 micron->ecc.forced = true; 472 micron->ecc.enabled = true; 473 } 474 475 /* 476 * In case of 4bit on-die ECC, we need a buffer to store a 477 * page dumped in raw mode so that we can compare its content 478 * to the same page after ECC correction happened and extract 479 * the real number of bitflips from this comparison. 480 * That's not needed for 8-bit ECC, because the status expose 481 * a better approximation of the number of bitflips in a page. 482 */ 483 if (chip->ecc_strength_ds == 4) { 484 micron->ecc.rawbuf = kmalloc(mtd->writesize + 485 mtd->oobsize, 486 GFP_KERNEL); 487 if (!micron->ecc.rawbuf) { 488 ret = -ENOMEM; 489 goto err_free_manuf_data; 490 } 491 } 492 493 if (chip->ecc_strength_ds == 4) 494 mtd_set_ooblayout(mtd, 495 µn_nand_on_die_4_ooblayout_ops); 496 else 497 mtd_set_ooblayout(mtd, 498 µn_nand_on_die_8_ooblayout_ops); 499 500 chip->ecc.bytes = chip->ecc_strength_ds * 2; 501 chip->ecc.size = 512; 502 chip->ecc.strength = chip->ecc_strength_ds; 503 chip->ecc.algo = NAND_ECC_BCH; 504 chip->ecc.read_page = micron_nand_read_page_on_die_ecc; 505 chip->ecc.write_page = micron_nand_write_page_on_die_ecc; 506 507 if (ondie == MICRON_ON_DIE_MANDATORY) { 508 chip->ecc.read_page_raw = nand_read_page_raw_notsupp; 509 chip->ecc.write_page_raw = nand_write_page_raw_notsupp; 510 } else { 511 chip->ecc.read_page_raw = nand_read_page_raw; 512 chip->ecc.write_page_raw = nand_write_page_raw; 513 } 514 } 515 516 return 0; 517 518 err_free_manuf_data: 519 kfree(micron->ecc.rawbuf); 520 kfree(micron); 521 522 return ret; 523 } 524 525 static void micron_nand_cleanup(struct nand_chip *chip) 526 { 527 struct micron_nand *micron = nand_get_manufacturer_data(chip); 528 529 kfree(micron->ecc.rawbuf); 530 kfree(micron); 531 } 532 533 static void micron_fixup_onfi_param_page(struct nand_chip *chip, 534 struct nand_onfi_params *p) 535 { 536 /* 537 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the 538 * revision number field of the ONFI parameter page. Assume ONFI 539 * version 1.0 if the revision number is 00 00. 540 */ 541 if (le16_to_cpu(p->revision) == 0) 542 p->revision = cpu_to_le16(ONFI_VERSION_1_0); 543 } 544 545 const struct nand_manufacturer_ops micron_nand_manuf_ops = { 546 .init = micron_nand_init, 547 .cleanup = micron_nand_cleanup, 548 .fixup_onfi_param_page = micron_fixup_onfi_param_page, 549 }; 550