Lines Matching +full:nand +full:- +full:ecc +full:- +full:algo
1 // SPDX-License-Identifier: GPL-2.0-only
4 * This is the generic MTD driver for NAND flash devices. It should be
5 * capable of working with almost all NAND chips currently available.
8 * http://www.linux-mtd.infradead.org/doc/nand.html
11 * 2002-2006 Thomas Gleixner (tglx@linutronix.de)
21 * Check, if mtd->ecctype should be set to MTD_ECC_HW
22 * if we have HW ECC support.
37 #include <linux/mtd/nand.h>
38 #include <linux/mtd/nand-ecc-sw-hamming.h>
39 #include <linux/mtd/nand-ecc-sw-bch.h>
52 int lastpage = (mtd->erasesize / mtd->writesize) - 1;
59 info->group = 0;
60 info->pair = (page + 1) / 2;
62 info->group = 1;
63 info->pair = (page + 1 - dist) / 2;
72 int lastpair = ((mtd->erasesize / mtd->writesize) - 1) / 2;
73 int page = info->pair * 2;
76 if (!info->group && !info->pair)
79 if (info->pair == lastpair && info->group)
82 if (!info->group)
83 page--;
84 else if (info->pair)
85 page += dist - 1;
87 if (page >= mtd->erasesize / mtd->writesize)
88 return -EINVAL;
104 if (ofs & ((1ULL << chip->phys_erase_shift) - 1)) {
106 ret = -EINVAL;
110 if (len & ((1ULL << chip->phys_erase_shift) - 1)) {
112 ret = -EINVAL;
119 * nand_extract_bits - Copy unaligned bits from one buffer to another one
139 n = min3(8 - dst_off, 8 - src_off, nbits);
141 tmp = (*src >> src_off) & GENMASK(n - 1, 0);
142 *dst &= ~GENMASK(n - 1 + dst_off, dst_off);
148 dst_off -= 8;
154 src_off -= 8;
157 nbits -= n;
163 * nand_select_target() - Select a NAND target (A.K.A. die)
164 * @chip: NAND chip object
168 * Select a NAND target so that further operations executed on @chip go to the
169 * selected NAND target.
177 if (WARN_ON(cs > nanddev_ntargets(&chip->base)))
180 chip->cur_cs = cs;
182 if (chip->legacy.select_chip)
183 chip->legacy.select_chip(chip, cs);
188 * nand_deselect_target() - Deselect the currently selected target
189 * @chip: NAND chip object
191 * Deselect the currently selected NAND target. The result of operations
196 if (chip->legacy.select_chip)
197 chip->legacy.select_chip(chip, -1);
199 chip->cur_cs = -1;
204 * nand_release_device - [GENERIC] release chip
205 * @chip: NAND chip object
212 mutex_unlock(&chip->controller->lock);
213 mutex_unlock(&chip->lock);
217 * nand_bbm_get_next_page - Get the next page for bad block markers
218 * @chip: NAND chip object
223 * available, -EINVAL is returned.
228 int last_page = ((mtd->erasesize - mtd->writesize) >>
229 chip->page_shift) & chip->pagemask;
233 if (page == 0 && !(chip->options & bbm_flags))
235 if (page == 0 && chip->options & NAND_BBM_FIRSTPAGE)
237 if (page <= 1 && chip->options & NAND_BBM_SECONDPAGE)
239 if (page <= last_page && chip->options & NAND_BBM_LASTPAGE)
242 return -EINVAL;
246 * nand_block_bad - [DEFAULT] Read bad block marker from the chip
247 * @chip: NAND chip object
258 first_page = (int)(ofs >> chip->page_shift) & chip->pagemask;
262 res = chip->ecc.read_oob(chip, first_page + page_offset);
266 bad = chip->oob_poi[chip->badblockpos];
268 if (likely(chip->badblockbits == 8))
271 res = hweight8(bad) < chip->badblockbits;
282 * nand_region_is_secured() - Check if the region is secured
283 * @chip: NAND chip object
296 for (i = 0; i < chip->nr_secure_regions; i++) {
297 const struct nand_secure_region *region = &chip->secure_regions[i];
299 if (offset + size <= region->offset ||
300 offset >= region->offset + region->size)
303 pr_debug("%s: Region 0x%llx - 0x%llx is secured!",
316 if (chip->options & NAND_NO_BBM_QUIRK)
320 if (nand_region_is_secured(chip, ofs, mtd->erasesize))
321 return -EIO;
326 if (chip->legacy.block_bad)
327 return chip->legacy.block_bad(chip, ofs);
333 * nand_get_device - [GENERIC] Get chip for selected access
334 * @chip: NAND chip structure
342 mutex_lock(&chip->lock);
343 if (!chip->suspended) {
344 mutex_lock(&chip->controller->lock);
347 mutex_unlock(&chip->lock);
349 wait_event(chip->resume_wq, !chip->suspended);
354 * nand_check_wp - [GENERIC] check if the chip is write protected
355 * @chip: NAND chip object
366 if (chip->options & NAND_BROKEN_XD)
369 /* controller responsible for NAND write protect */
370 if (chip->controller->controller_wp)
382 * nand_fill_oob - [INTERN] Transfer client buffer to oob
383 * @chip: NAND chip object
398 memset(chip->oob_poi, 0xff, mtd->oobsize);
400 switch (ops->mode) {
404 memcpy(chip->oob_poi + ops->ooboffs, oob, len);
408 ret = mtd_ooblayout_set_databytes(mtd, oob, chip->oob_poi,
409 ops->ooboffs, len);
420 * nand_do_write_oob - [MTD Interface] NAND write out-of-band
421 * @chip: NAND chip object
425 * NAND write out-of-band.
434 __func__, (unsigned int)to, (int)ops->ooblen);
439 if ((ops->ooboffs + ops->ooblen) > len) {
442 return -EINVAL;
446 if (nand_region_is_secured(chip, to, ops->ooblen))
447 return -EIO;
449 chipnr = (int)(to >> chip->chip_shift);
464 page = (int)(to >> chip->page_shift);
469 return -EROFS;
473 if (page == chip->pagecache.page)
474 chip->pagecache.page = -1;
476 nand_fill_oob(chip, ops->oobbuf, ops->ooblen, ops);
478 if (ops->mode == MTD_OPS_RAW)
479 status = chip->ecc.write_oob_raw(chip, page & chip->pagemask);
481 status = chip->ecc.write_oob(chip, page & chip->pagemask);
488 ops->oobretlen = ops->ooblen;
494 * nand_default_block_markbad - [DEFAULT] mark a block bad via bad block marker
495 * @chip: NAND chip object
511 ops.ooboffs = chip->badblockpos;
512 if (chip->options & NAND_BUSWIDTH_16) {
524 ofs + (page_offset * mtd->writesize),
537 * nand_markbad_bbm - mark a block by updating the BBM
538 * @chip: NAND chip object
543 if (chip->legacy.block_markbad)
544 return chip->legacy.block_markbad(chip, ofs);
550 * nand_block_markbad_lowlevel - mark a block bad
551 * @chip: NAND chip object
554 * This function performs the generic NAND bad block marking steps (i.e., bad
556 * specify how to write bad block markers to OOB (chip->legacy.block_markbad).
573 if (!(chip->bbt_options & NAND_BBT_NO_OOB_BBM)) {
579 einfo.len = 1ULL << chip->phys_erase_shift;
590 if (chip->bbt) {
597 mtd->ecc_stats.badblocks++;
603 * nand_block_isreserved - [GENERIC] Check if a block is marked reserved.
613 if (!chip->bbt)
620 * nand_block_checkbad - [GENERIC] Check if a block is marked bad
621 * @chip: NAND chip object
631 if (chip->bbt)
638 * nand_soft_waitrdy - Poll STATUS reg until RDY bit is set to 1
639 * @chip: NAND chip structure
642 * Poll the STATUS register using ->exec_op() until the RDY bit becomes 1.
643 * If that does not happen whitin the specified timeout, -ETIMEDOUT is
647 * to the NAND R/B pin.
649 * Be aware that calling this helper from an ->exec_op() implementation means
650 * ->exec_op() must be re-entrant.
652 * Return 0 if the NAND chip is ready, a negative error otherwise.
661 return -ENOTSUPP;
674 * small jiffy fraction - possibly leading to false timeout
704 return status & NAND_STATUS_READY ? 0 : -ETIMEDOUT;
709 * nand_gpio_waitrdy - Poll R/B GPIO pin until ready
710 * @chip: NAND chip structure
715 * whitin the specified timeout, -ETIMEDOUT is returned.
718 * NAND R/B pin over GPIO.
730 * small jiffy fraction - possibly leading to false timeout.
740 return gpiod_get_value_cansleep(gpiod) ? 0 : -ETIMEDOUT;
745 * panic_nand_wait - [GENERIC] wait until the command is done
746 * @chip: NAND chip structure
757 if (chip->legacy.dev_ready) {
758 if (chip->legacy.dev_ready(chip))
778 return (chip->parameters.supports_set_get_features &&
779 test_bit(addr, chip->parameters.get_feature_list));
784 return (chip->parameters.supports_set_get_features &&
785 test_bit(addr, chip->parameters.set_feature_list));
789 * nand_reset_interface - Reset data interface and timings
790 * @chip: The NAND chip
799 const struct nand_controller_ops *ops = chip->controller->ops;
808 * To transition from NV-DDR or NV-DDR2 to the SDR data
819 chip->current_interface_config = nand_get_reset_interface_config();
820 ret = ops->setup_interface(chip, chipnr,
821 chip->current_interface_config);
829 * nand_setup_interface - Setup the best data interface and timings
830 * @chip: The NAND chip
833 * Configure what has been reported to be the best data interface and NAND
840 const struct nand_controller_ops *ops = chip->controller->ops;
848 * A nand_reset_interface() put both the NAND chip and the NAND
851 * nand_setup_interface() uses ->set/get_features() which would
854 if (!chip->best_interface_config)
857 request = chip->best_interface_config->timings.mode;
858 if (nand_interface_is_sdr(chip->best_interface_config))
864 /* Change the mode on the chip side (if supported by the NAND chip) */
875 ret = ops->setup_interface(chip, chipnr, chip->best_interface_config);
892 pr_warn("%s timing mode %d not acknowledged by the NAND chip\n",
893 nand_interface_is_nvddr(chip->best_interface_config) ? "NV-DDR" : "SDR",
894 chip->best_interface_config->timings.mode);
895 pr_debug("NAND chip would work in %s timing mode %d\n",
896 tmode_param[0] & ONFI_DATA_INTERFACE_NVDDR ? "NV-DDR" : "SDR",
902 chip->current_interface_config = chip->best_interface_config;
920 * nand_choose_best_sdr_timings - Pick up the best SDR timings that both the
921 * NAND controller and the NAND chip support
922 * @chip: the NAND chip
933 const struct nand_controller_ops *ops = chip->controller->ops;
934 int best_mode = 0, mode, ret = -EOPNOTSUPP;
936 iface->type = NAND_SDR_IFACE;
939 iface->timings.sdr = *spec_timings;
940 iface->timings.mode = onfi_find_closest_sdr_mode(spec_timings);
943 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
946 chip->best_interface_config = iface;
951 best_mode = iface->timings.mode;
952 } else if (chip->parameters.onfi) {
953 best_mode = fls(chip->parameters.onfi->sdr_timing_modes) - 1;
956 for (mode = best_mode; mode >= 0; mode--) {
959 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
962 chip->best_interface_config = iface;
971 * nand_choose_best_nvddr_timings - Pick up the best NVDDR timings that both the
972 * NAND controller and the NAND chip support
973 * @chip: the NAND chip
984 const struct nand_controller_ops *ops = chip->controller->ops;
985 int best_mode = 0, mode, ret = -EOPNOTSUPP;
987 iface->type = NAND_NVDDR_IFACE;
990 iface->timings.nvddr = *spec_timings;
991 iface->timings.mode = onfi_find_closest_nvddr_mode(spec_timings);
994 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
997 chip->best_interface_config = iface;
1002 best_mode = iface->timings.mode;
1003 } else if (chip->parameters.onfi) {
1004 best_mode = fls(chip->parameters.onfi->nvddr_timing_modes) - 1;
1007 for (mode = best_mode; mode >= 0; mode--) {
1010 ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY,
1013 chip->best_interface_config = iface;
1022 * nand_choose_best_timings - Pick up the best NVDDR or SDR timings that both
1023 * NAND controller and the NAND chip support
1024 * @chip: the NAND chip
1035 /* Try the fastest timings: NV-DDR */
1045 * nand_choose_interface_config - find the best data interface and timings
1046 * @chip: The NAND chip
1048 * Find the best data interface and NAND timings supported by the chip
1049 * and the driver. Eventually let the NAND manufacturer driver propose his own
1052 * After this function nand_chip->interface_config is initialized with the best
1067 return -ENOMEM;
1069 if (chip->ops.choose_interface_config)
1070 ret = chip->ops.choose_interface_config(chip, iface);
1081 * nand_fill_column_cycles - fill the column cycles of an address
1082 * @chip: The NAND chip
1087 * on the NAND bus width and the page size.
1096 bool ident_stage = !mtd->writesize;
1098 /* Bypass all checks during NAND identification */
1101 if (offset_in_page > mtd->writesize + mtd->oobsize)
1102 return -EINVAL;
1109 if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize)
1110 offset_in_page -= mtd->writesize;
1113 * The offset in page is expressed in bytes, if the NAND bus is 16-bit
1116 if (chip->options & NAND_BUSWIDTH_16) {
1118 return -EINVAL;
1130 if (!ident_stage && mtd->writesize <= 512)
1153 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1158 op.ninstrs--;
1160 if (offset_in_page >= mtd->writesize)
1163 !(chip->options & NAND_BUSWIDTH_16))
1173 if (chip->options & NAND_ROW_ADDR_3) {
1196 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1201 op.ninstrs--;
1210 if (chip->options & NAND_ROW_ADDR_3) {
1221 return (lun * pages_per_lun) + pages_per_lun - 1;
1229 memorg = nanddev_get_memorg(&chip->base);
1230 ppl = memorg->pages_per_eraseblock * memorg->eraseblocks_per_lun;
1231 first_lun = chip->cont_read.first_page / ppl;
1232 last_lun = chip->cont_read.last_page / ppl;
1236 chip->cont_read.pause_page = rawnand_last_page_of_lun(ppl, first_lun);
1238 chip->cont_read.pause_page = chip->cont_read.last_page;
1240 if (chip->cont_read.first_page == chip->cont_read.pause_page) {
1241 chip->cont_read.first_page++;
1242 chip->cont_read.pause_page = min(chip->cont_read.last_page,
1246 if (chip->cont_read.first_page >= chip->cont_read.last_page)
1247 chip->cont_read.ongoing = false;
1268 NAND_OP_CMD(page == chip->cont_read.pause_page ?
1275 struct nand_operation start_op = NAND_OPERATION(chip->cur_cs, start_instrs);
1276 struct nand_operation cont_op = NAND_OPERATION(chip->cur_cs, cont_instrs);
1280 start_op.ninstrs--;
1281 cont_op.ninstrs--;
1291 if (chip->options & NAND_ROW_ADDR_3) {
1299 return -EOPNOTSUPP;
1304 if (page == chip->cont_read.first_page)
1311 if (!chip->cont_read.ongoing)
1314 if (page == chip->cont_read.last_page) {
1315 chip->cont_read.ongoing = false;
1316 } else if (page == chip->cont_read.pause_page) {
1317 chip->cont_read.first_page++;
1326 return chip->cont_read.ongoing && page >= chip->cont_read.first_page;
1330 * nand_read_page_op - Do a READ PAGE operation
1331 * @chip: The NAND chip
1348 return -EINVAL;
1350 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1351 return -EINVAL;
1354 if (mtd->writesize > 512) {
1369 chip->legacy.cmdfunc(chip, NAND_CMD_READ0, offset_in_page, page);
1371 chip->legacy.read_buf(chip, buf, len);
1378 * nand_read_param_page_op - Do a READ PARAMETER PAGE operation
1379 * @chip: The NAND chip
1396 return -EINVAL;
1409 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1413 op.ninstrs--;
1418 chip->legacy.cmdfunc(chip, NAND_CMD_PARAM, page, -1);
1420 p[i] = chip->legacy.read_byte(chip);
1426 * nand_change_read_column_op - Do a CHANGE READ COLUMN operation
1427 * @chip: The NAND chip
1431 * @force_8bit: force 8-bit bus access
1443 bool ident_stage = !mtd->writesize;
1446 return -EINVAL;
1449 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1450 return -EINVAL;
1453 if (mtd->writesize <= 512)
1454 return -ENOTSUPP;
1468 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1477 op.ninstrs--;
1484 chip->legacy.cmdfunc(chip, NAND_CMD_RNDOUT, offset_in_page, -1);
1486 chip->legacy.read_buf(chip, buf, len);
1493 * nand_read_oob_op - Do a READ OOB operation
1494 * @chip: The NAND chip
1511 return -EINVAL;
1513 if (offset_in_oob + len > mtd->oobsize)
1514 return -EINVAL;
1518 mtd->writesize + offset_in_oob,
1521 chip->legacy.cmdfunc(chip, NAND_CMD_READOOB, offset_in_oob, page);
1523 chip->legacy.read_buf(chip, buf, len);
1540 * with a large page NAND and adjusted if we're dealing
1541 * with a small page NAND and the page offset is > 255.
1551 struct nand_operation op = NAND_DESTRUCTIVE_OPERATION(chip->cur_cs,
1560 if (chip->options & NAND_ROW_ADDR_3)
1567 op.ninstrs -= 2;
1570 op.ninstrs--;
1573 if (mtd->writesize <= 512) {
1579 if (offset_in_page >= mtd->writesize)
1582 !(chip->options & NAND_BUSWIDTH_16))
1587 * NAND.
1590 op.ninstrs--;
1597 * nand_prog_page_begin_op - starts a PROG PAGE operation
1598 * @chip: The NAND chip
1616 return -EINVAL;
1618 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1619 return -EINVAL;
1625 chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, page);
1628 chip->legacy.write_buf(chip, buf, len);
1635 * nand_prog_page_end_op - ends a PROG PAGE operation
1636 * @chip: The NAND chip
1657 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1667 chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1668 ret = chip->legacy.waitfunc(chip);
1676 return -EIO;
1683 * nand_prog_page_op - Do a full PROG PAGE operation
1684 * @chip: The NAND chip
1704 return -EINVAL;
1706 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1707 return -EINVAL;
1719 chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page,
1721 chip->legacy.write_buf(chip, buf, len);
1722 chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1);
1723 ret = chip->legacy.waitfunc(chip);
1731 return -EIO;
1738 * nand_change_write_column_op - Do a CHANGE WRITE COLUMN operation
1739 * @chip: The NAND chip
1741 * @buf: buffer containing the data to send to the NAND
1743 * @force_8bit: force 8-bit bus access
1758 return -EINVAL;
1760 if (offset_in_page + len > mtd->writesize + mtd->oobsize)
1761 return -EINVAL;
1764 if (mtd->writesize <= 512)
1765 return -ENOTSUPP;
1776 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1787 op.ninstrs--;
1792 chip->legacy.cmdfunc(chip, NAND_CMD_RNDIN, offset_in_page, -1);
1794 chip->legacy.write_buf(chip, buf, len);
1801 * nand_readid_op - Do a READID operation
1802 * @chip: The NAND chip
1808 * NAND.
1820 return -EINVAL;
1831 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1834 /* READ_ID data bytes are received twice in NV-DDR mode */
1838 return -ENOMEM;
1846 op.ninstrs--;
1859 chip->legacy.cmdfunc(chip, NAND_CMD_READID, addr, -1);
1862 id[i] = chip->legacy.read_byte(chip);
1869 * nand_status_op - Do a STATUS operation
1870 * @chip: The NAND chip
1871 * @status: out variable to store the NAND status
1874 * the NAND.
1890 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1893 /* The status data byte will be received twice in NV-DDR mode */
1900 op.ninstrs--;
1909 chip->legacy.cmdfunc(chip, NAND_CMD_STATUS, -1, -1);
1911 *status = chip->legacy.read_byte(chip);
1918 * nand_exit_status_op - Exit a STATUS operation
1919 * @chip: The NAND chip
1934 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
1939 chip->legacy.cmdfunc(chip, NAND_CMD_READ0, -1, -1);
1946 * nand_erase_op - Do an erase operation
1947 * @chip: The NAND chip
1950 * This function sends an ERASE command and waits for the NAND to be ready
1959 (chip->phys_erase_shift - chip->page_shift);
1975 struct nand_operation op = NAND_DESTRUCTIVE_OPERATION(chip->cur_cs,
1978 if (chip->options & NAND_ROW_ADDR_3)
1989 chip->legacy.cmdfunc(chip, NAND_CMD_ERASE1, -1, page);
1990 chip->legacy.cmdfunc(chip, NAND_CMD_ERASE2, -1, -1);
1992 ret = chip->legacy.waitfunc(chip);
2000 return -EIO;
2007 * nand_set_features_op - Do a SET FEATURES operation
2008 * @chip: The NAND chip
2012 * This function sends a SET FEATURES command and waits for the NAND to be
2037 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2042 chip->legacy.cmdfunc(chip, NAND_CMD_SET_FEATURES, feature, -1);
2044 chip->legacy.write_byte(chip, params[i]);
2046 ret = chip->legacy.waitfunc(chip);
2051 return -EIO;
2057 * nand_get_features_op - Do a GET FEATURES operation
2058 * @chip: The NAND chip
2062 * This function sends a GET FEATURES command and waits for the NAND to be
2086 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2089 /* GET_FEATURE data bytes are received twice in NV-DDR mode */
2104 chip->legacy.cmdfunc(chip, NAND_CMD_GET_FEATURES, feature, -1);
2106 params[i] = chip->legacy.read_byte(chip);
2119 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2125 if (!chip->legacy.dev_ready)
2126 udelay(chip->legacy.chip_delay);
2134 * nand_reset_op - Do a reset operation
2135 * @chip: The NAND chip
2137 * This function sends a RESET command and waits for the NAND to be ready
2154 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2159 chip->legacy.cmdfunc(chip, NAND_CMD_RESET, -1, -1);
2166 * nand_read_data_op - Read data from the NAND
2167 * @chip: The NAND chip
2170 * @force_8bit: force 8-bit bus access
2175 * another NAND operation like nand_read_page_op().
2184 return -EINVAL;
2192 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2202 * case NV-DDR timings are being used the data will be received
2208 return -ENOMEM;
2241 p[i] = chip->legacy.read_byte(chip);
2243 chip->legacy.read_buf(chip, buf, len);
2251 * nand_write_data_op - Write data from the NAND
2252 * @chip: The NAND chip
2255 * @force_8bit: force 8-bit bus access
2258 * another NAND operation like nand_write_page_begin_op().
2267 return -EINVAL;
2273 struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs);
2285 chip->legacy.write_byte(chip, p[i]);
2287 chip->legacy.write_buf(chip, buf, len);
2295 * struct nand_op_parser_ctx - Context used by the parser
2298 * @subop: Sub-operation to be passed to the NAND controller
2300 * This structure is used by the core to split NAND operations into
2301 * sub-operations that can be handled by the NAND controller.
2310 * nand_op_parser_must_split_instr - Checks if an instruction must be split
2320 * Some NAND controllers are limited and cannot send X address cycles with a
2323 * controller-operation into two or more chunks.
2334 switch (pat->type) {
2336 if (!pat->ctx.addr.maxcycles)
2339 if (instr->ctx.addr.naddrs - *start_offset >
2340 pat->ctx.addr.maxcycles) {
2341 *start_offset += pat->ctx.addr.maxcycles;
2348 if (!pat->ctx.data.maxlen)
2351 if (instr->ctx.data.len - *start_offset >
2352 pat->ctx.data.maxlen) {
2353 *start_offset += pat->ctx.data.maxlen;
2366 * nand_op_parser_match_pat - Checks if a pattern matches the instructions
2371 * Check if @pat matches the set or a sub-set of instructions remaining in @ctx.
2373 * @ctx->subop is updated with the set of instructions to be passed to the
2380 unsigned int instr_offset = ctx->subop.first_instr_start_off;
2381 const struct nand_op_instr *end = ctx->instrs + ctx->ninstrs;
2382 const struct nand_op_instr *instr = ctx->subop.instrs;
2385 for (i = 0, ninstrs = 0; i < pat->nelems && instr < end; i++) {
2393 if (instr->type != pat->elems[i].type) {
2394 if (!pat->elems[i].optional)
2408 if (nand_op_parser_must_split_instr(&pat->elems[i], instr,
2434 for (; i < pat->nelems; i++) {
2435 if (!pat->elems[i].optional)
2443 ctx->subop.ninstrs = ninstrs;
2444 ctx->subop.last_instr_end_off = instr_offset;
2456 pr_debug("executing subop (CS%d):\n", ctx->subop.cs);
2458 for (i = 0; i < ctx->ninstrs; i++) {
2459 instr = &ctx->instrs[i];
2461 if (instr == &ctx->subop.instrs[0])
2462 prefix = " ->";
2466 if (instr == &ctx->subop.instrs[ctx->subop.ninstrs - 1])
2480 if (a->subop.ninstrs < b->subop.ninstrs)
2481 return -1;
2482 else if (a->subop.ninstrs > b->subop.ninstrs)
2485 if (a->subop.last_instr_end_off < b->subop.last_instr_end_off)
2486 return -1;
2487 else if (a->subop.last_instr_end_off > b->subop.last_instr_end_off)
2494 * nand_op_parser_exec_op - exec_op parser
2495 * @chip: the NAND chip
2497 * @op: the NAND operation to address
2501 * Helper function designed to ease integration of NAND controller drivers that
2504 * multiple sub-operations (if required) and pass them back to the ->exec()
2507 * NAND controller drivers should call this function from their own ->exec_op()
2513 * matching pattern->exec() hook.
2520 .subop.cs = op->cs,
2521 .subop.instrs = op->instrs,
2522 .instrs = op->instrs,
2523 .ninstrs = op->ninstrs,
2527 while (ctx.subop.instrs < op->instrs + op->ninstrs) {
2530 int ret, best_pattern = -1;
2532 for (i = 0; i < parser->npatterns; i++) {
2535 pattern = &parser->patterns[i];
2548 pr_debug("->exec_op() parser: pattern not found!\n");
2549 return -ENOTSUPP;
2556 pattern = &parser->patterns[best_pattern];
2557 ret = pattern->exec(chip, &ctx.subop);
2568 ctx.subop.instrs -= 1;
2579 return instr && (instr->type == NAND_OP_DATA_IN_INSTR ||
2580 instr->type == NAND_OP_DATA_OUT_INSTR);
2586 return subop && instr_idx < subop->ninstrs;
2595 return subop->first_instr_start_off;
2599 * nand_subop_get_addr_start_off - Get the start offset in an address array
2600 * @subop: The entire sub-operation
2601 * @instr_idx: Index of the instruction inside the sub-operation
2604 * ->addr.addrs field of address instructions. This is wrong as address
2613 subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2621 * nand_subop_get_num_addr_cyc - Get the remaining address cycles to assert
2622 * @subop: The entire sub-operation
2623 * @instr_idx: Index of the instruction inside the sub-operation
2626 * ->addr->naddrs field of a data instruction. This is wrong as instructions
2637 subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR))
2642 if (instr_idx == subop->ninstrs - 1 &&
2643 subop->last_instr_end_off)
2644 end_off = subop->last_instr_end_off;
2646 end_off = subop->instrs[instr_idx].ctx.addr.naddrs;
2648 return end_off - start_off;
2653 * nand_subop_get_data_start_off - Get the start offset in a data array
2654 * @subop: The entire sub-operation
2655 * @instr_idx: Index of the instruction inside the sub-operation
2658 * ->data->buf.{in,out} field of data instructions. This is wrong as data
2667 !nand_instr_is_data(&subop->instrs[instr_idx])))
2675 * nand_subop_get_data_len - Get the number of bytes to retrieve
2676 * @subop: The entire sub-operation
2677 * @instr_idx: Index of the instruction inside the sub-operation
2680 * ->data->len field of a data instruction. This is wrong as data instructions
2691 !nand_instr_is_data(&subop->instrs[instr_idx])))
2696 if (instr_idx == subop->ninstrs - 1 &&
2697 subop->last_instr_end_off)
2698 end_off = subop->last_instr_end_off;
2700 end_off = subop->instrs[instr_idx].ctx.data.len;
2702 return end_off - start_off;
2707 * nand_reset - Reset and initialize a NAND device
2708 * @chip: The NAND chip
2726 * The CS line has to be released before we can apply the new NAND
2745 * nand_get_features - wrapper to perform a GET_FEATURE
2746 * @chip: NAND chip info structure
2750 * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2757 return -ENOTSUPP;
2759 if (chip->legacy.get_features)
2760 return chip->legacy.get_features(chip, addr, subfeature_param);
2766 * nand_set_features - wrapper to perform a SET_FEATURE
2767 * @chip: NAND chip info structure
2771 * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the
2778 return -ENOTSUPP;
2780 if (chip->legacy.set_features)
2781 return chip->legacy.set_features(chip, addr, subfeature_param);
2787 * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
2802 * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2812 len--, bitmap++) {
2814 bitflips += BITS_PER_BYTE - weight;
2816 return -EBADMSG;
2820 len -= sizeof(long), bitmap += sizeof(long)) {
2825 bitflips += BITS_PER_LONG - weight;
2827 return -EBADMSG;
2830 for (; len > 0; len--, bitmap++) {
2832 bitflips += BITS_PER_BYTE - weight;
2834 return -EBADMSG;
2841 * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only
2845 * @ecc: ECC buffer
2846 * @ecclen: ECC length
2851 * Check if a data buffer and its associated ECC and OOB data contains only
2858 * 1/ ECC algorithms are working on pre-defined block sizes which are usually
2859 * different from the NAND page size. When fixing bitflips, ECC engines will
2860 * report the number of errors per chunk, and the NAND core infrastructure
2866 * the payload data but also their associated ECC data, because a user might
2868 * shouldn't consider the chunk as erased, and checking ECC bytes prevent
2871 * data are protected by the ECC engine.
2873 * extra OOB data to an ECC chunk.
2876 * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
2880 void *ecc, int ecclen,
2891 bitflips_threshold -= data_bitflips;
2893 ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold);
2897 bitflips_threshold -= ecc_bitflips;
2908 memset(ecc, 0xff, ecclen);
2918 * nand_read_page_raw_notsupp - dummy read raw page function
2919 * @chip: nand chip info structure
2921 * @oob_required: caller requires OOB data read to chip->oob_poi
2924 * Returns -ENOTSUPP unconditionally.
2929 return -ENOTSUPP;
2933 * nand_read_page_raw - [INTERN] read raw page data without ecc
2934 * @chip: nand chip info structure
2936 * @oob_required: caller requires OOB data read to chip->oob_poi
2939 * Not for syndrome calculating ECC controllers, which use a special oob layout.
2947 ret = nand_read_page_op(chip, page, 0, buf, mtd->writesize);
2952 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
2963 * nand_monolithic_read_page_raw - Monolithic page read in raw mode
2964 * @chip: NAND chip info structure
2966 * @oob_required: caller requires OOB data read to chip->oob_poi
2971 * eventually OOB) to be loaded in the NAND cache and sent over the
2972 * bus (from the NAND chip to the NAND controller) in a single
2981 unsigned int size = mtd->writesize;
2986 size += mtd->oobsize;
2988 if (buf != chip->data_buf)
2996 if (buf != chip->data_buf)
2997 memcpy(buf, read_buf, mtd->writesize);
3004 * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
3005 * @chip: nand chip info structure
3007 * @oob_required: caller requires OOB data read to chip->oob_poi
3016 int eccsize = chip->ecc.size;
3017 int eccbytes = chip->ecc.bytes;
3018 uint8_t *oob = chip->oob_poi;
3025 for (steps = chip->ecc.steps; steps > 0; steps--) {
3032 if (chip->ecc.prepad) {
3033 ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3038 oob += chip->ecc.prepad;
3047 if (chip->ecc.postpad) {
3048 ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3053 oob += chip->ecc.postpad;
3057 size = mtd->oobsize - (oob - chip->oob_poi);
3068 * nand_read_page_swecc - [REPLACEABLE] software ECC based page read function
3069 * @chip: nand chip info structure
3071 * @oob_required: caller requires OOB data read to chip->oob_poi
3078 int i, eccsize = chip->ecc.size, ret;
3079 int eccbytes = chip->ecc.bytes;
3080 int eccsteps = chip->ecc.steps;
3082 uint8_t *ecc_calc = chip->ecc.calc_buf;
3083 uint8_t *ecc_code = chip->ecc.code_buf;
3086 chip->ecc.read_page_raw(chip, buf, 1, page);
3088 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
3089 chip->ecc.calculate(chip, p, &ecc_calc[i]);
3091 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3092 chip->ecc.total);
3096 eccsteps = chip->ecc.steps;
3099 for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3102 stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3104 mtd->ecc_stats.failed++;
3106 mtd->ecc_stats.corrected += stat;
3114 * nand_read_subpage - [REPLACEABLE] ECC based sub-page read function
3115 * @chip: nand chip info structure
3129 int busw = (chip->options & NAND_BUSWIDTH_16) ? 2 : 1;
3134 /* Column address within the page aligned to ECC size (256bytes) */
3135 start_step = data_offs / chip->ecc.size;
3136 end_step = (data_offs + readlen - 1) / chip->ecc.size;
3137 num_steps = end_step - start_step + 1;
3138 index = start_step * chip->ecc.bytes;
3140 /* Data size aligned to ECC ecc.size */
3141 datafrag_len = num_steps * chip->ecc.size;
3142 eccfrag_len = num_steps * chip->ecc.bytes;
3144 data_col_addr = start_step * chip->ecc.size;
3151 /* Calculate ECC */
3152 for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size)
3153 chip->ecc.calculate(chip, p, &chip->ecc.calc_buf[i]);
3157 * ecc.pos. Let's make sure that there are no gaps in ECC positions.
3167 ret = nand_change_read_column_op(chip, mtd->writesize,
3168 chip->oob_poi, mtd->oobsize,
3174 * Send the command to read the particular ECC bytes take care
3177 aligned_pos = oobregion.offset & ~(busw - 1);
3179 if (oobregion.offset & (busw - 1))
3181 if ((oobregion.offset + (num_steps * chip->ecc.bytes)) &
3182 (busw - 1))
3186 mtd->writesize + aligned_pos,
3187 &chip->oob_poi[aligned_pos],
3193 ret = mtd_ooblayout_get_eccbytes(mtd, chip->ecc.code_buf,
3194 chip->oob_poi, index, eccfrag_len);
3199 for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) {
3202 stat = chip->ecc.correct(chip, p, &chip->ecc.code_buf[i],
3203 &chip->ecc.calc_buf[i]);
3204 if (stat == -EBADMSG &&
3205 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3207 stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3208 &chip->ecc.code_buf[i],
3209 chip->ecc.bytes,
3211 chip->ecc.strength);
3215 mtd->ecc_stats.failed++;
3217 mtd->ecc_stats.corrected += stat;
3225 * nand_read_page_hwecc - [REPLACEABLE] hardware ECC based page read function
3226 * @chip: nand chip info structure
3228 * @oob_required: caller requires OOB data read to chip->oob_poi
3231 * Not for syndrome calculating ECC controllers which need a special oob layout.
3237 int i, eccsize = chip->ecc.size, ret;
3238 int eccbytes = chip->ecc.bytes;
3239 int eccsteps = chip->ecc.steps;
3241 uint8_t *ecc_calc = chip->ecc.calc_buf;
3242 uint8_t *ecc_code = chip->ecc.code_buf;
3249 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3250 chip->ecc.hwctl(chip, NAND_ECC_READ);
3256 chip->ecc.calculate(chip, p, &ecc_calc[i]);
3259 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
3264 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3265 chip->ecc.total);
3269 eccsteps = chip->ecc.steps;
3272 for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3275 stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]);
3276 if (stat == -EBADMSG &&
3277 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3282 chip->ecc.strength);
3286 mtd->ecc_stats.failed++;
3288 mtd->ecc_stats.corrected += stat;
3296 * nand_read_page_hwecc_oob_first - Hardware ECC page read with ECC
3298 * @chip: nand chip info structure
3300 * @oob_required: caller requires OOB data read to chip->oob_poi
3303 * Hardware ECC for large page chips, which requires the ECC data to be
3310 int i, eccsize = chip->ecc.size, ret;
3311 int eccbytes = chip->ecc.bytes;
3312 int eccsteps = chip->ecc.steps;
3314 uint8_t *ecc_code = chip->ecc.code_buf;
3318 ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3327 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
3328 chip->ecc.total);
3332 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3335 chip->ecc.hwctl(chip, NAND_ECC_READ);
3341 stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
3342 if (stat == -EBADMSG &&
3343 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3348 chip->ecc.strength);
3352 mtd->ecc_stats.failed++;
3354 mtd->ecc_stats.corrected += stat;
3363 * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
3364 * @chip: nand chip info structure
3366 * @oob_required: caller requires OOB data read to chip->oob_poi
3376 int ret, i, eccsize = chip->ecc.size;
3377 int eccbytes = chip->ecc.bytes;
3378 int eccsteps = chip->ecc.steps;
3379 int eccpadbytes = eccbytes + chip->ecc.prepad + chip->ecc.postpad;
3381 uint8_t *oob = chip->oob_poi;
3388 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
3391 chip->ecc.hwctl(chip, NAND_ECC_READ);
3397 if (chip->ecc.prepad) {
3398 ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
3403 oob += chip->ecc.prepad;
3406 chip->ecc.hwctl(chip, NAND_ECC_READSYN);
3412 stat = chip->ecc.correct(chip, p, oob, NULL);
3416 if (chip->ecc.postpad) {
3417 ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
3422 oob += chip->ecc.postpad;
3425 if (stat == -EBADMSG &&
3426 (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
3428 stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
3429 oob - eccpadbytes,
3432 chip->ecc.strength);
3436 mtd->ecc_stats.failed++;
3438 mtd->ecc_stats.corrected += stat;
3444 i = mtd->oobsize - (oob - chip->oob_poi);
3455 * nand_transfer_oob - [INTERN] Transfer oob to client buffer
3456 * @chip: NAND chip object
3467 switch (ops->mode) {
3471 memcpy(oob, chip->oob_poi + ops->ooboffs, len);
3475 ret = mtd_ooblayout_get_databytes(mtd, oob, chip->oob_poi,
3476 ops->ooboffs, len);
3492 chip->cont_read.ongoing = false;
3494 if (!chip->controller->supported_op.cont_read)
3501 if (readlen < (2 * mtd->writesize))
3510 last_page = page + ((col + readlen) / mtd->writesize) - 1;
3514 chip->cont_read.first_page = first_page;
3515 chip->cont_read.last_page = last_page;
3516 chip->cont_read.ongoing = true;
3524 if (!chip->cont_read.ongoing || page != chip->cont_read.first_page)
3527 chip->cont_read.first_page++;
3532 * nand_setup_read_retry - [INTERN] Set the READ RETRY mode
3533 * @chip: NAND chip object
3537 * when there are too many bitflips in a page (i.e., ECC error). After setting
3544 if (retry_mode >= chip->read_retries)
3545 return -EINVAL;
3547 if (!chip->ops.setup_read_retry)
3548 return -EOPNOTSUPP;
3550 return chip->ops.setup_read_retry(chip, retry_mode);
3557 if (!(chip->options & NAND_NEED_READRDY))
3565 * nand_do_read_ops - [INTERN] Read data with ECC
3566 * @chip: NAND chip object
3578 uint32_t readlen = ops->len;
3579 uint32_t oobreadlen = ops->ooblen;
3590 return -EIO;
3592 chipnr = (int)(from >> chip->chip_shift);
3595 realpage = (int)(from >> chip->page_shift);
3596 page = realpage & chip->pagemask;
3598 col = (int)(from & (mtd->writesize - 1));
3600 buf = ops->datbuf;
3601 oob = ops->oobbuf;
3604 if (likely(ops->mode != MTD_OPS_RAW))
3608 struct mtd_ecc_stats ecc_stats = mtd->ecc_stats;
3610 bytes = min(mtd->writesize - col, readlen);
3611 aligned = (bytes == mtd->writesize);
3615 else if (chip->options & NAND_USES_DMA)
3618 chip->buf_align);
3623 if (realpage != chip->pagecache.page || oob) {
3624 bufpoi = use_bounce_buf ? chip->data_buf : buf;
3633 * the read methods return max bitflips per ecc step.
3635 if (unlikely(ops->mode == MTD_OPS_RAW))
3636 ret = chip->ecc.read_page_raw(chip, bufpoi,
3641 ret = chip->ecc.read_subpage(chip, col, bytes,
3644 ret = chip->ecc.read_page(chip, bufpoi,
3649 chip->pagecache.page = -1;
3659 !(mtd->ecc_stats.failed - ecc_stats.failed) &&
3660 (ops->mode != MTD_OPS_RAW)) {
3661 chip->pagecache.page = realpage;
3662 chip->pagecache.bitflips = ret;
3665 chip->pagecache.page = -1;
3676 oobreadlen -= toread;
3682 if (mtd->ecc_stats.failed - ecc_stats.failed) {
3683 if (retry_mode + 1 < chip->read_retries) {
3691 mtd->ecc_stats = ecc_stats;
3702 memcpy(buf, chip->data_buf + col, bytes);
3705 chip->pagecache.bitflips);
3710 readlen -= bytes;
3728 page = realpage & chip->pagemask;
3738 if (WARN_ON_ONCE(chip->cont_read.ongoing))
3739 chip->cont_read.ongoing = false;
3741 ops->retlen = ops->len - (size_t) readlen;
3743 ops->oobretlen = ops->ooblen - oobreadlen;
3749 return -EBADMSG;
3755 * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function
3756 * @chip: nand chip info structure
3763 return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
3768 * nand_read_oob_syndrome - [REPLACEABLE] OOB data read function for HW ECC
3770 * @chip: nand chip info structure
3776 int length = mtd->oobsize;
3777 int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3778 int eccsize = chip->ecc.size;
3779 uint8_t *bufpoi = chip->oob_poi;
3782 ret = nand_read_page_op(chip, page, chip->ecc.size, NULL, 0);
3786 for (i = 0; i < chip->ecc.steps; i++) {
3791 if (mtd->writesize > 512)
3810 length -= toread;
3822 * nand_write_oob_std - [REPLACEABLE] the most common OOB data write function
3823 * @chip: nand chip info structure
3830 return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
3831 mtd->oobsize);
3836 * nand_write_oob_syndrome - [REPLACEABLE] OOB data write function for HW ECC
3837 * with syndrome - only for large page flash
3838 * @chip: nand chip info structure
3844 int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
3845 int eccsize = chip->ecc.size, length = mtd->oobsize;
3846 int ret, i, len, pos, sndcmd = 0, steps = chip->ecc.steps;
3847 const uint8_t *bufpoi = chip->oob_poi;
3850 * data-ecc-data-ecc ... ecc-oob
3852 * data-pad-ecc-pad-data-pad .... ecc-pad-oob
3854 if (!chip->ecc.prepad && !chip->ecc.postpad) {
3866 if (mtd->writesize <= 512) {
3878 len -= num;
3897 length -= len;
3909 * nand_do_read_oob - [INTERN] NAND read out-of-band
3910 * @chip: NAND chip object
3914 * NAND read out-of-band data from the spare area.
3923 int readlen = ops->ooblen;
3925 uint8_t *buf = ops->oobbuf;
3933 return -EIO;
3935 stats = mtd->ecc_stats;
3939 chipnr = (int)(from >> chip->chip_shift);
3943 realpage = (int)(from >> chip->page_shift);
3944 page = realpage & chip->pagemask;
3947 if (ops->mode == MTD_OPS_RAW)
3948 ret = chip->ecc.read_oob_raw(chip, page);
3950 ret = chip->ecc.read_oob(chip, page);
3962 readlen -= len;
3969 page = realpage & chip->pagemask;
3979 ops->oobretlen = ops->ooblen - readlen;
3984 if (mtd->ecc_stats.failed - stats.failed)
3985 return -EBADMSG;
3991 * nand_read_oob - [MTD Interface] NAND read data and/or out-of-band
3996 * NAND read data and/or out-of-band data.
4005 ops->retlen = 0;
4007 if (ops->mode != MTD_OPS_PLACE_OOB &&
4008 ops->mode != MTD_OPS_AUTO_OOB &&
4009 ops->mode != MTD_OPS_RAW)
4010 return -ENOTSUPP;
4014 old_stats = mtd->ecc_stats;
4016 if (!ops->datbuf)
4021 if (ops->stats) {
4022 ops->stats->uncorrectable_errors +=
4023 mtd->ecc_stats.failed - old_stats.failed;
4024 ops->stats->corrected_bitflips +=
4025 mtd->ecc_stats.corrected - old_stats.corrected;
4033 * nand_write_page_raw_notsupp - dummy raw page write function
4034 * @chip: nand chip info structure
4036 * @oob_required: must write chip->oob_poi to OOB
4039 * Returns -ENOTSUPP unconditionally.
4044 return -ENOTSUPP;
4048 * nand_write_page_raw - [INTERN] raw page write function
4049 * @chip: nand chip info structure
4051 * @oob_required: must write chip->oob_poi to OOB
4054 * Not for syndrome calculating ECC controllers, which use a special oob layout.
4062 ret = nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
4067 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize,
4078 * nand_monolithic_write_page_raw - Monolithic page write in raw mode
4079 * @chip: NAND chip info structure
4081 * @oob_required: must write chip->oob_poi to OOB
4087 * into the NAND chip arrays in a single operation. This is an
4090 * cycles on the NAND bus, and finally sends the program command to
4091 * synchronyze the NAND chip cache.
4097 unsigned int size = mtd->writesize;
4101 size += mtd->oobsize;
4103 if (buf != chip->data_buf) {
4105 memcpy(write_buf, buf, mtd->writesize);
4114 * nand_write_page_raw_syndrome - [INTERN] raw page write function
4115 * @chip: nand chip info structure
4117 * @oob_required: must write chip->oob_poi to OOB
4120 * We need a special oob layout and handling even when ECC isn't checked.
4127 int eccsize = chip->ecc.size;
4128 int eccbytes = chip->ecc.bytes;
4129 uint8_t *oob = chip->oob_poi;
4136 for (steps = chip->ecc.steps; steps > 0; steps--) {
4143 if (chip->ecc.prepad) {
4144 ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4149 oob += chip->ecc.prepad;
4158 if (chip->ecc.postpad) {
4159 ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4164 oob += chip->ecc.postpad;
4168 size = mtd->oobsize - (oob - chip->oob_poi);
4178 * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function
4179 * @chip: nand chip info structure
4181 * @oob_required: must write chip->oob_poi to OOB
4188 int i, eccsize = chip->ecc.size, ret;
4189 int eccbytes = chip->ecc.bytes;
4190 int eccsteps = chip->ecc.steps;
4191 uint8_t *ecc_calc = chip->ecc.calc_buf;
4194 /* Software ECC calculation */
4195 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
4196 chip->ecc.calculate(chip, p, &ecc_calc[i]);
4198 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4199 chip->ecc.total);
4203 return chip->ecc.write_page_raw(chip, buf, 1, page);
4207 * nand_write_page_hwecc - [REPLACEABLE] hardware ECC based page write function
4208 * @chip: nand chip info structure
4210 * @oob_required: must write chip->oob_poi to OOB
4217 int i, eccsize = chip->ecc.size, ret;
4218 int eccbytes = chip->ecc.bytes;
4219 int eccsteps = chip->ecc.steps;
4220 uint8_t *ecc_calc = chip->ecc.calc_buf;
4227 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
4228 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4234 chip->ecc.calculate(chip, p, &ecc_calc[i]);
4237 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4238 chip->ecc.total);
4242 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
4251 * nand_write_subpage_hwecc - [REPLACEABLE] hardware ECC based subpage write
4252 * @chip: nand chip info structure
4256 * @oob_required: must write chip->oob_poi to OOB
4264 uint8_t *oob_buf = chip->oob_poi;
4265 uint8_t *ecc_calc = chip->ecc.calc_buf;
4266 int ecc_size = chip->ecc.size;
4267 int ecc_bytes = chip->ecc.bytes;
4268 int ecc_steps = chip->ecc.steps;
4270 uint32_t end_step = (offset + data_len - 1) / ecc_size;
4271 int oob_bytes = mtd->oobsize / ecc_steps;
4280 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4287 /* mask ECC of un-touched subpages by padding 0xFF */
4291 chip->ecc.calculate(chip, buf, ecc_calc);
4293 /* mask OOB of un-touched subpages by padding 0xFF */
4303 /* copy calculated ECC for whole page to chip->buffer->oob */
4304 /* this include masked-value(0xFF) for unwritten subpages */
4305 ecc_calc = chip->ecc.calc_buf;
4306 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
4307 chip->ecc.total);
4311 /* write OOB buffer to NAND device */
4312 ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
4321 * nand_write_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page write
4322 * @chip: nand chip info structure
4324 * @oob_required: must write chip->oob_poi to OOB
4334 int i, eccsize = chip->ecc.size;
4335 int eccbytes = chip->ecc.bytes;
4336 int eccsteps = chip->ecc.steps;
4338 uint8_t *oob = chip->oob_poi;
4345 for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
4346 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
4352 if (chip->ecc.prepad) {
4353 ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
4358 oob += chip->ecc.prepad;
4361 chip->ecc.calculate(chip, p, oob);
4369 if (chip->ecc.postpad) {
4370 ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
4375 oob += chip->ecc.postpad;
4380 i = mtd->oobsize - (oob - chip->oob_poi);
4391 * nand_write_page - write one page
4392 * @chip: NAND chip descriptor
4396 * @oob_required: must write chip->oob_poi to OOB
4407 if (!(chip->options & NAND_NO_SUBPAGE_WRITE) &&
4408 chip->ecc.write_subpage)
4409 subpage = offset || (data_len < mtd->writesize);
4414 status = chip->ecc.write_page_raw(chip, buf, oob_required,
4417 status = chip->ecc.write_subpage(chip, offset, data_len, buf,
4420 status = chip->ecc.write_page(chip, buf, oob_required, page);
4428 #define NOTALIGNED(x) ((x & (chip->subpagesize - 1)) != 0)
4431 * nand_do_write_ops - [INTERN] NAND write with ECC
4432 * @chip: NAND chip object
4436 * NAND write with ECC.
4443 uint32_t writelen = ops->len;
4445 uint32_t oobwritelen = ops->ooblen;
4448 uint8_t *oob = ops->oobbuf;
4449 uint8_t *buf = ops->datbuf;
4453 ops->retlen = 0;
4458 if (NOTALIGNED(to) || NOTALIGNED(ops->len)) {
4461 return -EINVAL;
4466 return -EIO;
4468 column = to & (mtd->writesize - 1);
4470 chipnr = (int)(to >> chip->chip_shift);
4475 ret = -EIO;
4479 realpage = (int)(to >> chip->page_shift);
4480 page = realpage & chip->pagemask;
4483 if (to <= ((loff_t)chip->pagecache.page << chip->page_shift) &&
4484 ((loff_t)chip->pagecache.page << chip->page_shift) < (to + ops->len))
4485 chip->pagecache.page = -1;
4488 if (oob && ops->ooboffs && (ops->ooboffs + ops->ooblen > oobmaxlen)) {
4489 ret = -EINVAL;
4494 int bytes = mtd->writesize;
4497 int part_pagewr = (column || writelen < mtd->writesize);
4501 else if (chip->options & NAND_USES_DMA)
4504 chip->buf_align);
4516 bytes = min_t(int, bytes - column, writelen);
4518 memset(wbuf, 0xff, mtd->writesize);
4525 oobwritelen -= len;
4528 memset(chip->oob_poi, 0xff, mtd->oobsize);
4533 (ops->mode == MTD_OPS_RAW));
4537 writelen -= bytes;
4545 page = realpage & chip->pagemask;
4554 ops->retlen = ops->len - writelen;
4556 ops->oobretlen = ops->ooblen;
4564 * panic_nand_write - [MTD Interface] NAND write with ECC
4571 * NAND write with ECC. Used when performing writes in interrupt context, this
4578 int chipnr = (int)(to >> chip->chip_shift);
4599 * nand_write_oob - [MTD Interface] NAND write data and/or out-of-band
4610 ops->retlen = 0;
4614 switch (ops->mode) {
4624 if (!ops->datbuf)
4635 * nand_erase - [MTD Interface] erase block(s)
4647 * nand_erase_nand - [INTERN] erase block(s)
4648 * @chip: NAND chip object
4661 __func__, (unsigned long long)instr->addr,
4662 (unsigned long long)instr->len);
4664 if (check_offs_len(chip, instr->addr, instr->len))
4665 return -EINVAL;
4668 if (nand_region_is_secured(chip, instr->addr, instr->len))
4669 return -EIO;
4675 page = (int)(instr->addr >> chip->page_shift);
4676 chipnr = (int)(instr->addr >> chip->chip_shift);
4679 pages_per_block = 1 << (chip->phys_erase_shift - chip->page_shift);
4681 /* Select the NAND device */
4688 ret = -EIO;
4693 len = instr->len;
4696 loff_t ofs = (loff_t)page << chip->page_shift;
4700 chip->page_shift, allowbbt)) {
4703 ret = -EIO;
4711 if (page <= chip->pagecache.page && chip->pagecache.page <
4713 chip->pagecache.page = -1;
4715 ret = nand_erase_op(chip, (page & chip->pagemask) >>
4716 (chip->phys_erase_shift - chip->page_shift));
4720 instr->fail_addr = ofs;
4725 len -= (1ULL << chip->phys_erase_shift);
4729 if (len && !(page & chip->pagemask)) {
4748 * nand_sync - [MTD Interface] sync
4766 * nand_block_isbad - [MTD Interface] Check if block at offset is bad
4773 int chipnr = (int)(offs >> chip->chip_shift);
4776 /* Select the NAND device */
4790 * nand_block_markbad - [MTD Interface] Mark block at the given offset as bad
4810 * nand_suspend - [MTD Interface] Suspend the NAND flash
4820 mutex_lock(&chip->lock);
4821 if (chip->ops.suspend)
4822 ret = chip->ops.suspend(chip);
4824 chip->suspended = 1;
4825 mutex_unlock(&chip->lock);
4831 * nand_resume - [MTD Interface] Resume the NAND flash
4838 mutex_lock(&chip->lock);
4839 if (chip->suspended) {
4840 if (chip->ops.resume)
4841 chip->ops.resume(chip);
4842 chip->suspended = 0;
4847 mutex_unlock(&chip->lock);
4849 wake_up_all(&chip->resume_wq);
4853 * nand_shutdown - [MTD Interface] Finish the current NAND operation and
4863 * nand_lock - [MTD Interface] Lock the NAND flash
4872 if (!chip->ops.lock_area)
4873 return -ENOTSUPP;
4875 return chip->ops.lock_area(chip, ofs, len);
4879 * nand_unlock - [MTD Interface] Unlock the NAND flash
4888 if (!chip->ops.unlock_area)
4889 return -ENOTSUPP;
4891 return chip->ops.unlock_area(chip, ofs, len);
4898 if (!chip->controller) {
4899 chip->controller = &chip->legacy.dummy_controller;
4900 nand_controller_init(chip->controller);
4905 if (!chip->buf_align)
4906 chip->buf_align = 1;
4915 s[len - 1] = 0;
4918 for (i = 0; i < len - 1; i++) {
4928 * nand_id_has_period - Check if an ID string has a given wraparound period
4935 * period of 3). This is a helper function for nand_id_len(). Returns non-zero
4949 * nand_id_len - Get the length of an ID string returned by CMD_READID
4960 /* Find last non-zero byte */
4961 for (last_nonzero = arrlen - 1; last_nonzero >= 0; last_nonzero--)
4979 if (last_nonzero < arrlen - 1)
4997 * Many new NAND share similar device ID codes, which represent the size of the
4999 * manufacturer-specific "extended ID" decoding patterns.
5006 u8 *id_data = chip->id.data;
5008 memorg = nanddev_get_memorg(&chip->base);
5011 memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
5016 memorg->pagesize = 1024 << (extid & 0x03);
5017 mtd->writesize = memorg->pagesize;
5020 memorg->oobsize = (8 << (extid & 0x01)) * (mtd->writesize >> 9);
5021 mtd->oobsize = memorg->oobsize;
5024 memorg->pages_per_eraseblock = ((64 * 1024) << (extid & 0x03)) /
5025 memorg->pagesize;
5026 mtd->erasesize = (64 * 1024) << (extid & 0x03);
5030 chip->options |= NAND_BUSWIDTH_16;
5044 memorg = nanddev_get_memorg(&chip->base);
5046 memorg->pages_per_eraseblock = type->erasesize / type->pagesize;
5047 mtd->erasesize = type->erasesize;
5048 memorg->pagesize = type->pagesize;
5049 mtd->writesize = memorg->pagesize;
5050 memorg->oobsize = memorg->pagesize / 32;
5051 mtd->oobsize = memorg->oobsize;
5053 /* All legacy ID NAND are small-page, SLC */
5054 memorg->bits_per_cell = 1;
5060 * page size, cell-type information).
5067 if (mtd->writesize > 512 || (chip->options & NAND_BUSWIDTH_16))
5068 chip->badblockpos = NAND_BBM_POS_LARGE;
5070 chip->badblockpos = NAND_BBM_POS_SMALL;
5075 return type->id_len;
5081 struct nand_device *base = &chip->base;
5085 u8 *id_data = chip->id.data;
5087 memorg = nanddev_get_memorg(&chip->base);
5089 if (!strncmp(type->id, id_data, type->id_len)) {
5090 memorg->pagesize = type->pagesize;
5091 mtd->writesize = memorg->pagesize;
5092 memorg->pages_per_eraseblock = type->erasesize /
5093 type->pagesize;
5094 mtd->erasesize = type->erasesize;
5095 memorg->oobsize = type->oobsize;
5096 mtd->oobsize = memorg->oobsize;
5098 memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]);
5099 memorg->eraseblocks_per_lun =
5100 DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
5101 memorg->pagesize *
5102 memorg->pages_per_eraseblock);
5103 chip->options |= type->options;
5108 chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
5109 if (!chip->parameters.model)
5118 * Manufacturer detection. Only used when the NAND is not ONFI or JEDEC
5119 * compliant and does not have a full-id or legacy-id entry in the nand_ids
5128 if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
5129 chip->manufacturer.desc->ops->detect) {
5132 memorg = nanddev_get_memorg(&chip->base);
5135 memorg->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]);
5136 chip->manufacturer.desc->ops->detect(chip);
5146 * their ->init() hook.
5150 if (!chip->manufacturer.desc || !chip->manufacturer.desc->ops ||
5151 !chip->manufacturer.desc->ops->init)
5154 return chip->manufacturer.desc->ops->init(chip);
5161 * ->cleanup() hook.
5166 if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
5167 chip->manufacturer.desc->ops->cleanup)
5168 chip->manufacturer.desc->ops->cleanup(chip);
5174 return manufacturer_desc ? manufacturer_desc->name : "Unknown";
5181 chip->controller->supported_op.data_only_read = 1;
5187 WARN_ON_ONCE(chip->controller->supported_op.data_only_read);
5199 if (!chip->parameters.supports_read_cache)
5202 if (chip->read_retries)
5206 mtd->writesize, true))
5207 chip->controller->supported_op.cont_read = 1;
5213 WARN_ON_ONCE(chip->controller->supported_op.cont_read);
5216 * Too many devices do not support sequential cached reads with on-die
5217 * ECC correction enabled, so in this case refuse to perform the
5220 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE)
5230 if (!(chip->ecc.read_page == nand_read_page_hwecc ||
5231 chip->ecc.read_page == nand_read_page_syndrome ||
5232 chip->ecc.read_page == nand_read_page_swecc))
5247 u8 *id_data = chip->id.data;
5253 * unassigned by the ID-based detection logic.
5255 memorg = nanddev_get_memorg(&chip->base);
5256 memorg->planes_per_lun = 1;
5257 memorg->luns_per_target = 1;
5261 * after power-up.
5282 * Try again to make sure, as some systems the bus-hold or other
5284 * possibly credible NAND flash to appear. If the two results do
5289 ret = nand_readid_op(chip, 0, id_data, sizeof(chip->id.data));
5296 return -ENODEV;
5299 chip->id.len = nand_id_len(id_data, ARRAY_SIZE(chip->id.data));
5303 chip->manufacturer.desc = manufacturer_desc;
5309 * Save the NAND_BUSWIDTH_16 flag before letting auto-detection logic
5311 * This is required to make sure initial NAND bus width set by the
5312 * NAND controller driver is coherent with the real NAND bus width
5313 * (extracted by auto-detection code).
5315 busw = chip->options & NAND_BUSWIDTH_16;
5319 * before starting auto-detection.
5321 chip->options &= ~NAND_BUSWIDTH_16;
5323 for (; type->name != NULL; type++) {
5327 } else if (dev_id == type->dev_id) {
5332 if (!type->name || !type->pagesize) {
5348 if (!type->name)
5349 return -ENODEV;
5351 chip->parameters.model = kstrdup(type->name, GFP_KERNEL);
5352 if (!chip->parameters.model)
5353 return -ENOMEM;
5355 if (!type->pagesize)
5361 chip->options |= type->options;
5363 memorg->eraseblocks_per_lun =
5364 DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20,
5365 memorg->pagesize *
5366 memorg->pages_per_eraseblock);
5369 if (!mtd->name)
5370 mtd->name = chip->parameters.model;
5372 if (chip->options & NAND_BUSWIDTH_AUTO) {
5375 } else if (busw != (chip->options & NAND_BUSWIDTH_16)) {
5383 mtd->name);
5385 (chip->options & NAND_BUSWIDTH_16) ? 16 : 8);
5386 ret = -EINVAL;
5394 chip->page_shift = ffs(mtd->writesize) - 1;
5395 /* Convert chipsize to number of pages per chip -1 */
5396 targetsize = nanddev_target_size(&chip->base);
5397 chip->pagemask = (targetsize >> chip->page_shift) - 1;
5399 chip->bbt_erase_shift = chip->phys_erase_shift =
5400 ffs(mtd->erasesize) - 1;
5402 chip->chip_shift = ffs((unsigned)targetsize) - 1;
5404 chip->chip_shift = ffs((unsigned)(targetsize >> 32));
5405 chip->chip_shift += 32 - 1;
5408 if (chip->chip_shift - chip->page_shift > 16)
5409 chip->options |= NAND_ROW_ADDR_3;
5411 chip->badblockbits = 8;
5418 chip->parameters.model);
5421 mtd->erasesize >> 10, mtd->writesize, mtd->oobsize);
5425 kfree(chip->parameters.model);
5448 [NAND_ECC_ON_DIE] = "on-die",
5454 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5487 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5501 err = of_property_read_string(np, "nand-ecc-mode", &pm);
5515 struct nand_ecc_props *user_conf = &chip->base.ecc.user_conf;
5517 if (user_conf->engine_type == NAND_ECC_ENGINE_TYPE_INVALID)
5518 user_conf->engine_type = of_get_rawnand_ecc_engine_type_legacy(dn);
5520 if (user_conf->algo == NAND_ECC_ALGO_UNKNOWN)
5521 user_conf->algo = of_get_rawnand_ecc_algo_legacy(dn);
5523 if (user_conf->placement == NAND_ECC_PLACEMENT_UNKNOWN)
5524 user_conf->placement = of_get_rawnand_ecc_placement_legacy(dn);
5533 ret = of_property_read_u32(dn, "nand-bus-width", &val);
5534 if (ret == -EINVAL)
5541 chip->options |= NAND_BUSWIDTH_16;
5543 return -EINVAL;
5553 /* Only proceed if the "secure-regions" property is present in DT */
5554 prop = of_find_property(dn, "secure-regions", NULL);
5558 nr_elem = of_property_count_elems_of_size(dn, "secure-regions", sizeof(u64));
5562 chip->nr_secure_regions = nr_elem / 2;
5563 chip->secure_regions = kcalloc(chip->nr_secure_regions, sizeof(*chip->secure_regions),
5565 if (!chip->secure_regions)
5566 return -ENOMEM;
5568 for (i = 0, j = 0; i < chip->nr_secure_regions; i++, j += 2) {
5569 of_property_read_u64_index(dn, "secure-regions", j,
5570 &chip->secure_regions[i].offset);
5571 of_property_read_u64_index(dn, "secure-regions", j + 1,
5572 &chip->secure_regions[i].size);
5579 * rawnand_dt_parse_gpio_cs - Parse the gpio-cs property of a controller
5593 dev_dbg(dev, "No valid cs-gpios property\n");
5599 return -ENOMEM;
5617 struct nand_device *nand = mtd_to_nanddev(nand_to_mtd(chip));
5628 if (of_property_read_bool(dn, "nand-is-boot-medium"))
5629 chip->options |= NAND_IS_BOOT_MEDIUM;
5631 if (of_property_read_bool(dn, "nand-on-flash-bbt"))
5632 chip->bbt_options |= NAND_BBT_USE_FLASH;
5634 of_get_nand_ecc_user_config(nand);
5638 * If neither the user nor the NAND controller have requested a specific
5639 * ECC engine type, we will default to NAND_ECC_ENGINE_TYPE_ON_HOST.
5641 nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
5645 * case default to the NAND controller choice, otherwise fallback to
5646 * the raw NAND default one.
5648 if (nand->ecc.user_conf.engine_type != NAND_ECC_ENGINE_TYPE_INVALID)
5649 chip->ecc.engine_type = nand->ecc.user_conf.engine_type;
5650 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_INVALID)
5651 chip->ecc.engine_type = nand->ecc.defaults.engine_type;
5653 chip->ecc.placement = nand->ecc.user_conf.placement;
5654 chip->ecc.algo = nand->ecc.user_conf.algo;
5655 chip->ecc.strength = nand->ecc.user_conf.strength;
5656 chip->ecc.size = nand->ecc.user_conf.step_size;
5662 * nand_scan_ident - Scan for the NAND device
5663 * @chip: NAND chip object
5665 * @table: alternative NAND ID table
5671 * to tweak some ECC-related parameters before nand_scan_tail(). This separation
5673 * as been banned for the benefit of the ->init_ecc()/cleanup_ecc() hooks.
5684 memorg = nanddev_get_memorg(&chip->base);
5687 chip->cur_cs = -1;
5689 mutex_init(&chip->lock);
5690 init_waitqueue_head(&chip->resume_wq);
5693 chip->current_interface_config = nand_get_reset_interface_config();
5699 if (!mtd->name && mtd->dev.parent)
5700 mtd->name = dev_name(mtd->dev.parent);
5709 memorg->ntargets = maxchips;
5714 if (!(chip->options & NAND_SCAN_SILENT_NODEV))
5715 pr_warn("No NAND device found\n");
5720 nand_maf_id = chip->id.data[0];
5721 nand_dev_id = chip->id.data[1];
5750 memorg->ntargets = i;
5751 mtd->size = i * nanddev_target_size(&chip->base);
5758 kfree(chip->parameters.model);
5759 kfree(chip->parameters.onfi);
5765 struct nand_device *base = &chip->base;
5768 base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
5769 base->ecc.user_conf.algo = NAND_ECC_ALGO_HAMMING;
5770 base->ecc.user_conf.strength = chip->ecc.strength;
5771 base->ecc.user_conf.step_size = chip->ecc.size;
5777 engine_conf = base->ecc.ctx.priv;
5779 if (chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER)
5780 engine_conf->sm_order = true;
5782 chip->ecc.size = base->ecc.ctx.conf.step_size;
5783 chip->ecc.strength = base->ecc.ctx.conf.strength;
5784 chip->ecc.total = base->ecc.ctx.total;
5785 chip->ecc.steps = nanddev_get_ecc_nsteps(base);
5786 chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base);
5796 struct nand_device *base = &chip->base;
5807 struct nand_device *base = &chip->base;
5815 struct nand_device *base = &chip->base;
5823 struct nand_device *base = &chip->base;
5827 base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
5828 base->ecc.user_conf.algo = NAND_ECC_ALGO_BCH;
5829 base->ecc.user_conf.step_size = chip->ecc.size;
5830 base->ecc.user_conf.strength = chip->ecc.strength;
5836 chip->ecc.size = ecc_conf->step_size;
5837 chip->ecc.strength = ecc_conf->strength;
5838 chip->ecc.total = base->ecc.ctx.total;
5839 chip->ecc.steps = nanddev_get_ecc_nsteps(base);
5840 chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base);
5850 struct nand_device *base = &chip->base;
5858 struct nand_device *base = &chip->base;
5866 struct nand_device *base = &chip->base;
5874 struct nand_ecc_ctrl *ecc = &chip->ecc;
5876 switch (ecc->placement) {
5880 if (!ecc->read_page)
5881 ecc->read_page = nand_read_page_hwecc;
5882 if (!ecc->write_page)
5883 ecc->write_page = nand_write_page_hwecc;
5884 if (!ecc->read_page_raw)
5885 ecc->read_page_raw = nand_read_page_raw;
5886 if (!ecc->write_page_raw)
5887 ecc->write_page_raw = nand_write_page_raw;
5888 if (!ecc->read_oob)
5889 ecc->read_oob = nand_read_oob_std;
5890 if (!ecc->write_oob)
5891 ecc->write_oob = nand_write_oob_std;
5892 if (!ecc->read_subpage)
5893 ecc->read_subpage = nand_read_subpage;
5894 if (!ecc->write_subpage && ecc->hwctl && ecc->calculate)
5895 ecc->write_subpage = nand_write_subpage_hwecc;
5899 if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) &&
5900 (!ecc->read_page ||
5901 ecc->read_page == nand_read_page_hwecc ||
5902 !ecc->write_page ||
5903 ecc->write_page == nand_write_page_hwecc)) {
5904 WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
5905 return -EINVAL;
5908 if (!ecc->read_page)
5909 ecc->read_page = nand_read_page_syndrome;
5910 if (!ecc->write_page)
5911 ecc->write_page = nand_write_page_syndrome;
5912 if (!ecc->read_page_raw)
5913 ecc->read_page_raw = nand_read_page_raw_syndrome;
5914 if (!ecc->write_page_raw)
5915 ecc->write_page_raw = nand_write_page_raw_syndrome;
5916 if (!ecc->read_oob)
5917 ecc->read_oob = nand_read_oob_syndrome;
5918 if (!ecc->write_oob)
5919 ecc->write_oob = nand_write_oob_syndrome;
5924 ecc->placement);
5925 return -EINVAL;
5935 struct nand_ecc_ctrl *ecc = &chip->ecc;
5938 if (WARN_ON(ecc->engine_type != NAND_ECC_ENGINE_TYPE_SOFT))
5939 return -EINVAL;
5941 switch (ecc->algo) {
5943 ecc->calculate = rawnand_sw_hamming_calculate;
5944 ecc->correct = rawnand_sw_hamming_correct;
5945 ecc->read_page = nand_read_page_swecc;
5946 ecc->read_subpage = nand_read_subpage;
5947 ecc->write_page = nand_write_page_swecc;
5948 if (!ecc->read_page_raw)
5949 ecc->read_page_raw = nand_read_page_raw;
5950 if (!ecc->write_page_raw)
5951 ecc->write_page_raw = nand_write_page_raw;
5952 ecc->read_oob = nand_read_oob_std;
5953 ecc->write_oob = nand_write_oob_std;
5954 if (!ecc->size)
5955 ecc->size = 256;
5956 ecc->bytes = 3;
5957 ecc->strength = 1;
5960 ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER;
5964 WARN(1, "Hamming ECC initialization failed!\n");
5972 return -EINVAL;
5974 ecc->calculate = rawnand_sw_bch_calculate;
5975 ecc->correct = rawnand_sw_bch_correct;
5976 ecc->read_page = nand_read_page_swecc;
5977 ecc->read_subpage = nand_read_subpage;
5978 ecc->write_page = nand_write_page_swecc;
5979 if (!ecc->read_page_raw)
5980 ecc->read_page_raw = nand_read_page_raw;
5981 if (!ecc->write_page_raw)
5982 ecc->write_page_raw = nand_write_page_raw;
5983 ecc->read_oob = nand_read_oob_std;
5984 ecc->write_oob = nand_write_oob_std;
5987 * We can only maximize ECC config when the default layout is
5991 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH &&
5992 mtd->ooblayout != nand_get_large_page_ooblayout())
5993 nanddev->ecc.user_conf.flags &= ~NAND_ECC_MAXIMIZE_STRENGTH;
5997 WARN(1, "BCH ECC initialization failed!\n");
6003 WARN(1, "Unsupported ECC algorithm!\n");
6004 return -EINVAL;
6009 * nand_check_ecc_caps - check the sanity of preset ECC settings
6010 * @chip: nand chip info structure
6011 * @caps: ECC caps info structure
6012 * @oobavail: OOB size that the ECC engine can use
6014 * When ECC step size and strength are already set, check if they are supported
6015 * by the controller and the calculated ECC bytes fit within the chip's OOB.
6016 * On success, the calculated ECC bytes is set.
6024 int preset_step = chip->ecc.size;
6025 int preset_strength = chip->ecc.strength;
6026 int ecc_bytes, nsteps = mtd->writesize / preset_step;
6029 for (i = 0; i < caps->nstepinfos; i++) {
6030 stepinfo = &caps->stepinfos[i];
6032 if (stepinfo->stepsize != preset_step)
6035 for (j = 0; j < stepinfo->nstrengths; j++) {
6036 if (stepinfo->strengths[j] != preset_strength)
6039 ecc_bytes = caps->calc_ecc_bytes(preset_step,
6045 pr_err("ECC (step, strength) = (%d, %d) does not fit in OOB",
6047 return -ENOSPC;
6050 chip->ecc.bytes = ecc_bytes;
6056 pr_err("ECC (step, strength) = (%d, %d) not supported on this controller",
6059 return -ENOTSUPP;
6063 * nand_match_ecc_req - meet the chip's requirement with least ECC bytes
6064 * @chip: nand chip info structure
6065 * @caps: ECC engine caps info structure
6066 * @oobavail: OOB size that the ECC engine can use
6068 * If a chip's ECC requirement is provided, try to meet it with the least
6069 * number of ECC bytes (i.e. with the largest number of OOB-free bytes).
6070 * On success, the chosen ECC settings are set.
6077 nanddev_get_ecc_requirements(&chip->base);
6080 int req_step = requirements->step_size;
6081 int req_strength = requirements->strength;
6087 /* No information provided by the NAND chip */
6089 return -ENOTSUPP;
6092 req_corr = mtd->writesize / req_step * req_strength;
6094 for (i = 0; i < caps->nstepinfos; i++) {
6095 stepinfo = &caps->stepinfos[i];
6096 step_size = stepinfo->stepsize;
6098 for (j = 0; j < stepinfo->nstrengths; j++) {
6099 strength = stepinfo->strengths[j];
6109 if (mtd->writesize % step_size)
6112 nsteps = mtd->writesize / step_size;
6114 ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
6125 * with the least number of ECC bytes.
6137 return -ENOTSUPP;
6139 chip->ecc.size = best_step;
6140 chip->ecc.strength = best_strength;
6141 chip->ecc.bytes = best_ecc_bytes;
6147 * nand_maximize_ecc - choose the max ECC strength available
6148 * @chip: nand chip info structure
6149 * @caps: ECC engine caps info structure
6150 * @oobavail: OOB size that the ECC engine can use
6152 * Choose the max ECC strength that is supported on the controller, and can fit
6153 * within the chip's OOB. On success, the chosen ECC settings are set.
6167 for (i = 0; i < caps->nstepinfos; i++) {
6168 stepinfo = &caps->stepinfos[i];
6169 step_size = stepinfo->stepsize;
6171 /* If chip->ecc.size is already set, respect it */
6172 if (chip->ecc.size && step_size != chip->ecc.size)
6175 for (j = 0; j < stepinfo->nstrengths; j++) {
6176 strength = stepinfo->strengths[j];
6178 if (mtd->writesize % step_size)
6181 nsteps = mtd->writesize / step_size;
6183 ecc_bytes = caps->calc_ecc_bytes(step_size, strength);
6207 return -ENOTSUPP;
6209 chip->ecc.size = best_step;
6210 chip->ecc.strength = best_strength;
6211 chip->ecc.bytes = best_ecc_bytes;
6217 * nand_ecc_choose_conf - Set the ECC strength and ECC step size
6218 * @chip: nand chip info structure
6219 * @caps: ECC engine caps info structure
6220 * @oobavail: OOB size that the ECC engine can use
6222 * Choose the ECC configuration according to following logic.
6224 * 1. If both ECC step size and ECC strength are already set (usually by DT)
6226 * 2. If the user provided the nand-ecc-maximize property, then select maximum
6227 * ECC strength.
6228 * 3. Otherwise, try to match the ECC step size and ECC strength closest
6230 * requirement then fallback to the maximum ECC step size and ECC strength.
6232 * On success, the chosen ECC settings are set.
6240 if (WARN_ON(oobavail < 0 || oobavail > mtd->oobsize))
6241 return -EINVAL;
6243 if (chip->ecc.size && chip->ecc.strength)
6246 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH)
6256 static int rawnand_erase(struct nand_device *nand, const struct nand_pos *pos)
6258 struct nand_chip *chip = container_of(nand, struct nand_chip,
6260 unsigned int eb = nanddev_pos_to_row(nand, pos);
6263 eb >>= nand->rowconv.eraseblock_addr_shift;
6265 nand_select_target(chip, pos->target);
6272 static int rawnand_markbad(struct nand_device *nand,
6275 struct nand_chip *chip = container_of(nand, struct nand_chip,
6278 return nand_markbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
6281 static bool rawnand_isbad(struct nand_device *nand, const struct nand_pos *pos)
6283 struct nand_chip *chip = container_of(nand, struct nand_chip,
6287 nand_select_target(chip, pos->target);
6288 ret = nand_isbad_bbm(chip, nanddev_pos_to_offs(nand, pos));
6301 * nand_scan_tail - Scan for the NAND device
6302 * @chip: NAND chip object
6311 struct nand_device *base = &chip->base;
6312 struct nand_ecc_ctrl *ecc = &chip->ecc;
6315 /* New bad blocks should be marked in OOB, flash-based BBT, or both */
6316 if (WARN_ON((chip->bbt_options & NAND_BBT_NO_OOB_BBM) &&
6317 !(chip->bbt_options & NAND_BBT_USE_FLASH))) {
6318 return -EINVAL;
6321 chip->data_buf = kmalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
6322 if (!chip->data_buf)
6323 return -ENOMEM;
6326 * FIXME: some NAND manufacturer drivers expect the first die to be
6327 * selected when manufacturer->init() is called. They should be fixed
6328 * to explictly select the relevant die when interacting with the NAND
6338 chip->oob_poi = chip->data_buf + mtd->writesize;
6343 if (!mtd->ooblayout &&
6344 !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
6345 ecc->algo == NAND_ECC_ALGO_BCH) &&
6346 !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
6347 ecc->algo == NAND_ECC_ALGO_HAMMING)) {
6348 switch (mtd->oobsize) {
6362 * ->oobsize, but we must keep the old large/small
6363 * page with ECC layout when ->oobsize <= 128 for
6366 if (ecc->engine_type == NAND_ECC_ENGINE_TYPE_NONE) {
6373 mtd->oobsize);
6374 ret = -EINVAL;
6380 * Check ECC mode, default to software if 3byte/512byte hardware ECC is
6381 * selected and we have 256 byte pagesize fallback to software ECC
6384 switch (ecc->engine_type) {
6390 if (mtd->writesize >= ecc->size) {
6391 if (!ecc->strength) {
6392 WARN(1, "Driver must set ecc.strength when using hardware ECC\n");
6393 ret = -EINVAL;
6398 pr_warn("%d byte HW ECC not possible on %d byte page size, fallback to SW ECC\n",
6399 ecc->size, mtd->writesize);
6400 ecc->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
6401 ecc->algo = NAND_ECC_ALGO_HAMMING;
6411 if (!ecc->read_page || !ecc->write_page) {
6412 WARN(1, "No ECC functions supplied; on-die ECC not possible\n");
6413 ret = -EINVAL;
6416 if (!ecc->read_oob)
6417 ecc->read_oob = nand_read_oob_std;
6418 if (!ecc->write_oob)
6419 ecc->write_oob = nand_write_oob_std;
6424 ecc->read_page = nand_read_page_raw;
6425 ecc->write_page = nand_write_page_raw;
6426 ecc->read_oob = nand_read_oob_std;
6427 ecc->read_page_raw = nand_read_page_raw;
6428 ecc->write_page_raw = nand_write_page_raw;
6429 ecc->write_oob = nand_write_oob_std;
6430 ecc->size = mtd->writesize;
6431 ecc->bytes = 0;
6432 ecc->strength = 0;
6436 WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->engine_type);
6437 ret = -EINVAL;
6441 if (ecc->correct || ecc->calculate) {
6442 ecc->calc_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
6443 ecc->code_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
6444 if (!ecc->calc_buf || !ecc->code_buf) {
6445 ret = -ENOMEM;
6451 if (!ecc->read_oob_raw)
6452 ecc->read_oob_raw = ecc->read_oob;
6453 if (!ecc->write_oob_raw)
6454 ecc->write_oob_raw = ecc->write_oob;
6456 /* Propagate ECC info to the generic NAND and MTD layers */
6457 mtd->ecc_strength = ecc->strength;
6458 if (!base->ecc.ctx.conf.strength)
6459 base->ecc.ctx.conf.strength = ecc->strength;
6460 mtd->ecc_step_size = ecc->size;
6461 if (!base->ecc.ctx.conf.step_size)
6462 base->ecc.ctx.conf.step_size = ecc->size;
6465 * Set the number of read / write steps for one page depending on ECC
6468 if (!ecc->steps)
6469 ecc->steps = mtd->writesize / ecc->size;
6470 if (!base->ecc.ctx.nsteps)
6471 base->ecc.ctx.nsteps = ecc->steps;
6472 if (ecc->steps * ecc->size != mtd->writesize) {
6473 WARN(1, "Invalid ECC parameters\n");
6474 ret = -EINVAL;
6478 if (!ecc->total) {
6479 ecc->total = ecc->steps * ecc->bytes;
6480 chip->base.ecc.ctx.total = ecc->total;
6483 if (ecc->total > mtd->oobsize) {
6484 WARN(1, "Total number of ECC bytes exceeded oobsize\n");
6485 ret = -EINVAL;
6497 mtd->oobavail = ret;
6499 /* ECC sanity check: warn if it's too weak */
6500 if (!nand_ecc_is_strong_enough(&chip->base))
6501 pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n",
6502 mtd->name, chip->ecc.strength, chip->ecc.size,
6503 nanddev_get_ecc_requirements(&chip->base)->strength,
6504 nanddev_get_ecc_requirements(&chip->base)->step_size);
6506 /* Allow subpage writes up to ecc.steps. Not possible for MLC flash */
6507 if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) {
6508 switch (ecc->steps) {
6510 mtd->subpage_sft = 1;
6515 mtd->subpage_sft = 2;
6519 chip->subpagesize = mtd->writesize >> mtd->subpage_sft;
6522 chip->pagecache.page = -1;
6524 /* Large page NAND with SOFT_ECC should support subpage reads */
6525 switch (ecc->engine_type) {
6527 if (chip->page_shift > 9)
6528 chip->options |= NAND_SUBPAGE_READ;
6535 ret = nanddev_init(&chip->base, &rawnand_ops, mtd->owner);
6540 if (chip->options & NAND_ROM)
6541 mtd->flags = MTD_CAP_ROM;
6544 mtd->_erase = nand_erase;
6545 mtd->_point = NULL;
6546 mtd->_unpoint = NULL;
6547 mtd->_panic_write = panic_nand_write;
6548 mtd->_read_oob = nand_read_oob;
6549 mtd->_write_oob = nand_write_oob;
6550 mtd->_sync = nand_sync;
6551 mtd->_lock = nand_lock;
6552 mtd->_unlock = nand_unlock;
6553 mtd->_suspend = nand_suspend;
6554 mtd->_resume = nand_resume;
6555 mtd->_reboot = nand_shutdown;
6556 mtd->_block_isreserved = nand_block_isreserved;
6557 mtd->_block_isbad = nand_block_isbad;
6558 mtd->_block_markbad = nand_block_markbad;
6559 mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks;
6566 if (!mtd->bitflip_threshold)
6567 mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4);
6575 for (i = 0; i < nanddev_ntargets(&chip->base); i++) {
6584 * Look for secure regions in the NAND chip. These regions are supposed
6594 if (chip->options & NAND_SKIP_BBTSCAN)
6605 kfree(chip->secure_regions);
6608 kfree(chip->best_interface_config);
6611 nanddev_cleanup(&chip->base);
6617 kfree(chip->data_buf);
6618 kfree(ecc->code_buf);
6619 kfree(ecc->calc_buf);
6626 if (chip->controller->ops && chip->controller->ops->attach_chip)
6627 return chip->controller->ops->attach_chip(chip);
6634 if (chip->controller->ops && chip->controller->ops->detach_chip)
6635 chip->controller->ops->detach_chip(chip);
6639 * nand_scan_with_ids - [NAND Interface] Scan for the NAND device
6640 * @chip: NAND chip object
6654 return -EINVAL;
6680 * nand_cleanup - [NAND Interface] Free resources held by the NAND device
6681 * @chip: NAND chip object
6685 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT) {
6686 if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING)
6688 else if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
6692 nanddev_cleanup(&chip->base);
6695 kfree(chip->secure_regions);
6698 kfree(chip->bbt);
6699 kfree(chip->data_buf);
6700 kfree(chip->ecc.code_buf);
6701 kfree(chip->ecc.calc_buf);
6704 if (chip->badblock_pattern && chip->badblock_pattern->options
6706 kfree(chip->badblock_pattern);
6709 kfree(chip->best_interface_config);
6726 MODULE_DESCRIPTION("Generic NAND flash driver code");