xref: /linux/drivers/mtd/nand/raw/marvell_nand.c (revision 70ab9ec9166db90ab8980aff4f7083512ecddd1f)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Marvell NAND flash controller driver
4  *
5  * Copyright (C) 2017 Marvell
6  * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7  *
8  *
9  * This NAND controller driver handles two versions of the hardware,
10  * one is called NFCv1 and is available on PXA SoCs and the other is
11  * called NFCv2 and is available on Armada SoCs.
12  *
13  * The main visible difference is that NFCv1 only has Hamming ECC
14  * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15  * is not used with NFCv2.
16  *
17  * The ECC layouts are depicted in details in Marvell AN-379, but here
18  * is a brief description.
19  *
20  * When using Hamming, the data is split in 512B chunks (either 1, 2
21  * or 4) and each chunk will have its own ECC "digest" of 6B at the
22  * beginning of the OOB area and eventually the remaining free OOB
23  * bytes (also called "spare" bytes in the driver). This engine
24  * corrects up to 1 bit per chunk and detects reliably an error if
25  * there are at most 2 bitflips. Here is the page layout used by the
26  * controller when Hamming is chosen:
27  *
28  * +-------------------------------------------------------------+
29  * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30  * +-------------------------------------------------------------+
31  *
32  * When using the BCH engine, there are N identical (data + free OOB +
33  * ECC) sections and potentially an extra one to deal with
34  * configurations where the chosen (data + free OOB + ECC) sizes do
35  * not align with the page (data + OOB) size. ECC bytes are always
36  * 30B per ECC chunk. Here is the page layout used by the controller
37  * when BCH is chosen:
38  *
39  * +-----------------------------------------
40  * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41  * +-----------------------------------------
42  *
43  *      -------------------------------------------
44  *       ... | Data N | Free OOB bytes N | ECC N |
45  *      -------------------------------------------
46  *
47  *           --------------------------------------------+
48  *            Last Data | Last Free OOB bytes | Last ECC |
49  *           --------------------------------------------+
50  *
51  * In both cases, the layout seen by the user is always: all data
52  * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53  * ECC bytes are 30B long and are padded with 0xFF to align on 32
54  * bytes.
55  *
56  * The controller has certain limitations that are handled by the
57  * driver:
58  *   - It can only read 2k at a time. To overcome this limitation, the
59  *     driver issues data cycles on the bus, without issuing new
60  *     CMD + ADDR cycles. The Marvell term is "naked" operations.
61  *   - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62  *     bits. What can be tuned is the ECC block size as long as it
63  *     stays between 512B and 2kiB. It's usually chosen based on the
64  *     chip ECC requirements. For instance, using 2kiB ECC chunks
65  *     provides 4b/512B correctability.
66  *   - The controller will always treat data bytes, free OOB bytes
67  *     and ECC bytes in that order, no matter what the real layout is
68  *     (which is usually all data then all OOB bytes). The
69  *     marvell_nfc_layouts array below contains the currently
70  *     supported layouts.
71  *   - Because of these weird layouts, the Bad Block Markers can be
72  *     located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73  *     option must be set to prevent scanning/writing bad block
74  *     markers.
75  */
76 
77 #include <linux/module.h>
78 #include <linux/clk.h>
79 #include <linux/mtd/rawnand.h>
80 #include <linux/of.h>
81 #include <linux/iopoll.h>
82 #include <linux/interrupt.h>
83 #include <linux/platform_device.h>
84 #include <linux/slab.h>
85 #include <linux/mfd/syscon.h>
86 #include <linux/regmap.h>
87 #include <asm/unaligned.h>
88 
89 #include <linux/dmaengine.h>
90 #include <linux/dma-mapping.h>
91 #include <linux/dma/pxa-dma.h>
92 #include <linux/platform_data/mtd-nand-pxa3xx.h>
93 
94 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
95 #define FIFO_DEPTH		8
96 #define FIFO_REP(x)		(x / sizeof(u32))
97 #define BCH_SEQ_READS		(32 / FIFO_DEPTH)
98 /* NFC does not support transfers of larger chunks at a time */
99 #define MAX_CHUNK_SIZE		2112
100 /* NFCv1 cannot read more that 7 bytes of ID */
101 #define NFCV1_READID_LEN	7
102 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
103 #define POLL_PERIOD		0
104 #define POLL_TIMEOUT		100000
105 /* Interrupt maximum wait period in ms */
106 #define IRQ_TIMEOUT		1000
107 /* Latency in clock cycles between SoC pins and NFC logic */
108 #define MIN_RD_DEL_CNT		3
109 /* Maximum number of contiguous address cycles */
110 #define MAX_ADDRESS_CYC_NFCV1	5
111 #define MAX_ADDRESS_CYC_NFCV2	7
112 /* System control registers/bits to enable the NAND controller on some SoCs */
113 #define GENCONF_SOC_DEVICE_MUX	0x208
114 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
115 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
116 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
117 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
118 #define GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN BIT(27)
119 #define GENCONF_CLK_GATING_CTRL	0x220
120 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
121 #define GENCONF_ND_CLK_CTRL	0x700
122 #define GENCONF_ND_CLK_CTRL_EN	BIT(0)
123 
124 /* NAND controller data flash control register */
125 #define NDCR			0x00
126 #define NDCR_ALL_INT		GENMASK(11, 0)
127 #define NDCR_CS1_CMDDM		BIT(7)
128 #define NDCR_CS0_CMDDM		BIT(8)
129 #define NDCR_RDYM		BIT(11)
130 #define NDCR_ND_ARB_EN		BIT(12)
131 #define NDCR_RA_START		BIT(15)
132 #define NDCR_RD_ID_CNT(x)	(min_t(unsigned int, x, 0x7) << 16)
133 #define NDCR_PAGE_SZ(x)		(x >= 2048 ? BIT(24) : 0)
134 #define NDCR_DWIDTH_M		BIT(26)
135 #define NDCR_DWIDTH_C		BIT(27)
136 #define NDCR_ND_RUN		BIT(28)
137 #define NDCR_DMA_EN		BIT(29)
138 #define NDCR_ECC_EN		BIT(30)
139 #define NDCR_SPARE_EN		BIT(31)
140 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
141 				    NDCR_DWIDTH_M | NDCR_DWIDTH_C))
142 
143 /* NAND interface timing parameter 0 register */
144 #define NDTR0			0x04
145 #define NDTR0_TRP(x)		((min_t(unsigned int, x, 0xF) & 0x7) << 0)
146 #define NDTR0_TRH(x)		(min_t(unsigned int, x, 0x7) << 3)
147 #define NDTR0_ETRP(x)		((min_t(unsigned int, x, 0xF) & 0x8) << 3)
148 #define NDTR0_SEL_NRE_EDGE	BIT(7)
149 #define NDTR0_TWP(x)		(min_t(unsigned int, x, 0x7) << 8)
150 #define NDTR0_TWH(x)		(min_t(unsigned int, x, 0x7) << 11)
151 #define NDTR0_TCS(x)		(min_t(unsigned int, x, 0x7) << 16)
152 #define NDTR0_TCH(x)		(min_t(unsigned int, x, 0x7) << 19)
153 #define NDTR0_RD_CNT_DEL(x)	(min_t(unsigned int, x, 0xF) << 22)
154 #define NDTR0_SELCNTR		BIT(26)
155 #define NDTR0_TADL(x)		(min_t(unsigned int, x, 0x1F) << 27)
156 
157 /* NAND interface timing parameter 1 register */
158 #define NDTR1			0x0C
159 #define NDTR1_TAR(x)		(min_t(unsigned int, x, 0xF) << 0)
160 #define NDTR1_TWHR(x)		(min_t(unsigned int, x, 0xF) << 4)
161 #define NDTR1_TRHW(x)		(min_t(unsigned int, x / 16, 0x3) << 8)
162 #define NDTR1_PRESCALE		BIT(14)
163 #define NDTR1_WAIT_MODE		BIT(15)
164 #define NDTR1_TR(x)		(min_t(unsigned int, x, 0xFFFF) << 16)
165 
166 /* NAND controller status register */
167 #define NDSR			0x14
168 #define NDSR_WRCMDREQ		BIT(0)
169 #define NDSR_RDDREQ		BIT(1)
170 #define NDSR_WRDREQ		BIT(2)
171 #define NDSR_CORERR		BIT(3)
172 #define NDSR_UNCERR		BIT(4)
173 #define NDSR_CMDD(cs)		BIT(8 - cs)
174 #define NDSR_RDY(rb)		BIT(11 + rb)
175 #define NDSR_ERRCNT(x)		((x >> 16) & 0x1F)
176 
177 /* NAND ECC control register */
178 #define NDECCCTRL		0x28
179 #define NDECCCTRL_BCH_EN	BIT(0)
180 
181 /* NAND controller data buffer register */
182 #define NDDB			0x40
183 
184 /* NAND controller command buffer 0 register */
185 #define NDCB0			0x48
186 #define NDCB0_CMD1(x)		((x & 0xFF) << 0)
187 #define NDCB0_CMD2(x)		((x & 0xFF) << 8)
188 #define NDCB0_ADDR_CYC(x)	((x & 0x7) << 16)
189 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
190 #define NDCB0_DBC		BIT(19)
191 #define NDCB0_CMD_TYPE(x)	((x & 0x7) << 21)
192 #define NDCB0_CSEL		BIT(24)
193 #define NDCB0_RDY_BYP		BIT(27)
194 #define NDCB0_LEN_OVRD		BIT(28)
195 #define NDCB0_CMD_XTYPE(x)	((x & 0x7) << 29)
196 
197 /* NAND controller command buffer 1 register */
198 #define NDCB1			0x4C
199 #define NDCB1_COLS(x)		((x & 0xFFFF) << 0)
200 #define NDCB1_ADDRS_PAGE(x)	(x << 16)
201 
202 /* NAND controller command buffer 2 register */
203 #define NDCB2			0x50
204 #define NDCB2_ADDR5_PAGE(x)	(((x >> 16) & 0xFF) << 0)
205 #define NDCB2_ADDR5_CYC(x)	((x & 0xFF) << 0)
206 
207 /* NAND controller command buffer 3 register */
208 #define NDCB3			0x54
209 #define NDCB3_ADDR6_CYC(x)	((x & 0xFF) << 16)
210 #define NDCB3_ADDR7_CYC(x)	((x & 0xFF) << 24)
211 
212 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
213 #define TYPE_READ		0
214 #define TYPE_WRITE		1
215 #define TYPE_ERASE		2
216 #define TYPE_READ_ID		3
217 #define TYPE_STATUS		4
218 #define TYPE_RESET		5
219 #define TYPE_NAKED_CMD		6
220 #define TYPE_NAKED_ADDR		7
221 #define TYPE_MASK		7
222 #define XTYPE_MONOLITHIC_RW	0
223 #define XTYPE_LAST_NAKED_RW	1
224 #define XTYPE_FINAL_COMMAND	3
225 #define XTYPE_READ		4
226 #define XTYPE_WRITE_DISPATCH	4
227 #define XTYPE_NAKED_RW		5
228 #define XTYPE_COMMAND_DISPATCH	6
229 #define XTYPE_MASK		7
230 
231 /**
232  * struct marvell_hw_ecc_layout - layout of Marvell ECC
233  *
234  * Marvell ECC engine works differently than the others, in order to limit the
235  * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
236  * per subpage, and depending on a the desired strength needed by the NAND chip,
237  * a particular layout mixing data/spare/ecc is defined, with a possible last
238  * chunk smaller that the others.
239  *
240  * @writesize:		Full page size on which the layout applies
241  * @chunk:		Desired ECC chunk size on which the layout applies
242  * @strength:		Desired ECC strength (per chunk size bytes) on which the
243  *			layout applies
244  * @nchunks:		Total number of chunks
245  * @full_chunk_cnt:	Number of full-sized chunks, which is the number of
246  *			repetitions of the pattern:
247  *			(data_bytes + spare_bytes + ecc_bytes).
248  * @data_bytes:		Number of data bytes per chunk
249  * @spare_bytes:	Number of spare bytes per chunk
250  * @ecc_bytes:		Number of ecc bytes per chunk
251  * @last_data_bytes:	Number of data bytes in the last chunk
252  * @last_spare_bytes:	Number of spare bytes in the last chunk
253  * @last_ecc_bytes:	Number of ecc bytes in the last chunk
254  */
255 struct marvell_hw_ecc_layout {
256 	/* Constraints */
257 	int writesize;
258 	int chunk;
259 	int strength;
260 	/* Corresponding layout */
261 	int nchunks;
262 	int full_chunk_cnt;
263 	int data_bytes;
264 	int spare_bytes;
265 	int ecc_bytes;
266 	int last_data_bytes;
267 	int last_spare_bytes;
268 	int last_ecc_bytes;
269 };
270 
271 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)	\
272 	{								\
273 		.writesize = ws,					\
274 		.chunk = dc,						\
275 		.strength = ds,						\
276 		.nchunks = nc,						\
277 		.full_chunk_cnt = fcc,					\
278 		.data_bytes = db,					\
279 		.spare_bytes = sb,					\
280 		.ecc_bytes = eb,					\
281 		.last_data_bytes = ldb,					\
282 		.last_spare_bytes = lsb,				\
283 		.last_ecc_bytes = leb,					\
284 	}
285 
286 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
287 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
288 	MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
289 	MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
290 	MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
291 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,32, 30),
292 	MARVELL_LAYOUT( 2048,   512,  8,  2,  1, 1024,  0, 30,1024,64, 30),
293 	MARVELL_LAYOUT( 2048,   512,  12, 3,  2, 704,   0, 30,640,  0, 30),
294 	MARVELL_LAYOUT( 2048,   512,  16, 5,  4, 512,   0, 30,  0, 32, 30),
295 	MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
296 	MARVELL_LAYOUT( 4096,   512,  8,  5,  4, 1024,  0, 30,  0, 64, 30),
297 	MARVELL_LAYOUT( 4096,   512,  12, 6,  5, 704,   0, 30,576, 32, 30),
298 	MARVELL_LAYOUT( 4096,   512,  16, 9,  8, 512,   0, 30,  0, 32, 30),
299 	MARVELL_LAYOUT( 8192,   512,  4,  4,  4, 2048,  0, 30,  0,  0,  0),
300 	MARVELL_LAYOUT( 8192,   512,  8,  9,  8, 1024,  0, 30,  0, 160, 30),
301 	MARVELL_LAYOUT( 8192,   512,  12, 12, 11, 704,  0, 30,448,  64, 30),
302 	MARVELL_LAYOUT( 8192,   512,  16, 17, 16, 512,  0, 30,  0,  32, 30),
303 };
304 
305 /**
306  * struct marvell_nand_chip_sel - CS line description
307  *
308  * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
309  * is made by a field in NDCB0 register, and in another field in NDCB2 register.
310  * The datasheet describes the logic with an error: ADDR5 field is once
311  * declared at the beginning of NDCB2, and another time at its end. Because the
312  * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
313  * to use the last bit of this field instead of the first ones.
314  *
315  * @cs:			Wanted CE lane.
316  * @ndcb0_csel:		Value of the NDCB0 register with or without the flag
317  *			selecting the wanted CE lane. This is set once when
318  *			the Device Tree is probed.
319  * @rb:			Ready/Busy pin for the flash chip
320  */
321 struct marvell_nand_chip_sel {
322 	unsigned int cs;
323 	u32 ndcb0_csel;
324 	unsigned int rb;
325 };
326 
327 /**
328  * struct marvell_nand_chip - stores NAND chip device related information
329  *
330  * @chip:		Base NAND chip structure
331  * @node:		Used to store NAND chips into a list
332  * @layout:		NAND layout when using hardware ECC
333  * @ndcr:		Controller register value for this NAND chip
334  * @ndtr0:		Timing registers 0 value for this NAND chip
335  * @ndtr1:		Timing registers 1 value for this NAND chip
336  * @addr_cyc:		Amount of cycles needed to pass column address
337  * @selected_die:	Current active CS
338  * @nsels:		Number of CS lines required by the NAND chip
339  * @sels:		Array of CS lines descriptions
340  */
341 struct marvell_nand_chip {
342 	struct nand_chip chip;
343 	struct list_head node;
344 	const struct marvell_hw_ecc_layout *layout;
345 	u32 ndcr;
346 	u32 ndtr0;
347 	u32 ndtr1;
348 	int addr_cyc;
349 	int selected_die;
350 	unsigned int nsels;
351 	struct marvell_nand_chip_sel sels[] __counted_by(nsels);
352 };
353 
354 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
355 {
356 	return container_of(chip, struct marvell_nand_chip, chip);
357 }
358 
359 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
360 							*nand)
361 {
362 	return &nand->sels[nand->selected_die];
363 }
364 
365 /**
366  * struct marvell_nfc_caps - NAND controller capabilities for distinction
367  *                           between compatible strings
368  *
369  * @max_cs_nb:		Number of Chip Select lines available
370  * @max_rb_nb:		Number of Ready/Busy lines available
371  * @need_system_controller: Indicates if the SoC needs to have access to the
372  *                      system controller (ie. to enable the NAND controller)
373  * @legacy_of_bindings:	Indicates if DT parsing must be done using the old
374  *			fashion way
375  * @is_nfcv2:		NFCv2 has numerous enhancements compared to NFCv1, ie.
376  *			BCH error detection and correction algorithm,
377  *			NDCB3 register has been added
378  * @use_dma:		Use dma for data transfers
379  * @max_mode_number:	Maximum timing mode supported by the controller
380  */
381 struct marvell_nfc_caps {
382 	unsigned int max_cs_nb;
383 	unsigned int max_rb_nb;
384 	bool need_system_controller;
385 	bool legacy_of_bindings;
386 	bool is_nfcv2;
387 	bool use_dma;
388 	unsigned int max_mode_number;
389 };
390 
391 /**
392  * struct marvell_nfc - stores Marvell NAND controller information
393  *
394  * @controller:		Base controller structure
395  * @dev:		Parent device (used to print error messages)
396  * @regs:		NAND controller registers
397  * @core_clk:		Core clock
398  * @reg_clk:		Registers clock
399  * @complete:		Completion object to wait for NAND controller events
400  * @assigned_cs:	Bitmask describing already assigned CS lines
401  * @chips:		List containing all the NAND chips attached to
402  *			this NAND controller
403  * @selected_chip:	Currently selected target chip
404  * @caps:		NAND controller capabilities for each compatible string
405  * @use_dma:		Whetner DMA is used
406  * @dma_chan:		DMA channel (NFCv1 only)
407  * @dma_buf:		32-bit aligned buffer for DMA transfers (NFCv1 only)
408  */
409 struct marvell_nfc {
410 	struct nand_controller controller;
411 	struct device *dev;
412 	void __iomem *regs;
413 	struct clk *core_clk;
414 	struct clk *reg_clk;
415 	struct completion complete;
416 	unsigned long assigned_cs;
417 	struct list_head chips;
418 	struct nand_chip *selected_chip;
419 	const struct marvell_nfc_caps *caps;
420 
421 	/* DMA (NFCv1 only) */
422 	bool use_dma;
423 	struct dma_chan *dma_chan;
424 	u8 *dma_buf;
425 };
426 
427 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
428 {
429 	return container_of(ctrl, struct marvell_nfc, controller);
430 }
431 
432 /**
433  * struct marvell_nfc_timings - NAND controller timings expressed in NAND
434  *                              Controller clock cycles
435  *
436  * @tRP:		ND_nRE pulse width
437  * @tRH:		ND_nRE high duration
438  * @tWP:		ND_nWE pulse time
439  * @tWH:		ND_nWE high duration
440  * @tCS:		Enable signal setup time
441  * @tCH:		Enable signal hold time
442  * @tADL:		Address to write data delay
443  * @tAR:		ND_ALE low to ND_nRE low delay
444  * @tWHR:		ND_nWE high to ND_nRE low for status read
445  * @tRHW:		ND_nRE high duration, read to write delay
446  * @tR:			ND_nWE high to ND_nRE low for read
447  */
448 struct marvell_nfc_timings {
449 	/* NDTR0 fields */
450 	unsigned int tRP;
451 	unsigned int tRH;
452 	unsigned int tWP;
453 	unsigned int tWH;
454 	unsigned int tCS;
455 	unsigned int tCH;
456 	unsigned int tADL;
457 	/* NDTR1 fields */
458 	unsigned int tAR;
459 	unsigned int tWHR;
460 	unsigned int tRHW;
461 	unsigned int tR;
462 };
463 
464 /**
465  * TO_CYCLES() - Derives a duration in numbers of clock cycles.
466  *
467  * @ps: Duration in pico-seconds
468  * @period_ns:  Clock period in nano-seconds
469  *
470  * Convert the duration in nano-seconds, then divide by the period and
471  * return the number of clock periods.
472  */
473 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
474 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
475 						     period_ns))
476 
477 /**
478  * struct marvell_nfc_op - filled during the parsing of the ->exec_op()
479  *                         subop subset of instructions.
480  *
481  * @ndcb:		Array of values written to NDCBx registers
482  * @cle_ale_delay_ns:	Optional delay after the last CMD or ADDR cycle
483  * @rdy_timeout_ms:	Timeout for waits on Ready/Busy pin
484  * @rdy_delay_ns:	Optional delay after waiting for the RB pin
485  * @data_delay_ns:	Optional delay after the data xfer
486  * @data_instr_idx:	Index of the data instruction in the subop
487  * @data_instr:		Pointer to the data instruction in the subop
488  */
489 struct marvell_nfc_op {
490 	u32 ndcb[4];
491 	unsigned int cle_ale_delay_ns;
492 	unsigned int rdy_timeout_ms;
493 	unsigned int rdy_delay_ns;
494 	unsigned int data_delay_ns;
495 	unsigned int data_instr_idx;
496 	const struct nand_op_instr *data_instr;
497 };
498 
499 /*
500  * Internal helper to conditionnally apply a delay (from the above structure,
501  * most of the time).
502  */
503 static void cond_delay(unsigned int ns)
504 {
505 	if (!ns)
506 		return;
507 
508 	if (ns < 10000)
509 		ndelay(ns);
510 	else
511 		udelay(DIV_ROUND_UP(ns, 1000));
512 }
513 
514 /*
515  * The controller has many flags that could generate interrupts, most of them
516  * are disabled and polling is used. For the very slow signals, using interrupts
517  * may relax the CPU charge.
518  */
519 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
520 {
521 	u32 reg;
522 
523 	/* Writing 1 disables the interrupt */
524 	reg = readl_relaxed(nfc->regs + NDCR);
525 	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
526 }
527 
528 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
529 {
530 	u32 reg;
531 
532 	/* Writing 0 enables the interrupt */
533 	reg = readl_relaxed(nfc->regs + NDCR);
534 	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
535 }
536 
537 static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
538 {
539 	u32 reg;
540 
541 	reg = readl_relaxed(nfc->regs + NDSR);
542 	writel_relaxed(int_mask, nfc->regs + NDSR);
543 
544 	return reg & int_mask;
545 }
546 
547 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
548 					  bool force_8bit)
549 {
550 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
551 	u32 ndcr;
552 
553 	/*
554 	 * Callers of this function do not verify if the NAND is using a 16-bit
555 	 * an 8-bit bus for normal operations, so we need to take care of that
556 	 * here by leaving the configuration unchanged if the NAND does not have
557 	 * the NAND_BUSWIDTH_16 flag set.
558 	 */
559 	if (!(chip->options & NAND_BUSWIDTH_16))
560 		return;
561 
562 	ndcr = readl_relaxed(nfc->regs + NDCR);
563 
564 	if (force_8bit)
565 		ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
566 	else
567 		ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
568 
569 	writel_relaxed(ndcr, nfc->regs + NDCR);
570 }
571 
572 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
573 {
574 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
575 	u32 val;
576 	int ret;
577 
578 	/*
579 	 * The command is being processed, wait for the ND_RUN bit to be
580 	 * cleared by the NFC. If not, we must clear it by hand.
581 	 */
582 	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
583 					 (val & NDCR_ND_RUN) == 0,
584 					 POLL_PERIOD, POLL_TIMEOUT);
585 	if (ret) {
586 		dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
587 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
588 			       nfc->regs + NDCR);
589 		return ret;
590 	}
591 
592 	return 0;
593 }
594 
595 /*
596  * Any time a command has to be sent to the controller, the following sequence
597  * has to be followed:
598  * - call marvell_nfc_prepare_cmd()
599  *      -> activate the ND_RUN bit that will kind of 'start a job'
600  *      -> wait the signal indicating the NFC is waiting for a command
601  * - send the command (cmd and address cycles)
602  * - enventually send or receive the data
603  * - call marvell_nfc_end_cmd() with the corresponding flag
604  *      -> wait the flag to be triggered or cancel the job with a timeout
605  *
606  * The following helpers are here to factorize the code a bit so that
607  * specialized functions responsible for executing the actual NAND
608  * operations do not have to replicate the same code blocks.
609  */
610 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
611 {
612 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
613 	u32 ndcr, val;
614 	int ret;
615 
616 	/* Poll ND_RUN and clear NDSR before issuing any command */
617 	ret = marvell_nfc_wait_ndrun(chip);
618 	if (ret) {
619 		dev_err(nfc->dev, "Last operation did not succeed\n");
620 		return ret;
621 	}
622 
623 	ndcr = readl_relaxed(nfc->regs + NDCR);
624 	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
625 
626 	/* Assert ND_RUN bit and wait the NFC to be ready */
627 	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
628 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
629 					 val & NDSR_WRCMDREQ,
630 					 POLL_PERIOD, POLL_TIMEOUT);
631 	if (ret) {
632 		dev_err(nfc->dev, "Timeout on WRCMDRE\n");
633 		return -ETIMEDOUT;
634 	}
635 
636 	/* Command may be written, clear WRCMDREQ status bit */
637 	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
638 
639 	return 0;
640 }
641 
642 static void marvell_nfc_send_cmd(struct nand_chip *chip,
643 				 struct marvell_nfc_op *nfc_op)
644 {
645 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
646 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
647 
648 	dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
649 		"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
650 		(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
651 		nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
652 
653 	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
654 		       nfc->regs + NDCB0);
655 	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
656 	writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
657 
658 	/*
659 	 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
660 	 * fields are used (only available on NFCv2).
661 	 */
662 	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
663 	    NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
664 		if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
665 			writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
666 	}
667 }
668 
669 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
670 			       const char *label)
671 {
672 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
673 	u32 val;
674 	int ret;
675 
676 	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
677 					 val & flag,
678 					 POLL_PERIOD, POLL_TIMEOUT);
679 
680 	if (ret) {
681 		dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
682 			label, val);
683 		if (nfc->dma_chan)
684 			dmaengine_terminate_all(nfc->dma_chan);
685 		return ret;
686 	}
687 
688 	/*
689 	 * DMA function uses this helper to poll on CMDD bits without wanting
690 	 * them to be cleared.
691 	 */
692 	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
693 		return 0;
694 
695 	writel_relaxed(flag, nfc->regs + NDSR);
696 
697 	return 0;
698 }
699 
700 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
701 {
702 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
703 	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
704 
705 	return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
706 }
707 
708 static int marvell_nfc_poll_status(struct marvell_nfc *nfc, u32 mask,
709 				   u32 expected_val, unsigned long timeout_ms)
710 {
711 	unsigned long limit;
712 	u32 st;
713 
714 	limit = jiffies + msecs_to_jiffies(timeout_ms);
715 	do {
716 		st = readl_relaxed(nfc->regs + NDSR);
717 		if (st & NDSR_RDY(1))
718 			st |= NDSR_RDY(0);
719 
720 		if ((st & mask) == expected_val)
721 			return 0;
722 
723 		cpu_relax();
724 	} while (time_after(limit, jiffies));
725 
726 	return -ETIMEDOUT;
727 }
728 
729 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
730 {
731 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
732 	struct mtd_info *mtd = nand_to_mtd(chip);
733 	u32 pending;
734 	int ret;
735 
736 	/* Timeout is expressed in ms */
737 	if (!timeout_ms)
738 		timeout_ms = IRQ_TIMEOUT;
739 
740 	if (mtd->oops_panic_write) {
741 		ret = marvell_nfc_poll_status(nfc, NDSR_RDY(0),
742 					      NDSR_RDY(0),
743 					      timeout_ms);
744 	} else {
745 		init_completion(&nfc->complete);
746 
747 		marvell_nfc_enable_int(nfc, NDCR_RDYM);
748 		ret = wait_for_completion_timeout(&nfc->complete,
749 						  msecs_to_jiffies(timeout_ms));
750 		marvell_nfc_disable_int(nfc, NDCR_RDYM);
751 	}
752 	pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
753 
754 	/*
755 	 * In case the interrupt was not served in the required time frame,
756 	 * check if the ISR was not served or if something went actually wrong.
757 	 */
758 	if (!ret && !pending) {
759 		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
760 		return -ETIMEDOUT;
761 	}
762 
763 	return 0;
764 }
765 
766 static void marvell_nfc_select_target(struct nand_chip *chip,
767 				      unsigned int die_nr)
768 {
769 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
770 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
771 	u32 ndcr_generic;
772 
773 	/*
774 	 * Reset the NDCR register to a clean state for this particular chip,
775 	 * also clear ND_RUN bit.
776 	 */
777 	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
778 		       NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
779 	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
780 
781 	/* Also reset the interrupt status register */
782 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
783 
784 	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
785 		return;
786 
787 	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
788 	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
789 
790 	nfc->selected_chip = chip;
791 	marvell_nand->selected_die = die_nr;
792 }
793 
794 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
795 {
796 	struct marvell_nfc *nfc = dev_id;
797 	u32 st = readl_relaxed(nfc->regs + NDSR);
798 	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
799 
800 	/*
801 	 * RDY interrupt mask is one bit in NDCR while there are two status
802 	 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
803 	 */
804 	if (st & NDSR_RDY(1))
805 		st |= NDSR_RDY(0);
806 
807 	if (!(st & ien))
808 		return IRQ_NONE;
809 
810 	marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
811 
812 	if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
813 		complete(&nfc->complete);
814 
815 	return IRQ_HANDLED;
816 }
817 
818 /* HW ECC related functions */
819 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
820 {
821 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
822 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
823 
824 	if (!(ndcr & NDCR_ECC_EN)) {
825 		writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
826 
827 		/*
828 		 * When enabling BCH, set threshold to 0 to always know the
829 		 * number of corrected bitflips.
830 		 */
831 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
832 			writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
833 	}
834 }
835 
836 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
837 {
838 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
839 	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
840 
841 	if (ndcr & NDCR_ECC_EN) {
842 		writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
843 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
844 			writel_relaxed(0, nfc->regs + NDECCCTRL);
845 	}
846 }
847 
848 /* DMA related helpers */
849 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
850 {
851 	u32 reg;
852 
853 	reg = readl_relaxed(nfc->regs + NDCR);
854 	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
855 }
856 
857 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
858 {
859 	u32 reg;
860 
861 	reg = readl_relaxed(nfc->regs + NDCR);
862 	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
863 }
864 
865 /* Read/write PIO/DMA accessors */
866 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
867 				     enum dma_data_direction direction,
868 				     unsigned int len)
869 {
870 	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
871 	struct dma_async_tx_descriptor *tx;
872 	struct scatterlist sg;
873 	dma_cookie_t cookie;
874 	int ret;
875 
876 	marvell_nfc_enable_dma(nfc);
877 	/* Prepare the DMA transfer */
878 	sg_init_one(&sg, nfc->dma_buf, dma_len);
879 	ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
880 	if (!ret) {
881 		dev_err(nfc->dev, "Could not map DMA S/G list\n");
882 		return -ENXIO;
883 	}
884 
885 	tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
886 				     direction == DMA_FROM_DEVICE ?
887 				     DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
888 				     DMA_PREP_INTERRUPT);
889 	if (!tx) {
890 		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
891 		dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
892 		return -ENXIO;
893 	}
894 
895 	/* Do the task and wait for it to finish */
896 	cookie = dmaengine_submit(tx);
897 	ret = dma_submit_error(cookie);
898 	if (ret)
899 		return -EIO;
900 
901 	dma_async_issue_pending(nfc->dma_chan);
902 	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
903 	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
904 	marvell_nfc_disable_dma(nfc);
905 	if (ret) {
906 		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
907 			dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
908 		dmaengine_terminate_all(nfc->dma_chan);
909 		return -ETIMEDOUT;
910 	}
911 
912 	return 0;
913 }
914 
915 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
916 					unsigned int len)
917 {
918 	unsigned int last_len = len % FIFO_DEPTH;
919 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
920 	int i;
921 
922 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
923 		ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
924 
925 	if (last_len) {
926 		u8 tmp_buf[FIFO_DEPTH];
927 
928 		ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
929 		memcpy(in + last_full_offset, tmp_buf, last_len);
930 	}
931 
932 	return 0;
933 }
934 
935 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
936 					 unsigned int len)
937 {
938 	unsigned int last_len = len % FIFO_DEPTH;
939 	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
940 	int i;
941 
942 	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
943 		iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
944 
945 	if (last_len) {
946 		u8 tmp_buf[FIFO_DEPTH];
947 
948 		memcpy(tmp_buf, out + last_full_offset, last_len);
949 		iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
950 	}
951 
952 	return 0;
953 }
954 
955 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
956 					  u8 *data, int data_len,
957 					  u8 *spare, int spare_len,
958 					  u8 *ecc, int ecc_len,
959 					  unsigned int *max_bitflips)
960 {
961 	struct mtd_info *mtd = nand_to_mtd(chip);
962 	int bf;
963 
964 	/*
965 	 * Blank pages (all 0xFF) that have not been written may be recognized
966 	 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
967 	 * check if the entire page (with ECC bytes) is actually blank or not.
968 	 */
969 	if (!data)
970 		data_len = 0;
971 	if (!spare)
972 		spare_len = 0;
973 	if (!ecc)
974 		ecc_len = 0;
975 
976 	bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
977 					 spare, spare_len, chip->ecc.strength);
978 	if (bf < 0) {
979 		mtd->ecc_stats.failed++;
980 		return;
981 	}
982 
983 	/* Update the stats and max_bitflips */
984 	mtd->ecc_stats.corrected += bf;
985 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
986 }
987 
988 /*
989  * Check if a chunk is correct or not according to the hardware ECC engine.
990  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
991  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
992  * value indicating that a check on the emptyness of the subpage must be
993  * performed before actually declaring the subpage as "corrupted".
994  */
995 static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip,
996 					     unsigned int *max_bitflips)
997 {
998 	struct mtd_info *mtd = nand_to_mtd(chip);
999 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1000 	int bf = 0;
1001 	u32 ndsr;
1002 
1003 	ndsr = readl_relaxed(nfc->regs + NDSR);
1004 
1005 	/* Check uncorrectable error flag */
1006 	if (ndsr & NDSR_UNCERR) {
1007 		writel_relaxed(ndsr, nfc->regs + NDSR);
1008 
1009 		/*
1010 		 * Do not increment ->ecc_stats.failed now, instead, return a
1011 		 * non-zero value to indicate that this chunk was apparently
1012 		 * bad, and it should be check to see if it empty or not. If
1013 		 * the chunk (with ECC bytes) is not declared empty, the calling
1014 		 * function must increment the failure count.
1015 		 */
1016 		return -EBADMSG;
1017 	}
1018 
1019 	/* Check correctable error flag */
1020 	if (ndsr & NDSR_CORERR) {
1021 		writel_relaxed(ndsr, nfc->regs + NDSR);
1022 
1023 		if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
1024 			bf = NDSR_ERRCNT(ndsr);
1025 		else
1026 			bf = 1;
1027 	}
1028 
1029 	/* Update the stats and max_bitflips */
1030 	mtd->ecc_stats.corrected += bf;
1031 	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
1032 
1033 	return 0;
1034 }
1035 
1036 /* Hamming read helpers */
1037 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
1038 					       u8 *data_buf, u8 *oob_buf,
1039 					       bool raw, int page)
1040 {
1041 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1042 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1043 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1044 	struct marvell_nfc_op nfc_op = {
1045 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1046 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1047 			   NDCB0_DBC |
1048 			   NDCB0_CMD1(NAND_CMD_READ0) |
1049 			   NDCB0_CMD2(NAND_CMD_READSTART),
1050 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1051 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1052 	};
1053 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1054 	int ret;
1055 
1056 	/* NFCv2 needs more information about the operation being executed */
1057 	if (nfc->caps->is_nfcv2)
1058 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1059 
1060 	ret = marvell_nfc_prepare_cmd(chip);
1061 	if (ret)
1062 		return ret;
1063 
1064 	marvell_nfc_send_cmd(chip, &nfc_op);
1065 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1066 				  "RDDREQ while draining FIFO (data/oob)");
1067 	if (ret)
1068 		return ret;
1069 
1070 	/*
1071 	 * Read the page then the OOB area. Unlike what is shown in current
1072 	 * documentation, spare bytes are protected by the ECC engine, and must
1073 	 * be at the beginning of the OOB area or running this driver on legacy
1074 	 * systems will prevent the discovery of the BBM/BBT.
1075 	 */
1076 	if (nfc->use_dma) {
1077 		marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1078 					  lt->data_bytes + oob_bytes);
1079 		memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1080 		memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1081 	} else {
1082 		marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1083 		marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1084 	}
1085 
1086 	ret = marvell_nfc_wait_cmdd(chip);
1087 	return ret;
1088 }
1089 
1090 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1091 						int oob_required, int page)
1092 {
1093 	marvell_nfc_select_target(chip, chip->cur_cs);
1094 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1095 						   true, page);
1096 }
1097 
1098 static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1099 					    int oob_required, int page)
1100 {
1101 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1102 	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1103 	int max_bitflips = 0, ret;
1104 	u8 *raw_buf;
1105 
1106 	marvell_nfc_select_target(chip, chip->cur_cs);
1107 	marvell_nfc_enable_hw_ecc(chip);
1108 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1109 					    page);
1110 	ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1111 	marvell_nfc_disable_hw_ecc(chip);
1112 
1113 	if (!ret)
1114 		return max_bitflips;
1115 
1116 	/*
1117 	 * When ECC failures are detected, check if the full page has been
1118 	 * written or not. Ignore the failure if it is actually empty.
1119 	 */
1120 	raw_buf = kmalloc(full_sz, GFP_KERNEL);
1121 	if (!raw_buf)
1122 		return -ENOMEM;
1123 
1124 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1125 					    lt->data_bytes, true, page);
1126 	marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1127 				      &max_bitflips);
1128 	kfree(raw_buf);
1129 
1130 	return max_bitflips;
1131 }
1132 
1133 /*
1134  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1135  * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1136  * also stands for ->read_oob().
1137  */
1138 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1139 {
1140 	u8 *buf = nand_get_data_buf(chip);
1141 
1142 	marvell_nfc_select_target(chip, chip->cur_cs);
1143 	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1144 						   true, page);
1145 }
1146 
1147 /* Hamming write helpers */
1148 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1149 						const u8 *data_buf,
1150 						const u8 *oob_buf, bool raw,
1151 						int page)
1152 {
1153 	const struct nand_sdr_timings *sdr =
1154 		nand_get_sdr_timings(nand_get_interface_config(chip));
1155 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1156 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1157 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1158 	struct marvell_nfc_op nfc_op = {
1159 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1160 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1161 			   NDCB0_CMD1(NAND_CMD_SEQIN) |
1162 			   NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1163 			   NDCB0_DBC,
1164 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1165 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1166 	};
1167 	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1168 	u8 status;
1169 	int ret;
1170 
1171 	/* NFCv2 needs more information about the operation being executed */
1172 	if (nfc->caps->is_nfcv2)
1173 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1174 
1175 	ret = marvell_nfc_prepare_cmd(chip);
1176 	if (ret)
1177 		return ret;
1178 
1179 	marvell_nfc_send_cmd(chip, &nfc_op);
1180 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1181 				  "WRDREQ while loading FIFO (data)");
1182 	if (ret)
1183 		return ret;
1184 
1185 	/* Write the page then the OOB area */
1186 	if (nfc->use_dma) {
1187 		memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1188 		memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1189 		marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1190 					  lt->ecc_bytes + lt->spare_bytes);
1191 	} else {
1192 		marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1193 		marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1194 	}
1195 
1196 	ret = marvell_nfc_wait_cmdd(chip);
1197 	if (ret)
1198 		return ret;
1199 
1200 	ret = marvell_nfc_wait_op(chip,
1201 				  PSEC_TO_MSEC(sdr->tPROG_max));
1202 	if (ret)
1203 		return ret;
1204 
1205 	/* Check write status on the chip side */
1206 	ret = nand_status_op(chip, &status);
1207 	if (ret)
1208 		return ret;
1209 
1210 	if (status & NAND_STATUS_FAIL)
1211 		return -EIO;
1212 
1213 	return 0;
1214 }
1215 
1216 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1217 						 const u8 *buf,
1218 						 int oob_required, int page)
1219 {
1220 	marvell_nfc_select_target(chip, chip->cur_cs);
1221 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1222 						    true, page);
1223 }
1224 
1225 static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1226 					     const u8 *buf,
1227 					     int oob_required, int page)
1228 {
1229 	int ret;
1230 
1231 	marvell_nfc_select_target(chip, chip->cur_cs);
1232 	marvell_nfc_enable_hw_ecc(chip);
1233 	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1234 						   false, page);
1235 	marvell_nfc_disable_hw_ecc(chip);
1236 
1237 	return ret;
1238 }
1239 
1240 /*
1241  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1242  * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1243  * also stands for ->write_oob().
1244  */
1245 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1246 						int page)
1247 {
1248 	struct mtd_info *mtd = nand_to_mtd(chip);
1249 	u8 *buf = nand_get_data_buf(chip);
1250 
1251 	memset(buf, 0xFF, mtd->writesize);
1252 
1253 	marvell_nfc_select_target(chip, chip->cur_cs);
1254 	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1255 						    true, page);
1256 }
1257 
1258 /* BCH read helpers */
1259 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1260 						int oob_required, int page)
1261 {
1262 	struct mtd_info *mtd = nand_to_mtd(chip);
1263 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1264 	u8 *oob = chip->oob_poi;
1265 	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1266 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1267 		lt->last_spare_bytes;
1268 	int data_len = lt->data_bytes;
1269 	int spare_len = lt->spare_bytes;
1270 	int ecc_len = lt->ecc_bytes;
1271 	int chunk;
1272 
1273 	marvell_nfc_select_target(chip, chip->cur_cs);
1274 
1275 	if (oob_required)
1276 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1277 
1278 	nand_read_page_op(chip, page, 0, NULL, 0);
1279 
1280 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1281 		/* Update last chunk length */
1282 		if (chunk >= lt->full_chunk_cnt) {
1283 			data_len = lt->last_data_bytes;
1284 			spare_len = lt->last_spare_bytes;
1285 			ecc_len = lt->last_ecc_bytes;
1286 		}
1287 
1288 		/* Read data bytes*/
1289 		nand_change_read_column_op(chip, chunk * chunk_size,
1290 					   buf + (lt->data_bytes * chunk),
1291 					   data_len, false);
1292 
1293 		/* Read spare bytes */
1294 		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1295 				  spare_len, false, false);
1296 
1297 		/* Read ECC bytes */
1298 		nand_read_data_op(chip, oob + ecc_offset +
1299 				  (ALIGN(lt->ecc_bytes, 32) * chunk),
1300 				  ecc_len, false, false);
1301 	}
1302 
1303 	return 0;
1304 }
1305 
1306 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1307 					      u8 *data, unsigned int data_len,
1308 					      u8 *spare, unsigned int spare_len,
1309 					      int page)
1310 {
1311 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1312 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1313 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1314 	int i, ret;
1315 	struct marvell_nfc_op nfc_op = {
1316 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1317 			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1318 			   NDCB0_LEN_OVRD,
1319 		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1320 		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1321 		.ndcb[3] = data_len + spare_len,
1322 	};
1323 
1324 	ret = marvell_nfc_prepare_cmd(chip);
1325 	if (ret)
1326 		return;
1327 
1328 	if (chunk == 0)
1329 		nfc_op.ndcb[0] |= NDCB0_DBC |
1330 				  NDCB0_CMD1(NAND_CMD_READ0) |
1331 				  NDCB0_CMD2(NAND_CMD_READSTART);
1332 
1333 	/*
1334 	 * Trigger the monolithic read on the first chunk, then naked read on
1335 	 * intermediate chunks and finally a last naked read on the last chunk.
1336 	 */
1337 	if (chunk == 0)
1338 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1339 	else if (chunk < lt->nchunks - 1)
1340 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1341 	else
1342 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1343 
1344 	marvell_nfc_send_cmd(chip, &nfc_op);
1345 
1346 	/*
1347 	 * According to the datasheet, when reading from NDDB
1348 	 * with BCH enabled, after each 32 bytes reads, we
1349 	 * have to make sure that the NDSR.RDDREQ bit is set.
1350 	 *
1351 	 * Drain the FIFO, 8 32-bit reads at a time, and skip
1352 	 * the polling on the last read.
1353 	 *
1354 	 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1355 	 */
1356 	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1357 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1358 				    "RDDREQ while draining FIFO (data)");
1359 		marvell_nfc_xfer_data_in_pio(nfc, data,
1360 					     FIFO_DEPTH * BCH_SEQ_READS);
1361 		data += FIFO_DEPTH * BCH_SEQ_READS;
1362 	}
1363 
1364 	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1365 		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1366 				    "RDDREQ while draining FIFO (OOB)");
1367 		marvell_nfc_xfer_data_in_pio(nfc, spare,
1368 					     FIFO_DEPTH * BCH_SEQ_READS);
1369 		spare += FIFO_DEPTH * BCH_SEQ_READS;
1370 	}
1371 }
1372 
1373 static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1374 					    u8 *buf, int oob_required,
1375 					    int page)
1376 {
1377 	struct mtd_info *mtd = nand_to_mtd(chip);
1378 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1379 	int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1380 	u8 *data = buf, *spare = chip->oob_poi;
1381 	int max_bitflips = 0;
1382 	u32 failure_mask = 0;
1383 	int chunk, ret;
1384 
1385 	marvell_nfc_select_target(chip, chip->cur_cs);
1386 
1387 	/*
1388 	 * With BCH, OOB is not fully used (and thus not read entirely), not
1389 	 * expected bytes could show up at the end of the OOB buffer if not
1390 	 * explicitly erased.
1391 	 */
1392 	if (oob_required)
1393 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1394 
1395 	marvell_nfc_enable_hw_ecc(chip);
1396 
1397 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1398 		/* Update length for the last chunk */
1399 		if (chunk >= lt->full_chunk_cnt) {
1400 			data_len = lt->last_data_bytes;
1401 			spare_len = lt->last_spare_bytes;
1402 		}
1403 
1404 		/* Read the chunk and detect number of bitflips */
1405 		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1406 						  spare, spare_len, page);
1407 		ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
1408 		if (ret)
1409 			failure_mask |= BIT(chunk);
1410 
1411 		data += data_len;
1412 		spare += spare_len;
1413 	}
1414 
1415 	marvell_nfc_disable_hw_ecc(chip);
1416 
1417 	if (!failure_mask)
1418 		return max_bitflips;
1419 
1420 	/*
1421 	 * Please note that dumping the ECC bytes during a normal read with OOB
1422 	 * area would add a significant overhead as ECC bytes are "consumed" by
1423 	 * the controller in normal mode and must be re-read in raw mode. To
1424 	 * avoid dropping the performances, we prefer not to include them. The
1425 	 * user should re-read the page in raw mode if ECC bytes are required.
1426 	 */
1427 
1428 	/*
1429 	 * In case there is any subpage read error, we usually re-read only ECC
1430 	 * bytes in raw mode and check if the whole page is empty. In this case,
1431 	 * it is normal that the ECC check failed and we just ignore the error.
1432 	 *
1433 	 * However, it has been empirically observed that for some layouts (e.g
1434 	 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1435 	 * bits and may create itself bitflips in the erased area. To overcome
1436 	 * this strange behavior, the whole page is re-read in raw mode, not
1437 	 * only the ECC bytes.
1438 	 */
1439 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1440 		int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1441 		int data_off, spare_off, ecc_off;
1442 		int data_len, spare_len, ecc_len;
1443 
1444 		/* No failure reported for this chunk, move to the next one */
1445 		if (!(failure_mask & BIT(chunk)))
1446 			continue;
1447 
1448 		data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1449 					    lt->ecc_bytes);
1450 		spare_off_in_page = data_off_in_page +
1451 			(chunk < lt->full_chunk_cnt ? lt->data_bytes :
1452 						      lt->last_data_bytes);
1453 		ecc_off_in_page = spare_off_in_page +
1454 			(chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1455 						      lt->last_spare_bytes);
1456 
1457 		data_off = chunk * lt->data_bytes;
1458 		spare_off = chunk * lt->spare_bytes;
1459 		ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1460 			  lt->last_spare_bytes +
1461 			  (chunk * (lt->ecc_bytes + 2));
1462 
1463 		data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1464 							lt->last_data_bytes;
1465 		spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1466 							 lt->last_spare_bytes;
1467 		ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1468 						       lt->last_ecc_bytes;
1469 
1470 		/*
1471 		 * Only re-read the ECC bytes, unless we are using the 2k/8b
1472 		 * layout which is buggy in the sense that the ECC engine will
1473 		 * try to correct data bytes anyway, creating bitflips. In this
1474 		 * case, re-read the entire page.
1475 		 */
1476 		if (lt->writesize == 2048 && lt->strength == 8) {
1477 			nand_change_read_column_op(chip, data_off_in_page,
1478 						   buf + data_off, data_len,
1479 						   false);
1480 			nand_change_read_column_op(chip, spare_off_in_page,
1481 						   chip->oob_poi + spare_off, spare_len,
1482 						   false);
1483 		}
1484 
1485 		nand_change_read_column_op(chip, ecc_off_in_page,
1486 					   chip->oob_poi + ecc_off, ecc_len,
1487 					   false);
1488 
1489 		/* Check the entire chunk (data + spare + ecc) for emptyness */
1490 		marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1491 					      chip->oob_poi + spare_off, spare_len,
1492 					      chip->oob_poi + ecc_off, ecc_len,
1493 					      &max_bitflips);
1494 	}
1495 
1496 	return max_bitflips;
1497 }
1498 
1499 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1500 {
1501 	u8 *buf = nand_get_data_buf(chip);
1502 
1503 	return chip->ecc.read_page_raw(chip, buf, true, page);
1504 }
1505 
1506 static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1507 {
1508 	u8 *buf = nand_get_data_buf(chip);
1509 
1510 	return chip->ecc.read_page(chip, buf, true, page);
1511 }
1512 
1513 /* BCH write helpers */
1514 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1515 						 const u8 *buf,
1516 						 int oob_required, int page)
1517 {
1518 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1519 	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1520 	int data_len = lt->data_bytes;
1521 	int spare_len = lt->spare_bytes;
1522 	int ecc_len = lt->ecc_bytes;
1523 	int spare_offset = 0;
1524 	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1525 		lt->last_spare_bytes;
1526 	int chunk;
1527 
1528 	marvell_nfc_select_target(chip, chip->cur_cs);
1529 
1530 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1531 
1532 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1533 		if (chunk >= lt->full_chunk_cnt) {
1534 			data_len = lt->last_data_bytes;
1535 			spare_len = lt->last_spare_bytes;
1536 			ecc_len = lt->last_ecc_bytes;
1537 		}
1538 
1539 		/* Point to the column of the next chunk */
1540 		nand_change_write_column_op(chip, chunk * full_chunk_size,
1541 					    NULL, 0, false);
1542 
1543 		/* Write the data */
1544 		nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1545 				   data_len, false);
1546 
1547 		if (!oob_required)
1548 			continue;
1549 
1550 		/* Write the spare bytes */
1551 		if (spare_len)
1552 			nand_write_data_op(chip, chip->oob_poi + spare_offset,
1553 					   spare_len, false);
1554 
1555 		/* Write the ECC bytes */
1556 		if (ecc_len)
1557 			nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1558 					   ecc_len, false);
1559 
1560 		spare_offset += spare_len;
1561 		ecc_offset += ALIGN(ecc_len, 32);
1562 	}
1563 
1564 	return nand_prog_page_end_op(chip);
1565 }
1566 
1567 static int
1568 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1569 				   const u8 *data, unsigned int data_len,
1570 				   const u8 *spare, unsigned int spare_len,
1571 				   int page)
1572 {
1573 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1574 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1575 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1576 	u32 xtype;
1577 	int ret;
1578 	struct marvell_nfc_op nfc_op = {
1579 		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1580 		.ndcb[3] = data_len + spare_len,
1581 	};
1582 
1583 	/*
1584 	 * First operation dispatches the CMD_SEQIN command, issue the address
1585 	 * cycles and asks for the first chunk of data.
1586 	 * All operations in the middle (if any) will issue a naked write and
1587 	 * also ask for data.
1588 	 * Last operation (if any) asks for the last chunk of data through a
1589 	 * last naked write.
1590 	 */
1591 	if (chunk == 0) {
1592 		if (lt->nchunks == 1)
1593 			xtype = XTYPE_MONOLITHIC_RW;
1594 		else
1595 			xtype = XTYPE_WRITE_DISPATCH;
1596 
1597 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1598 				  NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1599 				  NDCB0_CMD1(NAND_CMD_SEQIN);
1600 		nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1601 		nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1602 	} else if (chunk < lt->nchunks - 1) {
1603 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1604 	} else {
1605 		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1606 	}
1607 
1608 	/* Always dispatch the PAGEPROG command on the last chunk */
1609 	if (chunk == lt->nchunks - 1)
1610 		nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1611 
1612 	ret = marvell_nfc_prepare_cmd(chip);
1613 	if (ret)
1614 		return ret;
1615 
1616 	marvell_nfc_send_cmd(chip, &nfc_op);
1617 	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1618 				  "WRDREQ while loading FIFO (data)");
1619 	if (ret)
1620 		return ret;
1621 
1622 	/* Transfer the contents */
1623 	iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1624 	iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1625 
1626 	return 0;
1627 }
1628 
1629 static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1630 					     const u8 *buf,
1631 					     int oob_required, int page)
1632 {
1633 	const struct nand_sdr_timings *sdr =
1634 		nand_get_sdr_timings(nand_get_interface_config(chip));
1635 	struct mtd_info *mtd = nand_to_mtd(chip);
1636 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1637 	const u8 *data = buf;
1638 	const u8 *spare = chip->oob_poi;
1639 	int data_len = lt->data_bytes;
1640 	int spare_len = lt->spare_bytes;
1641 	int chunk, ret;
1642 	u8 status;
1643 
1644 	marvell_nfc_select_target(chip, chip->cur_cs);
1645 
1646 	/* Spare data will be written anyway, so clear it to avoid garbage */
1647 	if (!oob_required)
1648 		memset(chip->oob_poi, 0xFF, mtd->oobsize);
1649 
1650 	marvell_nfc_enable_hw_ecc(chip);
1651 
1652 	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1653 		if (chunk >= lt->full_chunk_cnt) {
1654 			data_len = lt->last_data_bytes;
1655 			spare_len = lt->last_spare_bytes;
1656 		}
1657 
1658 		marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1659 						   spare, spare_len, page);
1660 		data += data_len;
1661 		spare += spare_len;
1662 
1663 		/*
1664 		 * Waiting only for CMDD or PAGED is not enough, ECC are
1665 		 * partially written. No flag is set once the operation is
1666 		 * really finished but the ND_RUN bit is cleared, so wait for it
1667 		 * before stepping into the next command.
1668 		 */
1669 		marvell_nfc_wait_ndrun(chip);
1670 	}
1671 
1672 	ret = marvell_nfc_wait_op(chip, PSEC_TO_MSEC(sdr->tPROG_max));
1673 
1674 	marvell_nfc_disable_hw_ecc(chip);
1675 
1676 	if (ret)
1677 		return ret;
1678 
1679 	/* Check write status on the chip side */
1680 	ret = nand_status_op(chip, &status);
1681 	if (ret)
1682 		return ret;
1683 
1684 	if (status & NAND_STATUS_FAIL)
1685 		return -EIO;
1686 
1687 	return 0;
1688 }
1689 
1690 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1691 						int page)
1692 {
1693 	struct mtd_info *mtd = nand_to_mtd(chip);
1694 	u8 *buf = nand_get_data_buf(chip);
1695 
1696 	memset(buf, 0xFF, mtd->writesize);
1697 
1698 	return chip->ecc.write_page_raw(chip, buf, true, page);
1699 }
1700 
1701 static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1702 {
1703 	struct mtd_info *mtd = nand_to_mtd(chip);
1704 	u8 *buf = nand_get_data_buf(chip);
1705 
1706 	memset(buf, 0xFF, mtd->writesize);
1707 
1708 	return chip->ecc.write_page(chip, buf, true, page);
1709 }
1710 
1711 /* NAND framework ->exec_op() hooks and related helpers */
1712 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1713 					   const struct nand_subop *subop,
1714 					   struct marvell_nfc_op *nfc_op)
1715 {
1716 	const struct nand_op_instr *instr = NULL;
1717 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1718 	bool first_cmd = true;
1719 	unsigned int op_id;
1720 	int i;
1721 
1722 	/* Reset the input structure as most of its fields will be OR'ed */
1723 	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1724 
1725 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1726 		unsigned int offset, naddrs;
1727 		const u8 *addrs;
1728 		int len;
1729 
1730 		instr = &subop->instrs[op_id];
1731 
1732 		switch (instr->type) {
1733 		case NAND_OP_CMD_INSTR:
1734 			if (first_cmd)
1735 				nfc_op->ndcb[0] |=
1736 					NDCB0_CMD1(instr->ctx.cmd.opcode);
1737 			else
1738 				nfc_op->ndcb[0] |=
1739 					NDCB0_CMD2(instr->ctx.cmd.opcode) |
1740 					NDCB0_DBC;
1741 
1742 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1743 			first_cmd = false;
1744 			break;
1745 
1746 		case NAND_OP_ADDR_INSTR:
1747 			offset = nand_subop_get_addr_start_off(subop, op_id);
1748 			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1749 			addrs = &instr->ctx.addr.addrs[offset];
1750 
1751 			nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1752 
1753 			for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1754 				nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1755 
1756 			if (naddrs >= 5)
1757 				nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1758 			if (naddrs >= 6)
1759 				nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1760 			if (naddrs == 7)
1761 				nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1762 
1763 			nfc_op->cle_ale_delay_ns = instr->delay_ns;
1764 			break;
1765 
1766 		case NAND_OP_DATA_IN_INSTR:
1767 			nfc_op->data_instr = instr;
1768 			nfc_op->data_instr_idx = op_id;
1769 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1770 			if (nfc->caps->is_nfcv2) {
1771 				nfc_op->ndcb[0] |=
1772 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1773 					NDCB0_LEN_OVRD;
1774 				len = nand_subop_get_data_len(subop, op_id);
1775 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1776 			}
1777 			nfc_op->data_delay_ns = instr->delay_ns;
1778 			break;
1779 
1780 		case NAND_OP_DATA_OUT_INSTR:
1781 			nfc_op->data_instr = instr;
1782 			nfc_op->data_instr_idx = op_id;
1783 			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1784 			if (nfc->caps->is_nfcv2) {
1785 				nfc_op->ndcb[0] |=
1786 					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1787 					NDCB0_LEN_OVRD;
1788 				len = nand_subop_get_data_len(subop, op_id);
1789 				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1790 			}
1791 			nfc_op->data_delay_ns = instr->delay_ns;
1792 			break;
1793 
1794 		case NAND_OP_WAITRDY_INSTR:
1795 			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1796 			nfc_op->rdy_delay_ns = instr->delay_ns;
1797 			break;
1798 		}
1799 	}
1800 }
1801 
1802 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1803 				     const struct nand_subop *subop,
1804 				     struct marvell_nfc_op *nfc_op)
1805 {
1806 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1807 	const struct nand_op_instr *instr = nfc_op->data_instr;
1808 	unsigned int op_id = nfc_op->data_instr_idx;
1809 	unsigned int len = nand_subop_get_data_len(subop, op_id);
1810 	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1811 	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1812 	int ret;
1813 
1814 	if (instr->ctx.data.force_8bit)
1815 		marvell_nfc_force_byte_access(chip, true);
1816 
1817 	if (reading) {
1818 		u8 *in = instr->ctx.data.buf.in + offset;
1819 
1820 		ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1821 	} else {
1822 		const u8 *out = instr->ctx.data.buf.out + offset;
1823 
1824 		ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1825 	}
1826 
1827 	if (instr->ctx.data.force_8bit)
1828 		marvell_nfc_force_byte_access(chip, false);
1829 
1830 	return ret;
1831 }
1832 
1833 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1834 					      const struct nand_subop *subop)
1835 {
1836 	struct marvell_nfc_op nfc_op;
1837 	bool reading;
1838 	int ret;
1839 
1840 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1841 	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1842 
1843 	ret = marvell_nfc_prepare_cmd(chip);
1844 	if (ret)
1845 		return ret;
1846 
1847 	marvell_nfc_send_cmd(chip, &nfc_op);
1848 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1849 				  "RDDREQ/WRDREQ while draining raw data");
1850 	if (ret)
1851 		return ret;
1852 
1853 	cond_delay(nfc_op.cle_ale_delay_ns);
1854 
1855 	if (reading) {
1856 		if (nfc_op.rdy_timeout_ms) {
1857 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1858 			if (ret)
1859 				return ret;
1860 		}
1861 
1862 		cond_delay(nfc_op.rdy_delay_ns);
1863 	}
1864 
1865 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1866 	ret = marvell_nfc_wait_cmdd(chip);
1867 	if (ret)
1868 		return ret;
1869 
1870 	cond_delay(nfc_op.data_delay_ns);
1871 
1872 	if (!reading) {
1873 		if (nfc_op.rdy_timeout_ms) {
1874 			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1875 			if (ret)
1876 				return ret;
1877 		}
1878 
1879 		cond_delay(nfc_op.rdy_delay_ns);
1880 	}
1881 
1882 	/*
1883 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1884 	 * operation but experience shows that the behavior is buggy when it
1885 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1886 	 */
1887 	if (!reading) {
1888 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1889 
1890 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1891 			       nfc->regs + NDCR);
1892 	}
1893 
1894 	return 0;
1895 }
1896 
1897 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1898 					 const struct nand_subop *subop)
1899 {
1900 	struct marvell_nfc_op nfc_op;
1901 	int ret;
1902 
1903 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1904 
1905 	/*
1906 	 * Naked access are different in that they need to be flagged as naked
1907 	 * by the controller. Reset the controller registers fields that inform
1908 	 * on the type and refill them according to the ongoing operation.
1909 	 */
1910 	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1911 			    NDCB0_CMD_XTYPE(XTYPE_MASK));
1912 	switch (subop->instrs[0].type) {
1913 	case NAND_OP_CMD_INSTR:
1914 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1915 		break;
1916 	case NAND_OP_ADDR_INSTR:
1917 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1918 		break;
1919 	case NAND_OP_DATA_IN_INSTR:
1920 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1921 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1922 		break;
1923 	case NAND_OP_DATA_OUT_INSTR:
1924 		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1925 				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1926 		break;
1927 	default:
1928 		/* This should never happen */
1929 		break;
1930 	}
1931 
1932 	ret = marvell_nfc_prepare_cmd(chip);
1933 	if (ret)
1934 		return ret;
1935 
1936 	marvell_nfc_send_cmd(chip, &nfc_op);
1937 
1938 	if (!nfc_op.data_instr) {
1939 		ret = marvell_nfc_wait_cmdd(chip);
1940 		cond_delay(nfc_op.cle_ale_delay_ns);
1941 		return ret;
1942 	}
1943 
1944 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1945 				  "RDDREQ/WRDREQ while draining raw data");
1946 	if (ret)
1947 		return ret;
1948 
1949 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1950 	ret = marvell_nfc_wait_cmdd(chip);
1951 	if (ret)
1952 		return ret;
1953 
1954 	/*
1955 	 * NDCR ND_RUN bit should be cleared automatically at the end of each
1956 	 * operation but experience shows that the behavior is buggy when it
1957 	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1958 	 */
1959 	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1960 		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1961 
1962 		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1963 			       nfc->regs + NDCR);
1964 	}
1965 
1966 	return 0;
1967 }
1968 
1969 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1970 					  const struct nand_subop *subop)
1971 {
1972 	struct marvell_nfc_op nfc_op;
1973 	int ret;
1974 
1975 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1976 
1977 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1978 	cond_delay(nfc_op.rdy_delay_ns);
1979 
1980 	return ret;
1981 }
1982 
1983 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1984 					 const struct nand_subop *subop)
1985 {
1986 	struct marvell_nfc_op nfc_op;
1987 	int ret;
1988 
1989 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1990 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1991 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1992 
1993 	ret = marvell_nfc_prepare_cmd(chip);
1994 	if (ret)
1995 		return ret;
1996 
1997 	marvell_nfc_send_cmd(chip, &nfc_op);
1998 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1999 				  "RDDREQ while reading ID");
2000 	if (ret)
2001 		return ret;
2002 
2003 	cond_delay(nfc_op.cle_ale_delay_ns);
2004 
2005 	if (nfc_op.rdy_timeout_ms) {
2006 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2007 		if (ret)
2008 			return ret;
2009 	}
2010 
2011 	cond_delay(nfc_op.rdy_delay_ns);
2012 
2013 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
2014 	ret = marvell_nfc_wait_cmdd(chip);
2015 	if (ret)
2016 		return ret;
2017 
2018 	cond_delay(nfc_op.data_delay_ns);
2019 
2020 	return 0;
2021 }
2022 
2023 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
2024 					const struct nand_subop *subop)
2025 {
2026 	struct marvell_nfc_op nfc_op;
2027 	int ret;
2028 
2029 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2030 	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
2031 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
2032 
2033 	ret = marvell_nfc_prepare_cmd(chip);
2034 	if (ret)
2035 		return ret;
2036 
2037 	marvell_nfc_send_cmd(chip, &nfc_op);
2038 	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
2039 				  "RDDREQ while reading status");
2040 	if (ret)
2041 		return ret;
2042 
2043 	cond_delay(nfc_op.cle_ale_delay_ns);
2044 
2045 	if (nfc_op.rdy_timeout_ms) {
2046 		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2047 		if (ret)
2048 			return ret;
2049 	}
2050 
2051 	cond_delay(nfc_op.rdy_delay_ns);
2052 
2053 	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
2054 	ret = marvell_nfc_wait_cmdd(chip);
2055 	if (ret)
2056 		return ret;
2057 
2058 	cond_delay(nfc_op.data_delay_ns);
2059 
2060 	return 0;
2061 }
2062 
2063 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
2064 					   const struct nand_subop *subop)
2065 {
2066 	struct marvell_nfc_op nfc_op;
2067 	int ret;
2068 
2069 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2070 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
2071 
2072 	ret = marvell_nfc_prepare_cmd(chip);
2073 	if (ret)
2074 		return ret;
2075 
2076 	marvell_nfc_send_cmd(chip, &nfc_op);
2077 	ret = marvell_nfc_wait_cmdd(chip);
2078 	if (ret)
2079 		return ret;
2080 
2081 	cond_delay(nfc_op.cle_ale_delay_ns);
2082 
2083 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2084 	if (ret)
2085 		return ret;
2086 
2087 	cond_delay(nfc_op.rdy_delay_ns);
2088 
2089 	return 0;
2090 }
2091 
2092 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2093 					   const struct nand_subop *subop)
2094 {
2095 	struct marvell_nfc_op nfc_op;
2096 	int ret;
2097 
2098 	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2099 	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2100 
2101 	ret = marvell_nfc_prepare_cmd(chip);
2102 	if (ret)
2103 		return ret;
2104 
2105 	marvell_nfc_send_cmd(chip, &nfc_op);
2106 	ret = marvell_nfc_wait_cmdd(chip);
2107 	if (ret)
2108 		return ret;
2109 
2110 	cond_delay(nfc_op.cle_ale_delay_ns);
2111 
2112 	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2113 	if (ret)
2114 		return ret;
2115 
2116 	cond_delay(nfc_op.rdy_delay_ns);
2117 
2118 	return 0;
2119 }
2120 
2121 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2122 	/* Monolithic reads/writes */
2123 	NAND_OP_PARSER_PATTERN(
2124 		marvell_nfc_monolithic_access_exec,
2125 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2126 		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2127 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2128 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2129 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2130 	NAND_OP_PARSER_PATTERN(
2131 		marvell_nfc_monolithic_access_exec,
2132 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2133 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2134 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2135 		NAND_OP_PARSER_PAT_CMD_ELEM(true),
2136 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2137 	/* Naked commands */
2138 	NAND_OP_PARSER_PATTERN(
2139 		marvell_nfc_naked_access_exec,
2140 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2141 	NAND_OP_PARSER_PATTERN(
2142 		marvell_nfc_naked_access_exec,
2143 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2144 	NAND_OP_PARSER_PATTERN(
2145 		marvell_nfc_naked_access_exec,
2146 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2147 	NAND_OP_PARSER_PATTERN(
2148 		marvell_nfc_naked_access_exec,
2149 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2150 	NAND_OP_PARSER_PATTERN(
2151 		marvell_nfc_naked_waitrdy_exec,
2152 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2153 	);
2154 
2155 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2156 	/* Naked commands not supported, use a function for each pattern */
2157 	NAND_OP_PARSER_PATTERN(
2158 		marvell_nfc_read_id_type_exec,
2159 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2160 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2161 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2162 	NAND_OP_PARSER_PATTERN(
2163 		marvell_nfc_erase_cmd_type_exec,
2164 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2165 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2166 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2167 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2168 	NAND_OP_PARSER_PATTERN(
2169 		marvell_nfc_read_status_exec,
2170 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2171 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2172 	NAND_OP_PARSER_PATTERN(
2173 		marvell_nfc_reset_cmd_type_exec,
2174 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2175 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2176 	NAND_OP_PARSER_PATTERN(
2177 		marvell_nfc_naked_waitrdy_exec,
2178 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2179 	);
2180 
2181 static int marvell_nfc_exec_op(struct nand_chip *chip,
2182 			       const struct nand_operation *op,
2183 			       bool check_only)
2184 {
2185 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2186 
2187 	if (!check_only)
2188 		marvell_nfc_select_target(chip, op->cs);
2189 
2190 	if (nfc->caps->is_nfcv2)
2191 		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2192 					      op, check_only);
2193 	else
2194 		return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2195 					      op, check_only);
2196 }
2197 
2198 /*
2199  * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2200  * usable.
2201  */
2202 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2203 				      struct mtd_oob_region *oobregion)
2204 {
2205 	struct nand_chip *chip = mtd_to_nand(mtd);
2206 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2207 
2208 	if (section)
2209 		return -ERANGE;
2210 
2211 	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2212 			    lt->last_ecc_bytes;
2213 	oobregion->offset = mtd->oobsize - oobregion->length;
2214 
2215 	return 0;
2216 }
2217 
2218 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2219 				       struct mtd_oob_region *oobregion)
2220 {
2221 	struct nand_chip *chip = mtd_to_nand(mtd);
2222 	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2223 
2224 	if (section)
2225 		return -ERANGE;
2226 
2227 	/*
2228 	 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2229 	 * 4KB page / 4bit BCH combination.
2230 	 */
2231 	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2232 		oobregion->offset = 6;
2233 	else
2234 		oobregion->offset = 2;
2235 
2236 	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2237 			    lt->last_spare_bytes - oobregion->offset;
2238 
2239 	return 0;
2240 }
2241 
2242 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2243 	.ecc = marvell_nand_ooblayout_ecc,
2244 	.free = marvell_nand_ooblayout_free,
2245 };
2246 
2247 static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd,
2248 					       struct nand_ecc_ctrl *ecc)
2249 {
2250 	struct nand_chip *chip = mtd_to_nand(mtd);
2251 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2252 	const struct marvell_hw_ecc_layout *l;
2253 	int i;
2254 
2255 	if (!nfc->caps->is_nfcv2 &&
2256 	    (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2257 		dev_err(nfc->dev,
2258 			"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2259 			mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2260 		return -ENOTSUPP;
2261 	}
2262 
2263 	to_marvell_nand(chip)->layout = NULL;
2264 	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2265 		l = &marvell_nfc_layouts[i];
2266 		if (mtd->writesize == l->writesize &&
2267 		    ecc->size == l->chunk && ecc->strength == l->strength) {
2268 			to_marvell_nand(chip)->layout = l;
2269 			break;
2270 		}
2271 	}
2272 
2273 	if (!to_marvell_nand(chip)->layout ||
2274 	    (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2275 		dev_err(nfc->dev,
2276 			"ECC strength %d at page size %d is not supported\n",
2277 			ecc->strength, mtd->writesize);
2278 		return -ENOTSUPP;
2279 	}
2280 
2281 	/* Special care for the layout 2k/8-bit/512B  */
2282 	if (l->writesize == 2048 && l->strength == 8) {
2283 		if (mtd->oobsize < 128) {
2284 			dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2285 			return -ENOTSUPP;
2286 		} else {
2287 			chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2288 		}
2289 	}
2290 
2291 	mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2292 	ecc->steps = l->nchunks;
2293 	ecc->size = l->data_bytes;
2294 
2295 	if (ecc->strength == 1) {
2296 		chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
2297 		ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2298 		ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2299 		ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2300 		ecc->read_oob = ecc->read_oob_raw;
2301 		ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2302 		ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2303 		ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2304 		ecc->write_oob = ecc->write_oob_raw;
2305 	} else {
2306 		chip->ecc.algo = NAND_ECC_ALGO_BCH;
2307 		ecc->strength = 16;
2308 		ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2309 		ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2310 		ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2311 		ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2312 		ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2313 		ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2314 		ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2315 		ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2316 	}
2317 
2318 	return 0;
2319 }
2320 
2321 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2322 				 struct nand_ecc_ctrl *ecc)
2323 {
2324 	struct nand_chip *chip = mtd_to_nand(mtd);
2325 	const struct nand_ecc_props *requirements =
2326 		nanddev_get_ecc_requirements(&chip->base);
2327 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2328 	int ret;
2329 
2330 	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
2331 	    (!ecc->size || !ecc->strength)) {
2332 		if (requirements->step_size && requirements->strength) {
2333 			ecc->size = requirements->step_size;
2334 			ecc->strength = requirements->strength;
2335 		} else {
2336 			dev_info(nfc->dev,
2337 				 "No minimum ECC strength, using 1b/512B\n");
2338 			ecc->size = 512;
2339 			ecc->strength = 1;
2340 		}
2341 	}
2342 
2343 	switch (ecc->engine_type) {
2344 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
2345 		ret = marvell_nand_hw_ecc_controller_init(mtd, ecc);
2346 		if (ret)
2347 			return ret;
2348 		break;
2349 	case NAND_ECC_ENGINE_TYPE_NONE:
2350 	case NAND_ECC_ENGINE_TYPE_SOFT:
2351 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
2352 		if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2353 		    mtd->writesize != SZ_2K) {
2354 			dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2355 				mtd->writesize);
2356 			return -EINVAL;
2357 		}
2358 		break;
2359 	default:
2360 		return -EINVAL;
2361 	}
2362 
2363 	return 0;
2364 }
2365 
2366 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2367 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2368 
2369 static struct nand_bbt_descr bbt_main_descr = {
2370 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2371 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2372 	.offs =	8,
2373 	.len = 6,
2374 	.veroffs = 14,
2375 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2376 	.pattern = bbt_pattern
2377 };
2378 
2379 static struct nand_bbt_descr bbt_mirror_descr = {
2380 	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2381 		   NAND_BBT_2BIT | NAND_BBT_VERSION,
2382 	.offs =	8,
2383 	.len = 6,
2384 	.veroffs = 14,
2385 	.maxblocks = 8,	/* Last 8 blocks in each chip */
2386 	.pattern = bbt_mirror_pattern
2387 };
2388 
2389 static int marvell_nfc_setup_interface(struct nand_chip *chip, int chipnr,
2390 				       const struct nand_interface_config *conf)
2391 {
2392 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2393 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2394 	unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2395 	const struct nand_sdr_timings *sdr;
2396 	struct marvell_nfc_timings nfc_tmg;
2397 	int read_delay;
2398 
2399 	sdr = nand_get_sdr_timings(conf);
2400 	if (IS_ERR(sdr))
2401 		return PTR_ERR(sdr);
2402 
2403 	if (nfc->caps->max_mode_number && nfc->caps->max_mode_number < conf->timings.mode)
2404 		return -EOPNOTSUPP;
2405 
2406 	/*
2407 	 * SDR timings are given in pico-seconds while NFC timings must be
2408 	 * expressed in NAND controller clock cycles, which is half of the
2409 	 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2410 	 * This is not written anywhere in the datasheet but was observed
2411 	 * with an oscilloscope.
2412 	 *
2413 	 * NFC datasheet gives equations from which thoses calculations
2414 	 * are derived, they tend to be slightly more restrictives than the
2415 	 * given core timings and may improve the overall speed.
2416 	 */
2417 	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2418 	nfc_tmg.tRH = nfc_tmg.tRP;
2419 	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2420 	nfc_tmg.tWH = nfc_tmg.tWP;
2421 	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2422 	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2423 	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2424 	/*
2425 	 * Read delay is the time of propagation from SoC pins to NFC internal
2426 	 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2427 	 * EDO mode, an additional delay of tRH must be taken into account so
2428 	 * the data is sampled on the falling edge instead of the rising edge.
2429 	 */
2430 	read_delay = sdr->tRC_min >= 30000 ?
2431 		MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2432 
2433 	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2434 	/*
2435 	 * tWHR and tRHW are supposed to be read to write delays (and vice
2436 	 * versa) but in some cases, ie. when doing a change column, they must
2437 	 * be greater than that to be sure tCCS delay is respected.
2438 	 */
2439 	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2440 				 period_ns) - 2;
2441 	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2442 				 period_ns);
2443 
2444 	/*
2445 	 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2446 	 * NFCv1: No WAIT_MODE, tR must be maximal.
2447 	 */
2448 	if (nfc->caps->is_nfcv2) {
2449 		nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2450 	} else {
2451 		nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2452 					 period_ns);
2453 		if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2454 			nfc_tmg.tR = nfc_tmg.tCH - 3;
2455 		else
2456 			nfc_tmg.tR = 0;
2457 	}
2458 
2459 	if (chipnr < 0)
2460 		return 0;
2461 
2462 	marvell_nand->ndtr0 =
2463 		NDTR0_TRP(nfc_tmg.tRP) |
2464 		NDTR0_TRH(nfc_tmg.tRH) |
2465 		NDTR0_ETRP(nfc_tmg.tRP) |
2466 		NDTR0_TWP(nfc_tmg.tWP) |
2467 		NDTR0_TWH(nfc_tmg.tWH) |
2468 		NDTR0_TCS(nfc_tmg.tCS) |
2469 		NDTR0_TCH(nfc_tmg.tCH);
2470 
2471 	marvell_nand->ndtr1 =
2472 		NDTR1_TAR(nfc_tmg.tAR) |
2473 		NDTR1_TWHR(nfc_tmg.tWHR) |
2474 		NDTR1_TR(nfc_tmg.tR);
2475 
2476 	if (nfc->caps->is_nfcv2) {
2477 		marvell_nand->ndtr0 |=
2478 			NDTR0_RD_CNT_DEL(read_delay) |
2479 			NDTR0_SELCNTR |
2480 			NDTR0_TADL(nfc_tmg.tADL);
2481 
2482 		marvell_nand->ndtr1 |=
2483 			NDTR1_TRHW(nfc_tmg.tRHW) |
2484 			NDTR1_WAIT_MODE;
2485 	}
2486 
2487 	/*
2488 	 * Reset nfc->selected_chip so the next command will cause the timing
2489 	 * registers to be updated in marvell_nfc_select_target().
2490 	 */
2491 	nfc->selected_chip = NULL;
2492 
2493 	return 0;
2494 }
2495 
2496 static int marvell_nand_attach_chip(struct nand_chip *chip)
2497 {
2498 	struct mtd_info *mtd = nand_to_mtd(chip);
2499 	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2500 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2501 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2502 	int ret;
2503 
2504 	if (pdata && pdata->flash_bbt)
2505 		chip->bbt_options |= NAND_BBT_USE_FLASH;
2506 
2507 	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2508 		/*
2509 		 * We'll use a bad block table stored in-flash and don't
2510 		 * allow writing the bad block marker to the flash.
2511 		 */
2512 		chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2513 		chip->bbt_td = &bbt_main_descr;
2514 		chip->bbt_md = &bbt_mirror_descr;
2515 	}
2516 
2517 	/* Save the chip-specific fields of NDCR */
2518 	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2519 	if (chip->options & NAND_BUSWIDTH_16)
2520 		marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2521 
2522 	/*
2523 	 * On small page NANDs, only one cycle is needed to pass the
2524 	 * column address.
2525 	 */
2526 	if (mtd->writesize <= 512) {
2527 		marvell_nand->addr_cyc = 1;
2528 	} else {
2529 		marvell_nand->addr_cyc = 2;
2530 		marvell_nand->ndcr |= NDCR_RA_START;
2531 	}
2532 
2533 	/*
2534 	 * Now add the number of cycles needed to pass the row
2535 	 * address.
2536 	 *
2537 	 * Addressing a chip using CS 2 or 3 should also need the third row
2538 	 * cycle but due to inconsistance in the documentation and lack of
2539 	 * hardware to test this situation, this case is not supported.
2540 	 */
2541 	if (chip->options & NAND_ROW_ADDR_3)
2542 		marvell_nand->addr_cyc += 3;
2543 	else
2544 		marvell_nand->addr_cyc += 2;
2545 
2546 	if (pdata) {
2547 		chip->ecc.size = pdata->ecc_step_size;
2548 		chip->ecc.strength = pdata->ecc_strength;
2549 	}
2550 
2551 	ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2552 	if (ret) {
2553 		dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2554 		return ret;
2555 	}
2556 
2557 	if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
2558 		/*
2559 		 * Subpage write not available with hardware ECC, prohibit also
2560 		 * subpage read as in userspace subpage access would still be
2561 		 * allowed and subpage write, if used, would lead to numerous
2562 		 * uncorrectable ECC errors.
2563 		 */
2564 		chip->options |= NAND_NO_SUBPAGE_WRITE;
2565 	}
2566 
2567 	if (pdata || nfc->caps->legacy_of_bindings) {
2568 		/*
2569 		 * We keep the MTD name unchanged to avoid breaking platforms
2570 		 * where the MTD cmdline parser is used and the bootloader
2571 		 * has not been updated to use the new naming scheme.
2572 		 */
2573 		mtd->name = "pxa3xx_nand-0";
2574 	} else if (!mtd->name) {
2575 		/*
2576 		 * If the new bindings are used and the bootloader has not been
2577 		 * updated to pass a new mtdparts parameter on the cmdline, you
2578 		 * should define the following property in your NAND node, ie:
2579 		 *
2580 		 *	label = "main-storage";
2581 		 *
2582 		 * This way, mtd->name will be set by the core when
2583 		 * nand_set_flash_node() is called.
2584 		 */
2585 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2586 					   "%s:nand.%d", dev_name(nfc->dev),
2587 					   marvell_nand->sels[0].cs);
2588 		if (!mtd->name) {
2589 			dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2590 			return -ENOMEM;
2591 		}
2592 	}
2593 
2594 	return 0;
2595 }
2596 
2597 static const struct nand_controller_ops marvell_nand_controller_ops = {
2598 	.attach_chip = marvell_nand_attach_chip,
2599 	.exec_op = marvell_nfc_exec_op,
2600 	.setup_interface = marvell_nfc_setup_interface,
2601 };
2602 
2603 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2604 				  struct device_node *np)
2605 {
2606 	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2607 	struct marvell_nand_chip *marvell_nand;
2608 	struct mtd_info *mtd;
2609 	struct nand_chip *chip;
2610 	int nsels, ret, i;
2611 	u32 cs, rb;
2612 
2613 	/*
2614 	 * The legacy "num-cs" property indicates the number of CS on the only
2615 	 * chip connected to the controller (legacy bindings does not support
2616 	 * more than one chip). The CS and RB pins are always the #0.
2617 	 *
2618 	 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2619 	 * properties must be filled. For each chip, expressed as a subnode,
2620 	 * "reg" points to the CS lines and "nand-rb" to the RB line.
2621 	 */
2622 	if (pdata || nfc->caps->legacy_of_bindings) {
2623 		nsels = 1;
2624 	} else {
2625 		nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2626 		if (nsels <= 0) {
2627 			dev_err(dev, "missing/invalid reg property\n");
2628 			return -EINVAL;
2629 		}
2630 	}
2631 
2632 	/* Alloc the nand chip structure */
2633 	marvell_nand = devm_kzalloc(dev,
2634 				    struct_size(marvell_nand, sels, nsels),
2635 				    GFP_KERNEL);
2636 	if (!marvell_nand) {
2637 		dev_err(dev, "could not allocate chip structure\n");
2638 		return -ENOMEM;
2639 	}
2640 
2641 	marvell_nand->nsels = nsels;
2642 	marvell_nand->selected_die = -1;
2643 
2644 	for (i = 0; i < nsels; i++) {
2645 		if (pdata || nfc->caps->legacy_of_bindings) {
2646 			/*
2647 			 * Legacy bindings use the CS lines in natural
2648 			 * order (0, 1, ...)
2649 			 */
2650 			cs = i;
2651 		} else {
2652 			/* Retrieve CS id */
2653 			ret = of_property_read_u32_index(np, "reg", i, &cs);
2654 			if (ret) {
2655 				dev_err(dev, "could not retrieve reg property: %d\n",
2656 					ret);
2657 				return ret;
2658 			}
2659 		}
2660 
2661 		if (cs >= nfc->caps->max_cs_nb) {
2662 			dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2663 				cs, nfc->caps->max_cs_nb);
2664 			return -EINVAL;
2665 		}
2666 
2667 		if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2668 			dev_err(dev, "CS %d already assigned\n", cs);
2669 			return -EINVAL;
2670 		}
2671 
2672 		/*
2673 		 * The cs variable represents the chip select id, which must be
2674 		 * converted in bit fields for NDCB0 and NDCB2 to select the
2675 		 * right chip. Unfortunately, due to a lack of information on
2676 		 * the subject and incoherent documentation, the user should not
2677 		 * use CS1 and CS3 at all as asserting them is not supported in
2678 		 * a reliable way (due to multiplexing inside ADDR5 field).
2679 		 */
2680 		marvell_nand->sels[i].cs = cs;
2681 		switch (cs) {
2682 		case 0:
2683 		case 2:
2684 			marvell_nand->sels[i].ndcb0_csel = 0;
2685 			break;
2686 		case 1:
2687 		case 3:
2688 			marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2689 			break;
2690 		default:
2691 			return -EINVAL;
2692 		}
2693 
2694 		/* Retrieve RB id */
2695 		if (pdata || nfc->caps->legacy_of_bindings) {
2696 			/* Legacy bindings always use RB #0 */
2697 			rb = 0;
2698 		} else {
2699 			ret = of_property_read_u32_index(np, "nand-rb", i,
2700 							 &rb);
2701 			if (ret) {
2702 				dev_err(dev,
2703 					"could not retrieve RB property: %d\n",
2704 					ret);
2705 				return ret;
2706 			}
2707 		}
2708 
2709 		if (rb >= nfc->caps->max_rb_nb) {
2710 			dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2711 				rb, nfc->caps->max_rb_nb);
2712 			return -EINVAL;
2713 		}
2714 
2715 		marvell_nand->sels[i].rb = rb;
2716 	}
2717 
2718 	chip = &marvell_nand->chip;
2719 	chip->controller = &nfc->controller;
2720 	nand_set_flash_node(chip, np);
2721 
2722 	if (of_property_read_bool(np, "marvell,nand-keep-config"))
2723 		chip->options |= NAND_KEEP_TIMINGS;
2724 
2725 	mtd = nand_to_mtd(chip);
2726 	mtd->dev.parent = dev;
2727 
2728 	/*
2729 	 * Save a reference value for timing registers before
2730 	 * ->setup_interface() is called.
2731 	 */
2732 	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2733 	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2734 
2735 	chip->options |= NAND_BUSWIDTH_AUTO;
2736 
2737 	ret = nand_scan(chip, marvell_nand->nsels);
2738 	if (ret) {
2739 		dev_err(dev, "could not scan the nand chip\n");
2740 		return ret;
2741 	}
2742 
2743 	if (pdata)
2744 		/* Legacy bindings support only one chip */
2745 		ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2746 	else
2747 		ret = mtd_device_register(mtd, NULL, 0);
2748 	if (ret) {
2749 		dev_err(dev, "failed to register mtd device: %d\n", ret);
2750 		nand_cleanup(chip);
2751 		return ret;
2752 	}
2753 
2754 	list_add_tail(&marvell_nand->node, &nfc->chips);
2755 
2756 	return 0;
2757 }
2758 
2759 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2760 {
2761 	struct marvell_nand_chip *entry, *temp;
2762 	struct nand_chip *chip;
2763 	int ret;
2764 
2765 	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2766 		chip = &entry->chip;
2767 		ret = mtd_device_unregister(nand_to_mtd(chip));
2768 		WARN_ON(ret);
2769 		nand_cleanup(chip);
2770 		list_del(&entry->node);
2771 	}
2772 }
2773 
2774 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2775 {
2776 	struct device_node *np = dev->of_node;
2777 	struct device_node *nand_np;
2778 	int max_cs = nfc->caps->max_cs_nb;
2779 	int nchips;
2780 	int ret;
2781 
2782 	if (!np)
2783 		nchips = 1;
2784 	else
2785 		nchips = of_get_child_count(np);
2786 
2787 	if (nchips > max_cs) {
2788 		dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2789 			max_cs);
2790 		return -EINVAL;
2791 	}
2792 
2793 	/*
2794 	 * Legacy bindings do not use child nodes to exhibit NAND chip
2795 	 * properties and layout. Instead, NAND properties are mixed with the
2796 	 * controller ones, and partitions are defined as direct subnodes of the
2797 	 * NAND controller node.
2798 	 */
2799 	if (nfc->caps->legacy_of_bindings) {
2800 		ret = marvell_nand_chip_init(dev, nfc, np);
2801 		return ret;
2802 	}
2803 
2804 	for_each_child_of_node(np, nand_np) {
2805 		ret = marvell_nand_chip_init(dev, nfc, nand_np);
2806 		if (ret) {
2807 			of_node_put(nand_np);
2808 			goto cleanup_chips;
2809 		}
2810 	}
2811 
2812 	return 0;
2813 
2814 cleanup_chips:
2815 	marvell_nand_chips_cleanup(nfc);
2816 
2817 	return ret;
2818 }
2819 
2820 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2821 {
2822 	struct platform_device *pdev = container_of(nfc->dev,
2823 						    struct platform_device,
2824 						    dev);
2825 	struct dma_slave_config config = {};
2826 	struct resource *r;
2827 	int ret;
2828 
2829 	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2830 		dev_warn(nfc->dev,
2831 			 "DMA not enabled in configuration\n");
2832 		return -ENOTSUPP;
2833 	}
2834 
2835 	ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2836 	if (ret)
2837 		return ret;
2838 
2839 	nfc->dma_chan =	dma_request_chan(nfc->dev, "data");
2840 	if (IS_ERR(nfc->dma_chan)) {
2841 		ret = PTR_ERR(nfc->dma_chan);
2842 		nfc->dma_chan = NULL;
2843 		return dev_err_probe(nfc->dev, ret, "DMA channel request failed\n");
2844 	}
2845 
2846 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2847 	if (!r) {
2848 		ret = -ENXIO;
2849 		goto release_channel;
2850 	}
2851 
2852 	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2853 	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2854 	config.src_addr = r->start + NDDB;
2855 	config.dst_addr = r->start + NDDB;
2856 	config.src_maxburst = 32;
2857 	config.dst_maxburst = 32;
2858 	ret = dmaengine_slave_config(nfc->dma_chan, &config);
2859 	if (ret < 0) {
2860 		dev_err(nfc->dev, "Failed to configure DMA channel\n");
2861 		goto release_channel;
2862 	}
2863 
2864 	/*
2865 	 * DMA must act on length multiple of 32 and this length may be
2866 	 * bigger than the destination buffer. Use this buffer instead
2867 	 * for DMA transfers and then copy the desired amount of data to
2868 	 * the provided buffer.
2869 	 */
2870 	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2871 	if (!nfc->dma_buf) {
2872 		ret = -ENOMEM;
2873 		goto release_channel;
2874 	}
2875 
2876 	nfc->use_dma = true;
2877 
2878 	return 0;
2879 
2880 release_channel:
2881 	dma_release_channel(nfc->dma_chan);
2882 	nfc->dma_chan = NULL;
2883 
2884 	return ret;
2885 }
2886 
2887 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2888 {
2889 	/*
2890 	 * ECC operations and interruptions are only enabled when specifically
2891 	 * needed. ECC shall not be activated in the early stages (fails probe).
2892 	 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2893 	 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2894 	 * offset in the read page and this will fail the protection.
2895 	 */
2896 	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2897 		       NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2898 	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2899 	writel_relaxed(0, nfc->regs + NDECCCTRL);
2900 }
2901 
2902 static int marvell_nfc_init(struct marvell_nfc *nfc)
2903 {
2904 	struct device_node *np = nfc->dev->of_node;
2905 
2906 	/*
2907 	 * Some SoCs like A7k/A8k need to enable manually the NAND
2908 	 * controller, gated clocks and reset bits to avoid being bootloader
2909 	 * dependent. This is done through the use of the System Functions
2910 	 * registers.
2911 	 */
2912 	if (nfc->caps->need_system_controller) {
2913 		struct regmap *sysctrl_base =
2914 			syscon_regmap_lookup_by_phandle(np,
2915 							"marvell,system-controller");
2916 
2917 		if (IS_ERR(sysctrl_base))
2918 			return PTR_ERR(sysctrl_base);
2919 
2920 		regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2921 			     GENCONF_SOC_DEVICE_MUX_NFC_EN |
2922 			     GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2923 			     GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2924 			     GENCONF_SOC_DEVICE_MUX_NFC_INT_EN |
2925 			     GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN);
2926 
2927 		regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2928 				   GENCONF_CLK_GATING_CTRL_ND_GATE,
2929 				   GENCONF_CLK_GATING_CTRL_ND_GATE);
2930 	}
2931 
2932 	/* Configure the DMA if appropriate */
2933 	if (!nfc->caps->is_nfcv2)
2934 		marvell_nfc_init_dma(nfc);
2935 
2936 	marvell_nfc_reset(nfc);
2937 
2938 	return 0;
2939 }
2940 
2941 static int marvell_nfc_probe(struct platform_device *pdev)
2942 {
2943 	struct device *dev = &pdev->dev;
2944 	struct marvell_nfc *nfc;
2945 	int ret;
2946 	int irq;
2947 
2948 	nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2949 			   GFP_KERNEL);
2950 	if (!nfc)
2951 		return -ENOMEM;
2952 
2953 	nfc->dev = dev;
2954 	nand_controller_init(&nfc->controller);
2955 	nfc->controller.ops = &marvell_nand_controller_ops;
2956 	INIT_LIST_HEAD(&nfc->chips);
2957 
2958 	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
2959 	if (IS_ERR(nfc->regs))
2960 		return PTR_ERR(nfc->regs);
2961 
2962 	irq = platform_get_irq(pdev, 0);
2963 	if (irq < 0)
2964 		return irq;
2965 
2966 	nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2967 
2968 	/* Managed the legacy case (when the first clock was not named) */
2969 	if (nfc->core_clk == ERR_PTR(-ENOENT))
2970 		nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2971 
2972 	if (IS_ERR(nfc->core_clk))
2973 		return PTR_ERR(nfc->core_clk);
2974 
2975 	ret = clk_prepare_enable(nfc->core_clk);
2976 	if (ret)
2977 		return ret;
2978 
2979 	nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2980 	if (IS_ERR(nfc->reg_clk)) {
2981 		if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2982 			ret = PTR_ERR(nfc->reg_clk);
2983 			goto unprepare_core_clk;
2984 		}
2985 
2986 		nfc->reg_clk = NULL;
2987 	}
2988 
2989 	ret = clk_prepare_enable(nfc->reg_clk);
2990 	if (ret)
2991 		goto unprepare_core_clk;
2992 
2993 	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2994 	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2995 	ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2996 			       0, "marvell-nfc", nfc);
2997 	if (ret)
2998 		goto unprepare_reg_clk;
2999 
3000 	/* Get NAND controller capabilities */
3001 	if (pdev->id_entry)
3002 		nfc->caps = (void *)pdev->id_entry->driver_data;
3003 	else
3004 		nfc->caps = of_device_get_match_data(&pdev->dev);
3005 
3006 	if (!nfc->caps) {
3007 		dev_err(dev, "Could not retrieve NFC caps\n");
3008 		ret = -EINVAL;
3009 		goto unprepare_reg_clk;
3010 	}
3011 
3012 	/* Init the controller and then probe the chips */
3013 	ret = marvell_nfc_init(nfc);
3014 	if (ret)
3015 		goto unprepare_reg_clk;
3016 
3017 	platform_set_drvdata(pdev, nfc);
3018 
3019 	ret = marvell_nand_chips_init(dev, nfc);
3020 	if (ret)
3021 		goto release_dma;
3022 
3023 	return 0;
3024 
3025 release_dma:
3026 	if (nfc->use_dma)
3027 		dma_release_channel(nfc->dma_chan);
3028 unprepare_reg_clk:
3029 	clk_disable_unprepare(nfc->reg_clk);
3030 unprepare_core_clk:
3031 	clk_disable_unprepare(nfc->core_clk);
3032 
3033 	return ret;
3034 }
3035 
3036 static void marvell_nfc_remove(struct platform_device *pdev)
3037 {
3038 	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
3039 
3040 	marvell_nand_chips_cleanup(nfc);
3041 
3042 	if (nfc->use_dma) {
3043 		dmaengine_terminate_all(nfc->dma_chan);
3044 		dma_release_channel(nfc->dma_chan);
3045 	}
3046 
3047 	clk_disable_unprepare(nfc->reg_clk);
3048 	clk_disable_unprepare(nfc->core_clk);
3049 }
3050 
3051 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
3052 {
3053 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3054 	struct marvell_nand_chip *chip;
3055 
3056 	list_for_each_entry(chip, &nfc->chips, node)
3057 		marvell_nfc_wait_ndrun(&chip->chip);
3058 
3059 	clk_disable_unprepare(nfc->reg_clk);
3060 	clk_disable_unprepare(nfc->core_clk);
3061 
3062 	return 0;
3063 }
3064 
3065 static int __maybe_unused marvell_nfc_resume(struct device *dev)
3066 {
3067 	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3068 	int ret;
3069 
3070 	ret = clk_prepare_enable(nfc->core_clk);
3071 	if (ret < 0)
3072 		return ret;
3073 
3074 	ret = clk_prepare_enable(nfc->reg_clk);
3075 	if (ret < 0) {
3076 		clk_disable_unprepare(nfc->core_clk);
3077 		return ret;
3078 	}
3079 
3080 	/*
3081 	 * Reset nfc->selected_chip so the next command will cause the timing
3082 	 * registers to be restored in marvell_nfc_select_target().
3083 	 */
3084 	nfc->selected_chip = NULL;
3085 
3086 	/* Reset registers that have lost their contents */
3087 	marvell_nfc_reset(nfc);
3088 
3089 	return 0;
3090 }
3091 
3092 static const struct dev_pm_ops marvell_nfc_pm_ops = {
3093 	SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3094 };
3095 
3096 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3097 	.max_cs_nb = 4,
3098 	.max_rb_nb = 2,
3099 	.need_system_controller = true,
3100 	.is_nfcv2 = true,
3101 };
3102 
3103 static const struct marvell_nfc_caps marvell_ac5_caps = {
3104 	.max_cs_nb = 2,
3105 	.max_rb_nb = 1,
3106 	.is_nfcv2 = true,
3107 	.max_mode_number = 3,
3108 };
3109 
3110 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3111 	.max_cs_nb = 4,
3112 	.max_rb_nb = 2,
3113 	.is_nfcv2 = true,
3114 };
3115 
3116 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3117 	.max_cs_nb = 2,
3118 	.max_rb_nb = 1,
3119 	.use_dma = true,
3120 };
3121 
3122 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3123 	.max_cs_nb = 4,
3124 	.max_rb_nb = 2,
3125 	.need_system_controller = true,
3126 	.legacy_of_bindings = true,
3127 	.is_nfcv2 = true,
3128 };
3129 
3130 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3131 	.max_cs_nb = 4,
3132 	.max_rb_nb = 2,
3133 	.legacy_of_bindings = true,
3134 	.is_nfcv2 = true,
3135 };
3136 
3137 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3138 	.max_cs_nb = 2,
3139 	.max_rb_nb = 1,
3140 	.legacy_of_bindings = true,
3141 	.use_dma = true,
3142 };
3143 
3144 static const struct platform_device_id marvell_nfc_platform_ids[] = {
3145 	{
3146 		.name = "pxa3xx-nand",
3147 		.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3148 	},
3149 	{ /* sentinel */ },
3150 };
3151 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3152 
3153 static const struct of_device_id marvell_nfc_of_ids[] = {
3154 	{
3155 		.compatible = "marvell,armada-8k-nand-controller",
3156 		.data = &marvell_armada_8k_nfc_caps,
3157 	},
3158 	{
3159 		.compatible = "marvell,ac5-nand-controller",
3160 		.data = &marvell_ac5_caps,
3161 	},
3162 	{
3163 		.compatible = "marvell,armada370-nand-controller",
3164 		.data = &marvell_armada370_nfc_caps,
3165 	},
3166 	{
3167 		.compatible = "marvell,pxa3xx-nand-controller",
3168 		.data = &marvell_pxa3xx_nfc_caps,
3169 	},
3170 	/* Support for old/deprecated bindings: */
3171 	{
3172 		.compatible = "marvell,armada-8k-nand",
3173 		.data = &marvell_armada_8k_nfc_legacy_caps,
3174 	},
3175 	{
3176 		.compatible = "marvell,armada370-nand",
3177 		.data = &marvell_armada370_nfc_legacy_caps,
3178 	},
3179 	{
3180 		.compatible = "marvell,pxa3xx-nand",
3181 		.data = &marvell_pxa3xx_nfc_legacy_caps,
3182 	},
3183 	{ /* sentinel */ },
3184 };
3185 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3186 
3187 static struct platform_driver marvell_nfc_driver = {
3188 	.driver	= {
3189 		.name		= "marvell-nfc",
3190 		.of_match_table = marvell_nfc_of_ids,
3191 		.pm		= &marvell_nfc_pm_ops,
3192 	},
3193 	.id_table = marvell_nfc_platform_ids,
3194 	.probe = marvell_nfc_probe,
3195 	.remove_new = marvell_nfc_remove,
3196 };
3197 module_platform_driver(marvell_nfc_driver);
3198 
3199 MODULE_LICENSE("GPL");
3200 MODULE_DESCRIPTION("Marvell NAND controller driver");
3201