xref: /linux/drivers/mtd/nand/raw/qcom_nandc.c (revision bfd5bb6f90af092aa345b15cd78143956a13c2a8)
1 /*
2  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
3  *
4  * This software is licensed under the terms of the GNU General Public
5  * License version 2, as published by the Free Software Foundation, and
6  * may be copied, distributed, and modified under those terms.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  */
13 
14 #include <linux/clk.h>
15 #include <linux/slab.h>
16 #include <linux/bitops.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/dmaengine.h>
19 #include <linux/module.h>
20 #include <linux/mtd/rawnand.h>
21 #include <linux/mtd/partitions.h>
22 #include <linux/of.h>
23 #include <linux/of_device.h>
24 #include <linux/delay.h>
25 #include <linux/dma/qcom_bam_dma.h>
26 #include <linux/dma-direct.h> /* XXX: drivers shall never use this directly! */
27 
28 /* NANDc reg offsets */
29 #define	NAND_FLASH_CMD			0x00
30 #define	NAND_ADDR0			0x04
31 #define	NAND_ADDR1			0x08
32 #define	NAND_FLASH_CHIP_SELECT		0x0c
33 #define	NAND_EXEC_CMD			0x10
34 #define	NAND_FLASH_STATUS		0x14
35 #define	NAND_BUFFER_STATUS		0x18
36 #define	NAND_DEV0_CFG0			0x20
37 #define	NAND_DEV0_CFG1			0x24
38 #define	NAND_DEV0_ECC_CFG		0x28
39 #define	NAND_DEV1_ECC_CFG		0x2c
40 #define	NAND_DEV1_CFG0			0x30
41 #define	NAND_DEV1_CFG1			0x34
42 #define	NAND_READ_ID			0x40
43 #define	NAND_READ_STATUS		0x44
44 #define	NAND_DEV_CMD0			0xa0
45 #define	NAND_DEV_CMD1			0xa4
46 #define	NAND_DEV_CMD2			0xa8
47 #define	NAND_DEV_CMD_VLD		0xac
48 #define	SFLASHC_BURST_CFG		0xe0
49 #define	NAND_ERASED_CW_DETECT_CFG	0xe8
50 #define	NAND_ERASED_CW_DETECT_STATUS	0xec
51 #define	NAND_EBI2_ECC_BUF_CFG		0xf0
52 #define	FLASH_BUF_ACC			0x100
53 
54 #define	NAND_CTRL			0xf00
55 #define	NAND_VERSION			0xf08
56 #define	NAND_READ_LOCATION_0		0xf20
57 #define	NAND_READ_LOCATION_1		0xf24
58 #define	NAND_READ_LOCATION_2		0xf28
59 #define	NAND_READ_LOCATION_3		0xf2c
60 
61 /* dummy register offsets, used by write_reg_dma */
62 #define	NAND_DEV_CMD1_RESTORE		0xdead
63 #define	NAND_DEV_CMD_VLD_RESTORE	0xbeef
64 
65 /* NAND_FLASH_CMD bits */
66 #define	PAGE_ACC			BIT(4)
67 #define	LAST_PAGE			BIT(5)
68 
69 /* NAND_FLASH_CHIP_SELECT bits */
70 #define	NAND_DEV_SEL			0
71 #define	DM_EN				BIT(2)
72 
73 /* NAND_FLASH_STATUS bits */
74 #define	FS_OP_ERR			BIT(4)
75 #define	FS_READY_BSY_N			BIT(5)
76 #define	FS_MPU_ERR			BIT(8)
77 #define	FS_DEVICE_STS_ERR		BIT(16)
78 #define	FS_DEVICE_WP			BIT(23)
79 
80 /* NAND_BUFFER_STATUS bits */
81 #define	BS_UNCORRECTABLE_BIT		BIT(8)
82 #define	BS_CORRECTABLE_ERR_MSK		0x1f
83 
84 /* NAND_DEVn_CFG0 bits */
85 #define	DISABLE_STATUS_AFTER_WRITE	4
86 #define	CW_PER_PAGE			6
87 #define	UD_SIZE_BYTES			9
88 #define	ECC_PARITY_SIZE_BYTES_RS	19
89 #define	SPARE_SIZE_BYTES		23
90 #define	NUM_ADDR_CYCLES			27
91 #define	STATUS_BFR_READ			30
92 #define	SET_RD_MODE_AFTER_STATUS	31
93 
94 /* NAND_DEVn_CFG0 bits */
95 #define	DEV0_CFG1_ECC_DISABLE		0
96 #define	WIDE_FLASH			1
97 #define	NAND_RECOVERY_CYCLES		2
98 #define	CS_ACTIVE_BSY			5
99 #define	BAD_BLOCK_BYTE_NUM		6
100 #define	BAD_BLOCK_IN_SPARE_AREA		16
101 #define	WR_RD_BSY_GAP			17
102 #define	ENABLE_BCH_ECC			27
103 
104 /* NAND_DEV0_ECC_CFG bits */
105 #define	ECC_CFG_ECC_DISABLE		0
106 #define	ECC_SW_RESET			1
107 #define	ECC_MODE			4
108 #define	ECC_PARITY_SIZE_BYTES_BCH	8
109 #define	ECC_NUM_DATA_BYTES		16
110 #define	ECC_FORCE_CLK_OPEN		30
111 
112 /* NAND_DEV_CMD1 bits */
113 #define	READ_ADDR			0
114 
115 /* NAND_DEV_CMD_VLD bits */
116 #define	READ_START_VLD			BIT(0)
117 #define	READ_STOP_VLD			BIT(1)
118 #define	WRITE_START_VLD			BIT(2)
119 #define	ERASE_START_VLD			BIT(3)
120 #define	SEQ_READ_START_VLD		BIT(4)
121 
122 /* NAND_EBI2_ECC_BUF_CFG bits */
123 #define	NUM_STEPS			0
124 
125 /* NAND_ERASED_CW_DETECT_CFG bits */
126 #define	ERASED_CW_ECC_MASK		1
127 #define	AUTO_DETECT_RES			0
128 #define	MASK_ECC			(1 << ERASED_CW_ECC_MASK)
129 #define	RESET_ERASED_DET		(1 << AUTO_DETECT_RES)
130 #define	ACTIVE_ERASED_DET		(0 << AUTO_DETECT_RES)
131 #define	CLR_ERASED_PAGE_DET		(RESET_ERASED_DET | MASK_ECC)
132 #define	SET_ERASED_PAGE_DET		(ACTIVE_ERASED_DET | MASK_ECC)
133 
134 /* NAND_ERASED_CW_DETECT_STATUS bits */
135 #define	PAGE_ALL_ERASED			BIT(7)
136 #define	CODEWORD_ALL_ERASED		BIT(6)
137 #define	PAGE_ERASED			BIT(5)
138 #define	CODEWORD_ERASED			BIT(4)
139 #define	ERASED_PAGE			(PAGE_ALL_ERASED | PAGE_ERASED)
140 #define	ERASED_CW			(CODEWORD_ALL_ERASED | CODEWORD_ERASED)
141 
142 /* NAND_READ_LOCATION_n bits */
143 #define READ_LOCATION_OFFSET		0
144 #define READ_LOCATION_SIZE		16
145 #define READ_LOCATION_LAST		31
146 
147 /* Version Mask */
148 #define	NAND_VERSION_MAJOR_MASK		0xf0000000
149 #define	NAND_VERSION_MAJOR_SHIFT	28
150 #define	NAND_VERSION_MINOR_MASK		0x0fff0000
151 #define	NAND_VERSION_MINOR_SHIFT	16
152 
153 /* NAND OP_CMDs */
154 #define	PAGE_READ			0x2
155 #define	PAGE_READ_WITH_ECC		0x3
156 #define	PAGE_READ_WITH_ECC_SPARE	0x4
157 #define	PROGRAM_PAGE			0x6
158 #define	PAGE_PROGRAM_WITH_ECC		0x7
159 #define	PROGRAM_PAGE_SPARE		0x9
160 #define	BLOCK_ERASE			0xa
161 #define	FETCH_ID			0xb
162 #define	RESET_DEVICE			0xd
163 
164 /* Default Value for NAND_DEV_CMD_VLD */
165 #define NAND_DEV_CMD_VLD_VAL		(READ_START_VLD | WRITE_START_VLD | \
166 					 ERASE_START_VLD | SEQ_READ_START_VLD)
167 
168 /* NAND_CTRL bits */
169 #define	BAM_MODE_EN			BIT(0)
170 
171 /*
172  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
173  * the driver calls the chunks 'step' or 'codeword' interchangeably
174  */
175 #define	NANDC_STEP_SIZE			512
176 
177 /*
178  * the largest page size we support is 8K, this will have 16 steps/codewords
179  * of 512 bytes each
180  */
181 #define	MAX_NUM_STEPS			(SZ_8K / NANDC_STEP_SIZE)
182 
183 /* we read at most 3 registers per codeword scan */
184 #define	MAX_REG_RD			(3 * MAX_NUM_STEPS)
185 
186 /* ECC modes supported by the controller */
187 #define	ECC_NONE	BIT(0)
188 #define	ECC_RS_4BIT	BIT(1)
189 #define	ECC_BCH_4BIT	BIT(2)
190 #define	ECC_BCH_8BIT	BIT(3)
191 
192 #define nandc_set_read_loc(nandc, reg, offset, size, is_last)	\
193 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg,			\
194 	      ((offset) << READ_LOCATION_OFFSET) |		\
195 	      ((size) << READ_LOCATION_SIZE) |			\
196 	      ((is_last) << READ_LOCATION_LAST))
197 
198 /*
199  * Returns the actual register address for all NAND_DEV_ registers
200  * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
201  */
202 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
203 
204 /* Returns the NAND register physical address */
205 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
206 
207 /* Returns the dma address for reg read buffer */
208 #define reg_buf_dma_addr(chip, vaddr) \
209 	((chip)->reg_read_dma + \
210 	((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
211 
212 #define QPIC_PER_CW_CMD_ELEMENTS	32
213 #define QPIC_PER_CW_CMD_SGL		32
214 #define QPIC_PER_CW_DATA_SGL		8
215 
216 /*
217  * Flags used in DMA descriptor preparation helper functions
218  * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
219  */
220 /* Don't set the EOT in current tx BAM sgl */
221 #define NAND_BAM_NO_EOT			BIT(0)
222 /* Set the NWD flag in current BAM sgl */
223 #define NAND_BAM_NWD			BIT(1)
224 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
225 #define NAND_BAM_NEXT_SGL		BIT(2)
226 /*
227  * Erased codeword status is being used two times in single transfer so this
228  * flag will determine the current value of erased codeword status register
229  */
230 #define NAND_ERASED_CW_SET		BIT(4)
231 
232 /*
233  * This data type corresponds to the BAM transaction which will be used for all
234  * NAND transfers.
235  * @bam_ce - the array of BAM command elements
236  * @cmd_sgl - sgl for NAND BAM command pipe
237  * @data_sgl - sgl for NAND BAM consumer/producer pipe
238  * @bam_ce_pos - the index in bam_ce which is available for next sgl
239  * @bam_ce_start - the index in bam_ce which marks the start position ce
240  *		   for current sgl. It will be used for size calculation
241  *		   for current sgl
242  * @cmd_sgl_pos - current index in command sgl.
243  * @cmd_sgl_start - start index in command sgl.
244  * @tx_sgl_pos - current index in data sgl for tx.
245  * @tx_sgl_start - start index in data sgl for tx.
246  * @rx_sgl_pos - current index in data sgl for rx.
247  * @rx_sgl_start - start index in data sgl for rx.
248  */
249 struct bam_transaction {
250 	struct bam_cmd_element *bam_ce;
251 	struct scatterlist *cmd_sgl;
252 	struct scatterlist *data_sgl;
253 	u32 bam_ce_pos;
254 	u32 bam_ce_start;
255 	u32 cmd_sgl_pos;
256 	u32 cmd_sgl_start;
257 	u32 tx_sgl_pos;
258 	u32 tx_sgl_start;
259 	u32 rx_sgl_pos;
260 	u32 rx_sgl_start;
261 };
262 
263 /*
264  * This data type corresponds to the nand dma descriptor
265  * @list - list for desc_info
266  * @dir - DMA transfer direction
267  * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
268  *	      ADM
269  * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
270  * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
271  * @dma_desc - low level DMA engine descriptor
272  */
273 struct desc_info {
274 	struct list_head node;
275 
276 	enum dma_data_direction dir;
277 	union {
278 		struct scatterlist adm_sgl;
279 		struct {
280 			struct scatterlist *bam_sgl;
281 			int sgl_cnt;
282 		};
283 	};
284 	struct dma_async_tx_descriptor *dma_desc;
285 };
286 
287 /*
288  * holds the current register values that we want to write. acts as a contiguous
289  * chunk of memory which we use to write the controller registers through DMA.
290  */
291 struct nandc_regs {
292 	__le32 cmd;
293 	__le32 addr0;
294 	__le32 addr1;
295 	__le32 chip_sel;
296 	__le32 exec;
297 
298 	__le32 cfg0;
299 	__le32 cfg1;
300 	__le32 ecc_bch_cfg;
301 
302 	__le32 clrflashstatus;
303 	__le32 clrreadstatus;
304 
305 	__le32 cmd1;
306 	__le32 vld;
307 
308 	__le32 orig_cmd1;
309 	__le32 orig_vld;
310 
311 	__le32 ecc_buf_cfg;
312 	__le32 read_location0;
313 	__le32 read_location1;
314 	__le32 read_location2;
315 	__le32 read_location3;
316 
317 	__le32 erased_cw_detect_cfg_clr;
318 	__le32 erased_cw_detect_cfg_set;
319 };
320 
321 /*
322  * NAND controller data struct
323  *
324  * @controller:			base controller structure
325  * @host_list:			list containing all the chips attached to the
326  *				controller
327  * @dev:			parent device
328  * @base:			MMIO base
329  * @base_phys:			physical base address of controller registers
330  * @base_dma:			dma base address of controller registers
331  * @core_clk:			controller clock
332  * @aon_clk:			another controller clock
333  *
334  * @chan:			dma channel
335  * @cmd_crci:			ADM DMA CRCI for command flow control
336  * @data_crci:			ADM DMA CRCI for data flow control
337  * @desc_list:			DMA descriptor list (list of desc_infos)
338  *
339  * @data_buffer:		our local DMA buffer for page read/writes,
340  *				used when we can't use the buffer provided
341  *				by upper layers directly
342  * @buf_size/count/start:	markers for chip->read_buf/write_buf functions
343  * @reg_read_buf:		local buffer for reading back registers via DMA
344  * @reg_read_dma:		contains dma address for register read buffer
345  * @reg_read_pos:		marker for data read in reg_read_buf
346  *
347  * @regs:			a contiguous chunk of memory for DMA register
348  *				writes. contains the register values to be
349  *				written to controller
350  * @cmd1/vld:			some fixed controller register values
351  * @props:			properties of current NAND controller,
352  *				initialized via DT match data
353  * @max_cwperpage:		maximum QPIC codewords required. calculated
354  *				from all connected NAND devices pagesize
355  */
356 struct qcom_nand_controller {
357 	struct nand_hw_control controller;
358 	struct list_head host_list;
359 
360 	struct device *dev;
361 
362 	void __iomem *base;
363 	phys_addr_t base_phys;
364 	dma_addr_t base_dma;
365 
366 	struct clk *core_clk;
367 	struct clk *aon_clk;
368 
369 	union {
370 		/* will be used only by QPIC for BAM DMA */
371 		struct {
372 			struct dma_chan *tx_chan;
373 			struct dma_chan *rx_chan;
374 			struct dma_chan *cmd_chan;
375 		};
376 
377 		/* will be used only by EBI2 for ADM DMA */
378 		struct {
379 			struct dma_chan *chan;
380 			unsigned int cmd_crci;
381 			unsigned int data_crci;
382 		};
383 	};
384 
385 	struct list_head desc_list;
386 	struct bam_transaction *bam_txn;
387 
388 	u8		*data_buffer;
389 	int		buf_size;
390 	int		buf_count;
391 	int		buf_start;
392 	unsigned int	max_cwperpage;
393 
394 	__le32 *reg_read_buf;
395 	dma_addr_t reg_read_dma;
396 	int reg_read_pos;
397 
398 	struct nandc_regs *regs;
399 
400 	u32 cmd1, vld;
401 	const struct qcom_nandc_props *props;
402 };
403 
404 /*
405  * NAND chip structure
406  *
407  * @chip:			base NAND chip structure
408  * @node:			list node to add itself to host_list in
409  *				qcom_nand_controller
410  *
411  * @cs:				chip select value for this chip
412  * @cw_size:			the number of bytes in a single step/codeword
413  *				of a page, consisting of all data, ecc, spare
414  *				and reserved bytes
415  * @cw_data:			the number of bytes within a codeword protected
416  *				by ECC
417  * @use_ecc:			request the controller to use ECC for the
418  *				upcoming read/write
419  * @bch_enabled:		flag to tell whether BCH ECC mode is used
420  * @ecc_bytes_hw:		ECC bytes used by controller hardware for this
421  *				chip
422  * @status:			value to be returned if NAND_CMD_STATUS command
423  *				is executed
424  * @last_command:		keeps track of last command on this chip. used
425  *				for reading correct status
426  *
427  * @cfg0, cfg1, cfg0_raw..:	NANDc register configurations needed for
428  *				ecc/non-ecc mode for the current nand flash
429  *				device
430  */
431 struct qcom_nand_host {
432 	struct nand_chip chip;
433 	struct list_head node;
434 
435 	int cs;
436 	int cw_size;
437 	int cw_data;
438 	bool use_ecc;
439 	bool bch_enabled;
440 	int ecc_bytes_hw;
441 	int spare_bytes;
442 	int bbm_size;
443 	u8 status;
444 	int last_command;
445 
446 	u32 cfg0, cfg1;
447 	u32 cfg0_raw, cfg1_raw;
448 	u32 ecc_buf_cfg;
449 	u32 ecc_bch_cfg;
450 	u32 clrflashstatus;
451 	u32 clrreadstatus;
452 };
453 
454 /*
455  * This data type corresponds to the NAND controller properties which varies
456  * among different NAND controllers.
457  * @ecc_modes - ecc mode for NAND
458  * @is_bam - whether NAND controller is using BAM
459  * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
460  */
461 struct qcom_nandc_props {
462 	u32 ecc_modes;
463 	bool is_bam;
464 	u32 dev_cmd_reg_start;
465 };
466 
467 /* Frees the BAM transaction memory */
468 static void free_bam_transaction(struct qcom_nand_controller *nandc)
469 {
470 	struct bam_transaction *bam_txn = nandc->bam_txn;
471 
472 	devm_kfree(nandc->dev, bam_txn);
473 }
474 
475 /* Allocates and Initializes the BAM transaction */
476 static struct bam_transaction *
477 alloc_bam_transaction(struct qcom_nand_controller *nandc)
478 {
479 	struct bam_transaction *bam_txn;
480 	size_t bam_txn_size;
481 	unsigned int num_cw = nandc->max_cwperpage;
482 	void *bam_txn_buf;
483 
484 	bam_txn_size =
485 		sizeof(*bam_txn) + num_cw *
486 		((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
487 		(sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
488 		(sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
489 
490 	bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
491 	if (!bam_txn_buf)
492 		return NULL;
493 
494 	bam_txn = bam_txn_buf;
495 	bam_txn_buf += sizeof(*bam_txn);
496 
497 	bam_txn->bam_ce = bam_txn_buf;
498 	bam_txn_buf +=
499 		sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
500 
501 	bam_txn->cmd_sgl = bam_txn_buf;
502 	bam_txn_buf +=
503 		sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
504 
505 	bam_txn->data_sgl = bam_txn_buf;
506 
507 	return bam_txn;
508 }
509 
510 /* Clears the BAM transaction indexes */
511 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
512 {
513 	struct bam_transaction *bam_txn = nandc->bam_txn;
514 
515 	if (!nandc->props->is_bam)
516 		return;
517 
518 	bam_txn->bam_ce_pos = 0;
519 	bam_txn->bam_ce_start = 0;
520 	bam_txn->cmd_sgl_pos = 0;
521 	bam_txn->cmd_sgl_start = 0;
522 	bam_txn->tx_sgl_pos = 0;
523 	bam_txn->tx_sgl_start = 0;
524 	bam_txn->rx_sgl_pos = 0;
525 	bam_txn->rx_sgl_start = 0;
526 
527 	sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
528 		      QPIC_PER_CW_CMD_SGL);
529 	sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
530 		      QPIC_PER_CW_DATA_SGL);
531 }
532 
533 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
534 {
535 	return container_of(chip, struct qcom_nand_host, chip);
536 }
537 
538 static inline struct qcom_nand_controller *
539 get_qcom_nand_controller(struct nand_chip *chip)
540 {
541 	return container_of(chip->controller, struct qcom_nand_controller,
542 			    controller);
543 }
544 
545 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
546 {
547 	return ioread32(nandc->base + offset);
548 }
549 
550 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
551 			       u32 val)
552 {
553 	iowrite32(val, nandc->base + offset);
554 }
555 
556 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
557 					  bool is_cpu)
558 {
559 	if (!nandc->props->is_bam)
560 		return;
561 
562 	if (is_cpu)
563 		dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
564 					MAX_REG_RD *
565 					sizeof(*nandc->reg_read_buf),
566 					DMA_FROM_DEVICE);
567 	else
568 		dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
569 					   MAX_REG_RD *
570 					   sizeof(*nandc->reg_read_buf),
571 					   DMA_FROM_DEVICE);
572 }
573 
574 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
575 {
576 	switch (offset) {
577 	case NAND_FLASH_CMD:
578 		return &regs->cmd;
579 	case NAND_ADDR0:
580 		return &regs->addr0;
581 	case NAND_ADDR1:
582 		return &regs->addr1;
583 	case NAND_FLASH_CHIP_SELECT:
584 		return &regs->chip_sel;
585 	case NAND_EXEC_CMD:
586 		return &regs->exec;
587 	case NAND_FLASH_STATUS:
588 		return &regs->clrflashstatus;
589 	case NAND_DEV0_CFG0:
590 		return &regs->cfg0;
591 	case NAND_DEV0_CFG1:
592 		return &regs->cfg1;
593 	case NAND_DEV0_ECC_CFG:
594 		return &regs->ecc_bch_cfg;
595 	case NAND_READ_STATUS:
596 		return &regs->clrreadstatus;
597 	case NAND_DEV_CMD1:
598 		return &regs->cmd1;
599 	case NAND_DEV_CMD1_RESTORE:
600 		return &regs->orig_cmd1;
601 	case NAND_DEV_CMD_VLD:
602 		return &regs->vld;
603 	case NAND_DEV_CMD_VLD_RESTORE:
604 		return &regs->orig_vld;
605 	case NAND_EBI2_ECC_BUF_CFG:
606 		return &regs->ecc_buf_cfg;
607 	case NAND_READ_LOCATION_0:
608 		return &regs->read_location0;
609 	case NAND_READ_LOCATION_1:
610 		return &regs->read_location1;
611 	case NAND_READ_LOCATION_2:
612 		return &regs->read_location2;
613 	case NAND_READ_LOCATION_3:
614 		return &regs->read_location3;
615 	default:
616 		return NULL;
617 	}
618 }
619 
620 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
621 			  u32 val)
622 {
623 	struct nandc_regs *regs = nandc->regs;
624 	__le32 *reg;
625 
626 	reg = offset_to_nandc_reg(regs, offset);
627 
628 	if (reg)
629 		*reg = cpu_to_le32(val);
630 }
631 
632 /* helper to configure address register values */
633 static void set_address(struct qcom_nand_host *host, u16 column, int page)
634 {
635 	struct nand_chip *chip = &host->chip;
636 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
637 
638 	if (chip->options & NAND_BUSWIDTH_16)
639 		column >>= 1;
640 
641 	nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
642 	nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
643 }
644 
645 /*
646  * update_rw_regs:	set up read/write register values, these will be
647  *			written to the NAND controller registers via DMA
648  *
649  * @num_cw:		number of steps for the read/write operation
650  * @read:		read or write operation
651  */
652 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
653 {
654 	struct nand_chip *chip = &host->chip;
655 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
656 	u32 cmd, cfg0, cfg1, ecc_bch_cfg;
657 
658 	if (read) {
659 		if (host->use_ecc)
660 			cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
661 		else
662 			cmd = PAGE_READ | PAGE_ACC | LAST_PAGE;
663 	} else {
664 			cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
665 	}
666 
667 	if (host->use_ecc) {
668 		cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
669 				(num_cw - 1) << CW_PER_PAGE;
670 
671 		cfg1 = host->cfg1;
672 		ecc_bch_cfg = host->ecc_bch_cfg;
673 	} else {
674 		cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
675 				(num_cw - 1) << CW_PER_PAGE;
676 
677 		cfg1 = host->cfg1_raw;
678 		ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
679 	}
680 
681 	nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
682 	nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
683 	nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
684 	nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
685 	nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
686 	nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
687 	nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
688 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
689 
690 	if (read)
691 		nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
692 				   host->cw_data : host->cw_size, 1);
693 }
694 
695 /*
696  * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
697  * for BAM. This descriptor will be added in the NAND DMA descriptor queue
698  * which will be submitted to DMA engine.
699  */
700 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
701 				  struct dma_chan *chan,
702 				  unsigned long flags)
703 {
704 	struct desc_info *desc;
705 	struct scatterlist *sgl;
706 	unsigned int sgl_cnt;
707 	int ret;
708 	struct bam_transaction *bam_txn = nandc->bam_txn;
709 	enum dma_transfer_direction dir_eng;
710 	struct dma_async_tx_descriptor *dma_desc;
711 
712 	desc = kzalloc(sizeof(*desc), GFP_KERNEL);
713 	if (!desc)
714 		return -ENOMEM;
715 
716 	if (chan == nandc->cmd_chan) {
717 		sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
718 		sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
719 		bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
720 		dir_eng = DMA_MEM_TO_DEV;
721 		desc->dir = DMA_TO_DEVICE;
722 	} else if (chan == nandc->tx_chan) {
723 		sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
724 		sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
725 		bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
726 		dir_eng = DMA_MEM_TO_DEV;
727 		desc->dir = DMA_TO_DEVICE;
728 	} else {
729 		sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
730 		sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
731 		bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
732 		dir_eng = DMA_DEV_TO_MEM;
733 		desc->dir = DMA_FROM_DEVICE;
734 	}
735 
736 	sg_mark_end(sgl + sgl_cnt - 1);
737 	ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
738 	if (ret == 0) {
739 		dev_err(nandc->dev, "failure in mapping desc\n");
740 		kfree(desc);
741 		return -ENOMEM;
742 	}
743 
744 	desc->sgl_cnt = sgl_cnt;
745 	desc->bam_sgl = sgl;
746 
747 	dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
748 					   flags);
749 
750 	if (!dma_desc) {
751 		dev_err(nandc->dev, "failure in prep desc\n");
752 		dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
753 		kfree(desc);
754 		return -EINVAL;
755 	}
756 
757 	desc->dma_desc = dma_desc;
758 
759 	list_add_tail(&desc->node, &nandc->desc_list);
760 
761 	return 0;
762 }
763 
764 /*
765  * Prepares the command descriptor for BAM DMA which will be used for NAND
766  * register reads and writes. The command descriptor requires the command
767  * to be formed in command element type so this function uses the command
768  * element from bam transaction ce array and fills the same with required
769  * data. A single SGL can contain multiple command elements so
770  * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
771  * after the current command element.
772  */
773 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
774 				 int reg_off, const void *vaddr,
775 				 int size, unsigned int flags)
776 {
777 	int bam_ce_size;
778 	int i, ret;
779 	struct bam_cmd_element *bam_ce_buffer;
780 	struct bam_transaction *bam_txn = nandc->bam_txn;
781 
782 	bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
783 
784 	/* fill the command desc */
785 	for (i = 0; i < size; i++) {
786 		if (read)
787 			bam_prep_ce(&bam_ce_buffer[i],
788 				    nandc_reg_phys(nandc, reg_off + 4 * i),
789 				    BAM_READ_COMMAND,
790 				    reg_buf_dma_addr(nandc,
791 						     (__le32 *)vaddr + i));
792 		else
793 			bam_prep_ce_le32(&bam_ce_buffer[i],
794 					 nandc_reg_phys(nandc, reg_off + 4 * i),
795 					 BAM_WRITE_COMMAND,
796 					 *((__le32 *)vaddr + i));
797 	}
798 
799 	bam_txn->bam_ce_pos += size;
800 
801 	/* use the separate sgl after this command */
802 	if (flags & NAND_BAM_NEXT_SGL) {
803 		bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
804 		bam_ce_size = (bam_txn->bam_ce_pos -
805 				bam_txn->bam_ce_start) *
806 				sizeof(struct bam_cmd_element);
807 		sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
808 			   bam_ce_buffer, bam_ce_size);
809 		bam_txn->cmd_sgl_pos++;
810 		bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
811 
812 		if (flags & NAND_BAM_NWD) {
813 			ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
814 						     DMA_PREP_FENCE |
815 						     DMA_PREP_CMD);
816 			if (ret)
817 				return ret;
818 		}
819 	}
820 
821 	return 0;
822 }
823 
824 /*
825  * Prepares the data descriptor for BAM DMA which will be used for NAND
826  * data reads and writes.
827  */
828 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
829 				  const void *vaddr,
830 				  int size, unsigned int flags)
831 {
832 	int ret;
833 	struct bam_transaction *bam_txn = nandc->bam_txn;
834 
835 	if (read) {
836 		sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
837 			   vaddr, size);
838 		bam_txn->rx_sgl_pos++;
839 	} else {
840 		sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
841 			   vaddr, size);
842 		bam_txn->tx_sgl_pos++;
843 
844 		/*
845 		 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
846 		 * is not set, form the DMA descriptor
847 		 */
848 		if (!(flags & NAND_BAM_NO_EOT)) {
849 			ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
850 						     DMA_PREP_INTERRUPT);
851 			if (ret)
852 				return ret;
853 		}
854 	}
855 
856 	return 0;
857 }
858 
859 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
860 			     int reg_off, const void *vaddr, int size,
861 			     bool flow_control)
862 {
863 	struct desc_info *desc;
864 	struct dma_async_tx_descriptor *dma_desc;
865 	struct scatterlist *sgl;
866 	struct dma_slave_config slave_conf;
867 	enum dma_transfer_direction dir_eng;
868 	int ret;
869 
870 	desc = kzalloc(sizeof(*desc), GFP_KERNEL);
871 	if (!desc)
872 		return -ENOMEM;
873 
874 	sgl = &desc->adm_sgl;
875 
876 	sg_init_one(sgl, vaddr, size);
877 
878 	if (read) {
879 		dir_eng = DMA_DEV_TO_MEM;
880 		desc->dir = DMA_FROM_DEVICE;
881 	} else {
882 		dir_eng = DMA_MEM_TO_DEV;
883 		desc->dir = DMA_TO_DEVICE;
884 	}
885 
886 	ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
887 	if (ret == 0) {
888 		ret = -ENOMEM;
889 		goto err;
890 	}
891 
892 	memset(&slave_conf, 0x00, sizeof(slave_conf));
893 
894 	slave_conf.device_fc = flow_control;
895 	if (read) {
896 		slave_conf.src_maxburst = 16;
897 		slave_conf.src_addr = nandc->base_dma + reg_off;
898 		slave_conf.slave_id = nandc->data_crci;
899 	} else {
900 		slave_conf.dst_maxburst = 16;
901 		slave_conf.dst_addr = nandc->base_dma + reg_off;
902 		slave_conf.slave_id = nandc->cmd_crci;
903 	}
904 
905 	ret = dmaengine_slave_config(nandc->chan, &slave_conf);
906 	if (ret) {
907 		dev_err(nandc->dev, "failed to configure dma channel\n");
908 		goto err;
909 	}
910 
911 	dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
912 	if (!dma_desc) {
913 		dev_err(nandc->dev, "failed to prepare desc\n");
914 		ret = -EINVAL;
915 		goto err;
916 	}
917 
918 	desc->dma_desc = dma_desc;
919 
920 	list_add_tail(&desc->node, &nandc->desc_list);
921 
922 	return 0;
923 err:
924 	kfree(desc);
925 
926 	return ret;
927 }
928 
929 /*
930  * read_reg_dma:	prepares a descriptor to read a given number of
931  *			contiguous registers to the reg_read_buf pointer
932  *
933  * @first:		offset of the first register in the contiguous block
934  * @num_regs:		number of registers to read
935  * @flags:		flags to control DMA descriptor preparation
936  */
937 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
938 			int num_regs, unsigned int flags)
939 {
940 	bool flow_control = false;
941 	void *vaddr;
942 
943 	vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
944 	nandc->reg_read_pos += num_regs;
945 
946 	if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
947 		first = dev_cmd_reg_addr(nandc, first);
948 
949 	if (nandc->props->is_bam)
950 		return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
951 					     num_regs, flags);
952 
953 	if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
954 		flow_control = true;
955 
956 	return prep_adm_dma_desc(nandc, true, first, vaddr,
957 				 num_regs * sizeof(u32), flow_control);
958 }
959 
960 /*
961  * write_reg_dma:	prepares a descriptor to write a given number of
962  *			contiguous registers
963  *
964  * @first:		offset of the first register in the contiguous block
965  * @num_regs:		number of registers to write
966  * @flags:		flags to control DMA descriptor preparation
967  */
968 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
969 			 int num_regs, unsigned int flags)
970 {
971 	bool flow_control = false;
972 	struct nandc_regs *regs = nandc->regs;
973 	void *vaddr;
974 
975 	vaddr = offset_to_nandc_reg(regs, first);
976 
977 	if (first == NAND_ERASED_CW_DETECT_CFG) {
978 		if (flags & NAND_ERASED_CW_SET)
979 			vaddr = &regs->erased_cw_detect_cfg_set;
980 		else
981 			vaddr = &regs->erased_cw_detect_cfg_clr;
982 	}
983 
984 	if (first == NAND_EXEC_CMD)
985 		flags |= NAND_BAM_NWD;
986 
987 	if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
988 		first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
989 
990 	if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
991 		first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
992 
993 	if (nandc->props->is_bam)
994 		return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
995 					     num_regs, flags);
996 
997 	if (first == NAND_FLASH_CMD)
998 		flow_control = true;
999 
1000 	return prep_adm_dma_desc(nandc, false, first, vaddr,
1001 				 num_regs * sizeof(u32), flow_control);
1002 }
1003 
1004 /*
1005  * read_data_dma:	prepares a DMA descriptor to transfer data from the
1006  *			controller's internal buffer to the buffer 'vaddr'
1007  *
1008  * @reg_off:		offset within the controller's data buffer
1009  * @vaddr:		virtual address of the buffer we want to write to
1010  * @size:		DMA transaction size in bytes
1011  * @flags:		flags to control DMA descriptor preparation
1012  */
1013 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1014 			 const u8 *vaddr, int size, unsigned int flags)
1015 {
1016 	if (nandc->props->is_bam)
1017 		return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1018 
1019 	return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1020 }
1021 
1022 /*
1023  * write_data_dma:	prepares a DMA descriptor to transfer data from
1024  *			'vaddr' to the controller's internal buffer
1025  *
1026  * @reg_off:		offset within the controller's data buffer
1027  * @vaddr:		virtual address of the buffer we want to read from
1028  * @size:		DMA transaction size in bytes
1029  * @flags:		flags to control DMA descriptor preparation
1030  */
1031 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1032 			  const u8 *vaddr, int size, unsigned int flags)
1033 {
1034 	if (nandc->props->is_bam)
1035 		return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1036 
1037 	return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1038 }
1039 
1040 /*
1041  * Helper to prepare DMA descriptors for configuring registers
1042  * before reading a NAND page.
1043  */
1044 static void config_nand_page_read(struct qcom_nand_controller *nandc)
1045 {
1046 	write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1047 	write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1048 	write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1049 	write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1050 	write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1051 		      NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1052 }
1053 
1054 /*
1055  * Helper to prepare DMA descriptors for configuring registers
1056  * before reading each codeword in NAND page.
1057  */
1058 static void config_nand_cw_read(struct qcom_nand_controller *nandc)
1059 {
1060 	if (nandc->props->is_bam)
1061 		write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
1062 			      NAND_BAM_NEXT_SGL);
1063 
1064 	write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1065 	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1066 
1067 	read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1068 	read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1069 		     NAND_BAM_NEXT_SGL);
1070 }
1071 
1072 /*
1073  * Helper to prepare dma descriptors to configure registers needed for reading a
1074  * single codeword in page
1075  */
1076 static void config_nand_single_cw_page_read(struct qcom_nand_controller *nandc)
1077 {
1078 	config_nand_page_read(nandc);
1079 	config_nand_cw_read(nandc);
1080 }
1081 
1082 /*
1083  * Helper to prepare DMA descriptors used to configure registers needed for
1084  * before writing a NAND page.
1085  */
1086 static void config_nand_page_write(struct qcom_nand_controller *nandc)
1087 {
1088 	write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1089 	write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1090 	write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1091 		      NAND_BAM_NEXT_SGL);
1092 }
1093 
1094 /*
1095  * Helper to prepare DMA descriptors for configuring registers
1096  * before writing each codeword in NAND page.
1097  */
1098 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1099 {
1100 	write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1101 	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1102 
1103 	read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1104 
1105 	write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1106 	write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1107 }
1108 
1109 /*
1110  * the following functions are used within chip->cmdfunc() to perform different
1111  * NAND_CMD_* commands
1112  */
1113 
1114 /* sets up descriptors for NAND_CMD_PARAM */
1115 static int nandc_param(struct qcom_nand_host *host)
1116 {
1117 	struct nand_chip *chip = &host->chip;
1118 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1119 
1120 	/*
1121 	 * NAND_CMD_PARAM is called before we know much about the FLASH chip
1122 	 * in use. we configure the controller to perform a raw read of 512
1123 	 * bytes to read onfi params
1124 	 */
1125 	nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE);
1126 	nandc_set_reg(nandc, NAND_ADDR0, 0);
1127 	nandc_set_reg(nandc, NAND_ADDR1, 0);
1128 	nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1129 					| 512 << UD_SIZE_BYTES
1130 					| 5 << NUM_ADDR_CYCLES
1131 					| 0 << SPARE_SIZE_BYTES);
1132 	nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1133 					| 0 << CS_ACTIVE_BSY
1134 					| 17 << BAD_BLOCK_BYTE_NUM
1135 					| 1 << BAD_BLOCK_IN_SPARE_AREA
1136 					| 2 << WR_RD_BSY_GAP
1137 					| 0 << WIDE_FLASH
1138 					| 1 << DEV0_CFG1_ECC_DISABLE);
1139 	nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1140 
1141 	/* configure CMD1 and VLD for ONFI param probing */
1142 	nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1143 		      (nandc->vld & ~READ_START_VLD));
1144 	nandc_set_reg(nandc, NAND_DEV_CMD1,
1145 		      (nandc->cmd1 & ~(0xFF << READ_ADDR))
1146 		      | NAND_CMD_PARAM << READ_ADDR);
1147 
1148 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1149 
1150 	nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1151 	nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1152 	nandc_set_read_loc(nandc, 0, 0, 512, 1);
1153 
1154 	write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1155 	write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1156 
1157 	nandc->buf_count = 512;
1158 	memset(nandc->data_buffer, 0xff, nandc->buf_count);
1159 
1160 	config_nand_single_cw_page_read(nandc);
1161 
1162 	read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1163 		      nandc->buf_count, 0);
1164 
1165 	/* restore CMD1 and VLD regs */
1166 	write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1167 	write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1168 
1169 	return 0;
1170 }
1171 
1172 /* sets up descriptors for NAND_CMD_ERASE1 */
1173 static int erase_block(struct qcom_nand_host *host, int page_addr)
1174 {
1175 	struct nand_chip *chip = &host->chip;
1176 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1177 
1178 	nandc_set_reg(nandc, NAND_FLASH_CMD,
1179 		      BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1180 	nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1181 	nandc_set_reg(nandc, NAND_ADDR1, 0);
1182 	nandc_set_reg(nandc, NAND_DEV0_CFG0,
1183 		      host->cfg0_raw & ~(7 << CW_PER_PAGE));
1184 	nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1185 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1186 	nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1187 	nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1188 
1189 	write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1190 	write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1191 	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1192 
1193 	read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1194 
1195 	write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1196 	write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1197 
1198 	return 0;
1199 }
1200 
1201 /* sets up descriptors for NAND_CMD_READID */
1202 static int read_id(struct qcom_nand_host *host, int column)
1203 {
1204 	struct nand_chip *chip = &host->chip;
1205 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1206 
1207 	if (column == -1)
1208 		return 0;
1209 
1210 	nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID);
1211 	nandc_set_reg(nandc, NAND_ADDR0, column);
1212 	nandc_set_reg(nandc, NAND_ADDR1, 0);
1213 	nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1214 		      nandc->props->is_bam ? 0 : DM_EN);
1215 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1216 
1217 	write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1218 	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1219 
1220 	read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1221 
1222 	return 0;
1223 }
1224 
1225 /* sets up descriptors for NAND_CMD_RESET */
1226 static int reset(struct qcom_nand_host *host)
1227 {
1228 	struct nand_chip *chip = &host->chip;
1229 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1230 
1231 	nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE);
1232 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1233 
1234 	write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1235 	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1236 
1237 	read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1238 
1239 	return 0;
1240 }
1241 
1242 /* helpers to submit/free our list of dma descriptors */
1243 static int submit_descs(struct qcom_nand_controller *nandc)
1244 {
1245 	struct desc_info *desc;
1246 	dma_cookie_t cookie = 0;
1247 	struct bam_transaction *bam_txn = nandc->bam_txn;
1248 	int r;
1249 
1250 	if (nandc->props->is_bam) {
1251 		if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1252 			r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1253 			if (r)
1254 				return r;
1255 		}
1256 
1257 		if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1258 			r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1259 						   DMA_PREP_INTERRUPT);
1260 			if (r)
1261 				return r;
1262 		}
1263 
1264 		if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1265 			r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1266 						   DMA_PREP_CMD);
1267 			if (r)
1268 				return r;
1269 		}
1270 	}
1271 
1272 	list_for_each_entry(desc, &nandc->desc_list, node)
1273 		cookie = dmaengine_submit(desc->dma_desc);
1274 
1275 	if (nandc->props->is_bam) {
1276 		dma_async_issue_pending(nandc->tx_chan);
1277 		dma_async_issue_pending(nandc->rx_chan);
1278 
1279 		if (dma_sync_wait(nandc->cmd_chan, cookie) != DMA_COMPLETE)
1280 			return -ETIMEDOUT;
1281 	} else {
1282 		if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1283 			return -ETIMEDOUT;
1284 	}
1285 
1286 	return 0;
1287 }
1288 
1289 static void free_descs(struct qcom_nand_controller *nandc)
1290 {
1291 	struct desc_info *desc, *n;
1292 
1293 	list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1294 		list_del(&desc->node);
1295 
1296 		if (nandc->props->is_bam)
1297 			dma_unmap_sg(nandc->dev, desc->bam_sgl,
1298 				     desc->sgl_cnt, desc->dir);
1299 		else
1300 			dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1301 				     desc->dir);
1302 
1303 		kfree(desc);
1304 	}
1305 }
1306 
1307 /* reset the register read buffer for next NAND operation */
1308 static void clear_read_regs(struct qcom_nand_controller *nandc)
1309 {
1310 	nandc->reg_read_pos = 0;
1311 	nandc_read_buffer_sync(nandc, false);
1312 }
1313 
1314 static void pre_command(struct qcom_nand_host *host, int command)
1315 {
1316 	struct nand_chip *chip = &host->chip;
1317 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1318 
1319 	nandc->buf_count = 0;
1320 	nandc->buf_start = 0;
1321 	host->use_ecc = false;
1322 	host->last_command = command;
1323 
1324 	clear_read_regs(nandc);
1325 
1326 	if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1327 	    command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1328 		clear_bam_transaction(nandc);
1329 }
1330 
1331 /*
1332  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1333  * privately maintained status byte, this status byte can be read after
1334  * NAND_CMD_STATUS is called
1335  */
1336 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1337 {
1338 	struct nand_chip *chip = &host->chip;
1339 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1340 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1341 	int num_cw;
1342 	int i;
1343 
1344 	num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1345 	nandc_read_buffer_sync(nandc, true);
1346 
1347 	for (i = 0; i < num_cw; i++) {
1348 		u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1349 
1350 		if (flash_status & FS_MPU_ERR)
1351 			host->status &= ~NAND_STATUS_WP;
1352 
1353 		if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1354 						 (flash_status &
1355 						  FS_DEVICE_STS_ERR)))
1356 			host->status |= NAND_STATUS_FAIL;
1357 	}
1358 }
1359 
1360 static void post_command(struct qcom_nand_host *host, int command)
1361 {
1362 	struct nand_chip *chip = &host->chip;
1363 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1364 
1365 	switch (command) {
1366 	case NAND_CMD_READID:
1367 		nandc_read_buffer_sync(nandc, true);
1368 		memcpy(nandc->data_buffer, nandc->reg_read_buf,
1369 		       nandc->buf_count);
1370 		break;
1371 	case NAND_CMD_PAGEPROG:
1372 	case NAND_CMD_ERASE1:
1373 		parse_erase_write_errors(host, command);
1374 		break;
1375 	default:
1376 		break;
1377 	}
1378 }
1379 
1380 /*
1381  * Implements chip->cmdfunc. It's  only used for a limited set of commands.
1382  * The rest of the commands wouldn't be called by upper layers. For example,
1383  * NAND_CMD_READOOB would never be called because we have our own versions
1384  * of read_oob ops for nand_ecc_ctrl.
1385  */
1386 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
1387 			       int column, int page_addr)
1388 {
1389 	struct nand_chip *chip = mtd_to_nand(mtd);
1390 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1391 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1392 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1393 	bool wait = false;
1394 	int ret = 0;
1395 
1396 	pre_command(host, command);
1397 
1398 	switch (command) {
1399 	case NAND_CMD_RESET:
1400 		ret = reset(host);
1401 		wait = true;
1402 		break;
1403 
1404 	case NAND_CMD_READID:
1405 		nandc->buf_count = 4;
1406 		ret = read_id(host, column);
1407 		wait = true;
1408 		break;
1409 
1410 	case NAND_CMD_PARAM:
1411 		ret = nandc_param(host);
1412 		wait = true;
1413 		break;
1414 
1415 	case NAND_CMD_ERASE1:
1416 		ret = erase_block(host, page_addr);
1417 		wait = true;
1418 		break;
1419 
1420 	case NAND_CMD_READ0:
1421 		/* we read the entire page for now */
1422 		WARN_ON(column != 0);
1423 
1424 		host->use_ecc = true;
1425 		set_address(host, 0, page_addr);
1426 		update_rw_regs(host, ecc->steps, true);
1427 		break;
1428 
1429 	case NAND_CMD_SEQIN:
1430 		WARN_ON(column != 0);
1431 		set_address(host, 0, page_addr);
1432 		break;
1433 
1434 	case NAND_CMD_PAGEPROG:
1435 	case NAND_CMD_STATUS:
1436 	case NAND_CMD_NONE:
1437 	default:
1438 		break;
1439 	}
1440 
1441 	if (ret) {
1442 		dev_err(nandc->dev, "failure executing command %d\n",
1443 			command);
1444 		free_descs(nandc);
1445 		return;
1446 	}
1447 
1448 	if (wait) {
1449 		ret = submit_descs(nandc);
1450 		if (ret)
1451 			dev_err(nandc->dev,
1452 				"failure submitting descs for command %d\n",
1453 				command);
1454 	}
1455 
1456 	free_descs(nandc);
1457 
1458 	post_command(host, command);
1459 }
1460 
1461 /*
1462  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1463  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1464  *
1465  * when using RS ECC, the HW reports the same erros when reading an erased CW,
1466  * but it notifies that it is an erased CW by placing special characters at
1467  * certain offsets in the buffer.
1468  *
1469  * verify if the page is erased or not, and fix up the page for RS ECC by
1470  * replacing the special characters with 0xff.
1471  */
1472 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1473 {
1474 	u8 empty1, empty2;
1475 
1476 	/*
1477 	 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1478 	 * is erased by looking for 0x54s at offsets 3 and 175 from the
1479 	 * beginning of each codeword
1480 	 */
1481 
1482 	empty1 = data_buf[3];
1483 	empty2 = data_buf[175];
1484 
1485 	/*
1486 	 * if the erased codework markers, if they exist override them with
1487 	 * 0xffs
1488 	 */
1489 	if ((empty1 == 0x54 && empty2 == 0xff) ||
1490 	    (empty1 == 0xff && empty2 == 0x54)) {
1491 		data_buf[3] = 0xff;
1492 		data_buf[175] = 0xff;
1493 	}
1494 
1495 	/*
1496 	 * check if the entire chunk contains 0xffs or not. if it doesn't, then
1497 	 * restore the original values at the special offsets
1498 	 */
1499 	if (memchr_inv(data_buf, 0xff, data_len)) {
1500 		data_buf[3] = empty1;
1501 		data_buf[175] = empty2;
1502 
1503 		return false;
1504 	}
1505 
1506 	return true;
1507 }
1508 
1509 struct read_stats {
1510 	__le32 flash;
1511 	__le32 buffer;
1512 	__le32 erased_cw;
1513 };
1514 
1515 /*
1516  * reads back status registers set by the controller to notify page read
1517  * errors. this is equivalent to what 'ecc->correct()' would do.
1518  */
1519 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1520 			     u8 *oob_buf)
1521 {
1522 	struct nand_chip *chip = &host->chip;
1523 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1524 	struct mtd_info *mtd = nand_to_mtd(chip);
1525 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1526 	unsigned int max_bitflips = 0;
1527 	struct read_stats *buf;
1528 	int i;
1529 
1530 	buf = (struct read_stats *)nandc->reg_read_buf;
1531 	nandc_read_buffer_sync(nandc, true);
1532 
1533 	for (i = 0; i < ecc->steps; i++, buf++) {
1534 		u32 flash, buffer, erased_cw;
1535 		int data_len, oob_len;
1536 
1537 		if (i == (ecc->steps - 1)) {
1538 			data_len = ecc->size - ((ecc->steps - 1) << 2);
1539 			oob_len = ecc->steps << 2;
1540 		} else {
1541 			data_len = host->cw_data;
1542 			oob_len = 0;
1543 		}
1544 
1545 		flash = le32_to_cpu(buf->flash);
1546 		buffer = le32_to_cpu(buf->buffer);
1547 		erased_cw = le32_to_cpu(buf->erased_cw);
1548 
1549 		if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1550 			bool erased;
1551 
1552 			/* ignore erased codeword errors */
1553 			if (host->bch_enabled) {
1554 				erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1555 					 true : false;
1556 			} else {
1557 				erased = erased_chunk_check_and_fixup(data_buf,
1558 								      data_len);
1559 			}
1560 
1561 			if (erased) {
1562 				data_buf += data_len;
1563 				if (oob_buf)
1564 					oob_buf += oob_len + ecc->bytes;
1565 				continue;
1566 			}
1567 
1568 			if (buffer & BS_UNCORRECTABLE_BIT) {
1569 				int ret, ecclen, extraooblen;
1570 				void *eccbuf;
1571 
1572 				eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1573 				ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1574 				extraooblen = oob_buf ? oob_len : 0;
1575 
1576 				/*
1577 				 * make sure it isn't an erased page reported
1578 				 * as not-erased by HW because of a few bitflips
1579 				 */
1580 				ret = nand_check_erased_ecc_chunk(data_buf,
1581 					data_len, eccbuf, ecclen, oob_buf,
1582 					extraooblen, ecc->strength);
1583 				if (ret < 0) {
1584 					mtd->ecc_stats.failed++;
1585 				} else {
1586 					mtd->ecc_stats.corrected += ret;
1587 					max_bitflips =
1588 						max_t(unsigned int, max_bitflips, ret);
1589 				}
1590 			}
1591 		} else {
1592 			unsigned int stat;
1593 
1594 			stat = buffer & BS_CORRECTABLE_ERR_MSK;
1595 			mtd->ecc_stats.corrected += stat;
1596 			max_bitflips = max(max_bitflips, stat);
1597 		}
1598 
1599 		data_buf += data_len;
1600 		if (oob_buf)
1601 			oob_buf += oob_len + ecc->bytes;
1602 	}
1603 
1604 	return max_bitflips;
1605 }
1606 
1607 /*
1608  * helper to perform the actual page read operation, used by ecc->read_page(),
1609  * ecc->read_oob()
1610  */
1611 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1612 			 u8 *oob_buf)
1613 {
1614 	struct nand_chip *chip = &host->chip;
1615 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1616 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1617 	int i, ret;
1618 
1619 	config_nand_page_read(nandc);
1620 
1621 	/* queue cmd descs for each codeword */
1622 	for (i = 0; i < ecc->steps; i++) {
1623 		int data_size, oob_size;
1624 
1625 		if (i == (ecc->steps - 1)) {
1626 			data_size = ecc->size - ((ecc->steps - 1) << 2);
1627 			oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1628 				   host->spare_bytes;
1629 		} else {
1630 			data_size = host->cw_data;
1631 			oob_size = host->ecc_bytes_hw + host->spare_bytes;
1632 		}
1633 
1634 		if (nandc->props->is_bam) {
1635 			if (data_buf && oob_buf) {
1636 				nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1637 				nandc_set_read_loc(nandc, 1, data_size,
1638 						   oob_size, 1);
1639 			} else if (data_buf) {
1640 				nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1641 			} else {
1642 				nandc_set_read_loc(nandc, 0, data_size,
1643 						   oob_size, 1);
1644 			}
1645 		}
1646 
1647 		config_nand_cw_read(nandc);
1648 
1649 		if (data_buf)
1650 			read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1651 				      data_size, 0);
1652 
1653 		/*
1654 		 * when ecc is enabled, the controller doesn't read the real
1655 		 * or dummy bad block markers in each chunk. To maintain a
1656 		 * consistent layout across RAW and ECC reads, we just
1657 		 * leave the real/dummy BBM offsets empty (i.e, filled with
1658 		 * 0xffs)
1659 		 */
1660 		if (oob_buf) {
1661 			int j;
1662 
1663 			for (j = 0; j < host->bbm_size; j++)
1664 				*oob_buf++ = 0xff;
1665 
1666 			read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1667 				      oob_buf, oob_size, 0);
1668 		}
1669 
1670 		if (data_buf)
1671 			data_buf += data_size;
1672 		if (oob_buf)
1673 			oob_buf += oob_size;
1674 	}
1675 
1676 	ret = submit_descs(nandc);
1677 	if (ret)
1678 		dev_err(nandc->dev, "failure to read page/oob\n");
1679 
1680 	free_descs(nandc);
1681 
1682 	return ret;
1683 }
1684 
1685 /*
1686  * a helper that copies the last step/codeword of a page (containing free oob)
1687  * into our local buffer
1688  */
1689 static int copy_last_cw(struct qcom_nand_host *host, int page)
1690 {
1691 	struct nand_chip *chip = &host->chip;
1692 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1693 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1694 	int size;
1695 	int ret;
1696 
1697 	clear_read_regs(nandc);
1698 
1699 	size = host->use_ecc ? host->cw_data : host->cw_size;
1700 
1701 	/* prepare a clean read buffer */
1702 	memset(nandc->data_buffer, 0xff, size);
1703 
1704 	set_address(host, host->cw_size * (ecc->steps - 1), page);
1705 	update_rw_regs(host, 1, true);
1706 
1707 	config_nand_single_cw_page_read(nandc);
1708 
1709 	read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1710 
1711 	ret = submit_descs(nandc);
1712 	if (ret)
1713 		dev_err(nandc->dev, "failed to copy last codeword\n");
1714 
1715 	free_descs(nandc);
1716 
1717 	return ret;
1718 }
1719 
1720 /* implements ecc->read_page() */
1721 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1722 				uint8_t *buf, int oob_required, int page)
1723 {
1724 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1725 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1726 	u8 *data_buf, *oob_buf = NULL;
1727 	int ret;
1728 
1729 	nand_read_page_op(chip, page, 0, NULL, 0);
1730 	data_buf = buf;
1731 	oob_buf = oob_required ? chip->oob_poi : NULL;
1732 
1733 	clear_bam_transaction(nandc);
1734 	ret = read_page_ecc(host, data_buf, oob_buf);
1735 	if (ret) {
1736 		dev_err(nandc->dev, "failure to read page\n");
1737 		return ret;
1738 	}
1739 
1740 	return parse_read_errors(host, data_buf, oob_buf);
1741 }
1742 
1743 /* implements ecc->read_page_raw() */
1744 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1745 				    struct nand_chip *chip, uint8_t *buf,
1746 				    int oob_required, int page)
1747 {
1748 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1749 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1750 	u8 *data_buf, *oob_buf;
1751 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1752 	int i, ret;
1753 	int read_loc;
1754 
1755 	nand_read_page_op(chip, page, 0, NULL, 0);
1756 	data_buf = buf;
1757 	oob_buf = chip->oob_poi;
1758 
1759 	host->use_ecc = false;
1760 
1761 	clear_bam_transaction(nandc);
1762 	update_rw_regs(host, ecc->steps, true);
1763 	config_nand_page_read(nandc);
1764 
1765 	for (i = 0; i < ecc->steps; i++) {
1766 		int data_size1, data_size2, oob_size1, oob_size2;
1767 		int reg_off = FLASH_BUF_ACC;
1768 
1769 		data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1770 		oob_size1 = host->bbm_size;
1771 
1772 		if (i == (ecc->steps - 1)) {
1773 			data_size2 = ecc->size - data_size1 -
1774 				     ((ecc->steps - 1) << 2);
1775 			oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1776 				    host->spare_bytes;
1777 		} else {
1778 			data_size2 = host->cw_data - data_size1;
1779 			oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1780 		}
1781 
1782 		if (nandc->props->is_bam) {
1783 			read_loc = 0;
1784 			nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1785 			read_loc += data_size1;
1786 
1787 			nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1788 			read_loc += oob_size1;
1789 
1790 			nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1791 			read_loc += data_size2;
1792 
1793 			nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1794 		}
1795 
1796 		config_nand_cw_read(nandc);
1797 
1798 		read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1799 		reg_off += data_size1;
1800 		data_buf += data_size1;
1801 
1802 		read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1803 		reg_off += oob_size1;
1804 		oob_buf += oob_size1;
1805 
1806 		read_data_dma(nandc, reg_off, data_buf, data_size2, 0);
1807 		reg_off += data_size2;
1808 		data_buf += data_size2;
1809 
1810 		read_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
1811 		oob_buf += oob_size2;
1812 	}
1813 
1814 	ret = submit_descs(nandc);
1815 	if (ret)
1816 		dev_err(nandc->dev, "failure to read raw page\n");
1817 
1818 	free_descs(nandc);
1819 
1820 	return 0;
1821 }
1822 
1823 /* implements ecc->read_oob() */
1824 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1825 			       int page)
1826 {
1827 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1828 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1829 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1830 	int ret;
1831 
1832 	clear_read_regs(nandc);
1833 	clear_bam_transaction(nandc);
1834 
1835 	host->use_ecc = true;
1836 	set_address(host, 0, page);
1837 	update_rw_regs(host, ecc->steps, true);
1838 
1839 	ret = read_page_ecc(host, NULL, chip->oob_poi);
1840 	if (ret)
1841 		dev_err(nandc->dev, "failure to read oob\n");
1842 
1843 	return ret;
1844 }
1845 
1846 /* implements ecc->write_page() */
1847 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1848 				 const uint8_t *buf, int oob_required, int page)
1849 {
1850 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1851 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1852 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1853 	u8 *data_buf, *oob_buf;
1854 	int i, ret;
1855 
1856 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1857 
1858 	clear_read_regs(nandc);
1859 	clear_bam_transaction(nandc);
1860 
1861 	data_buf = (u8 *)buf;
1862 	oob_buf = chip->oob_poi;
1863 
1864 	host->use_ecc = true;
1865 	update_rw_regs(host, ecc->steps, false);
1866 	config_nand_page_write(nandc);
1867 
1868 	for (i = 0; i < ecc->steps; i++) {
1869 		int data_size, oob_size;
1870 
1871 		if (i == (ecc->steps - 1)) {
1872 			data_size = ecc->size - ((ecc->steps - 1) << 2);
1873 			oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1874 				   host->spare_bytes;
1875 		} else {
1876 			data_size = host->cw_data;
1877 			oob_size = ecc->bytes;
1878 		}
1879 
1880 
1881 		write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
1882 			       i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
1883 
1884 		/*
1885 		 * when ECC is enabled, we don't really need to write anything
1886 		 * to oob for the first n - 1 codewords since these oob regions
1887 		 * just contain ECC bytes that's written by the controller
1888 		 * itself. For the last codeword, we skip the bbm positions and
1889 		 * write to the free oob area.
1890 		 */
1891 		if (i == (ecc->steps - 1)) {
1892 			oob_buf += host->bbm_size;
1893 
1894 			write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1895 				       oob_buf, oob_size, 0);
1896 		}
1897 
1898 		config_nand_cw_write(nandc);
1899 
1900 		data_buf += data_size;
1901 		oob_buf += oob_size;
1902 	}
1903 
1904 	ret = submit_descs(nandc);
1905 	if (ret)
1906 		dev_err(nandc->dev, "failure to write page\n");
1907 
1908 	free_descs(nandc);
1909 
1910 	if (!ret)
1911 		ret = nand_prog_page_end_op(chip);
1912 
1913 	return ret;
1914 }
1915 
1916 /* implements ecc->write_page_raw() */
1917 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1918 				     struct nand_chip *chip, const uint8_t *buf,
1919 				     int oob_required, int page)
1920 {
1921 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1922 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1923 	struct nand_ecc_ctrl *ecc = &chip->ecc;
1924 	u8 *data_buf, *oob_buf;
1925 	int i, ret;
1926 
1927 	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1928 	clear_read_regs(nandc);
1929 	clear_bam_transaction(nandc);
1930 
1931 	data_buf = (u8 *)buf;
1932 	oob_buf = chip->oob_poi;
1933 
1934 	host->use_ecc = false;
1935 	update_rw_regs(host, ecc->steps, false);
1936 	config_nand_page_write(nandc);
1937 
1938 	for (i = 0; i < ecc->steps; i++) {
1939 		int data_size1, data_size2, oob_size1, oob_size2;
1940 		int reg_off = FLASH_BUF_ACC;
1941 
1942 		data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1943 		oob_size1 = host->bbm_size;
1944 
1945 		if (i == (ecc->steps - 1)) {
1946 			data_size2 = ecc->size - data_size1 -
1947 				     ((ecc->steps - 1) << 2);
1948 			oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1949 				    host->spare_bytes;
1950 		} else {
1951 			data_size2 = host->cw_data - data_size1;
1952 			oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1953 		}
1954 
1955 		write_data_dma(nandc, reg_off, data_buf, data_size1,
1956 			       NAND_BAM_NO_EOT);
1957 		reg_off += data_size1;
1958 		data_buf += data_size1;
1959 
1960 		write_data_dma(nandc, reg_off, oob_buf, oob_size1,
1961 			       NAND_BAM_NO_EOT);
1962 		reg_off += oob_size1;
1963 		oob_buf += oob_size1;
1964 
1965 		write_data_dma(nandc, reg_off, data_buf, data_size2,
1966 			       NAND_BAM_NO_EOT);
1967 		reg_off += data_size2;
1968 		data_buf += data_size2;
1969 
1970 		write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
1971 		oob_buf += oob_size2;
1972 
1973 		config_nand_cw_write(nandc);
1974 	}
1975 
1976 	ret = submit_descs(nandc);
1977 	if (ret)
1978 		dev_err(nandc->dev, "failure to write raw page\n");
1979 
1980 	free_descs(nandc);
1981 
1982 	if (!ret)
1983 		ret = nand_prog_page_end_op(chip);
1984 
1985 	return ret;
1986 }
1987 
1988 /*
1989  * implements ecc->write_oob()
1990  *
1991  * the NAND controller cannot write only data or only oob within a codeword,
1992  * since ecc is calculated for the combined codeword. we first copy the
1993  * entire contents for the last codeword(data + oob), replace the old oob
1994  * with the new one in chip->oob_poi, and then write the entire codeword.
1995  * this read-copy-write operation results in a slight performance loss.
1996  */
1997 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1998 				int page)
1999 {
2000 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
2001 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2002 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2003 	u8 *oob = chip->oob_poi;
2004 	int data_size, oob_size;
2005 	int ret;
2006 
2007 	host->use_ecc = true;
2008 
2009 	clear_bam_transaction(nandc);
2010 	ret = copy_last_cw(host, page);
2011 	if (ret)
2012 		return ret;
2013 
2014 	clear_read_regs(nandc);
2015 	clear_bam_transaction(nandc);
2016 
2017 	/* calculate the data and oob size for the last codeword/step */
2018 	data_size = ecc->size - ((ecc->steps - 1) << 2);
2019 	oob_size = mtd->oobavail;
2020 
2021 	/* override new oob content to last codeword */
2022 	mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2023 				    0, mtd->oobavail);
2024 
2025 	set_address(host, host->cw_size * (ecc->steps - 1), page);
2026 	update_rw_regs(host, 1, false);
2027 
2028 	config_nand_page_write(nandc);
2029 	write_data_dma(nandc, FLASH_BUF_ACC,
2030 		       nandc->data_buffer, data_size + oob_size, 0);
2031 	config_nand_cw_write(nandc);
2032 
2033 	ret = submit_descs(nandc);
2034 
2035 	free_descs(nandc);
2036 
2037 	if (ret) {
2038 		dev_err(nandc->dev, "failure to write oob\n");
2039 		return -EIO;
2040 	}
2041 
2042 	return nand_prog_page_end_op(chip);
2043 }
2044 
2045 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
2046 {
2047 	struct nand_chip *chip = mtd_to_nand(mtd);
2048 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
2049 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2050 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2051 	int page, ret, bbpos, bad = 0;
2052 	u32 flash_status;
2053 
2054 	page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2055 
2056 	/*
2057 	 * configure registers for a raw sub page read, the address is set to
2058 	 * the beginning of the last codeword, we don't care about reading ecc
2059 	 * portion of oob. we just want the first few bytes from this codeword
2060 	 * that contains the BBM
2061 	 */
2062 	host->use_ecc = false;
2063 
2064 	clear_bam_transaction(nandc);
2065 	ret = copy_last_cw(host, page);
2066 	if (ret)
2067 		goto err;
2068 
2069 	flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
2070 
2071 	if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
2072 		dev_warn(nandc->dev, "error when trying to read BBM\n");
2073 		goto err;
2074 	}
2075 
2076 	bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2077 
2078 	bad = nandc->data_buffer[bbpos] != 0xff;
2079 
2080 	if (chip->options & NAND_BUSWIDTH_16)
2081 		bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2082 err:
2083 	return bad;
2084 }
2085 
2086 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
2087 {
2088 	struct nand_chip *chip = mtd_to_nand(mtd);
2089 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
2090 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2091 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2092 	int page, ret;
2093 
2094 	clear_read_regs(nandc);
2095 	clear_bam_transaction(nandc);
2096 
2097 	/*
2098 	 * to mark the BBM as bad, we flash the entire last codeword with 0s.
2099 	 * we don't care about the rest of the content in the codeword since
2100 	 * we aren't going to use this block again
2101 	 */
2102 	memset(nandc->data_buffer, 0x00, host->cw_size);
2103 
2104 	page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2105 
2106 	/* prepare write */
2107 	host->use_ecc = false;
2108 	set_address(host, host->cw_size * (ecc->steps - 1), page);
2109 	update_rw_regs(host, 1, false);
2110 
2111 	config_nand_page_write(nandc);
2112 	write_data_dma(nandc, FLASH_BUF_ACC,
2113 		       nandc->data_buffer, host->cw_size, 0);
2114 	config_nand_cw_write(nandc);
2115 
2116 	ret = submit_descs(nandc);
2117 
2118 	free_descs(nandc);
2119 
2120 	if (ret) {
2121 		dev_err(nandc->dev, "failure to update BBM\n");
2122 		return -EIO;
2123 	}
2124 
2125 	return nand_prog_page_end_op(chip);
2126 }
2127 
2128 /*
2129  * the three functions below implement chip->read_byte(), chip->read_buf()
2130  * and chip->write_buf() respectively. these aren't used for
2131  * reading/writing page data, they are used for smaller data like reading
2132  * id, status etc
2133  */
2134 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
2135 {
2136 	struct nand_chip *chip = mtd_to_nand(mtd);
2137 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
2138 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2139 	u8 *buf = nandc->data_buffer;
2140 	u8 ret = 0x0;
2141 
2142 	if (host->last_command == NAND_CMD_STATUS) {
2143 		ret = host->status;
2144 
2145 		host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2146 
2147 		return ret;
2148 	}
2149 
2150 	if (nandc->buf_start < nandc->buf_count)
2151 		ret = buf[nandc->buf_start++];
2152 
2153 	return ret;
2154 }
2155 
2156 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
2157 {
2158 	struct nand_chip *chip = mtd_to_nand(mtd);
2159 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2160 	int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2161 
2162 	memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2163 	nandc->buf_start += real_len;
2164 }
2165 
2166 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
2167 				 int len)
2168 {
2169 	struct nand_chip *chip = mtd_to_nand(mtd);
2170 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2171 	int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2172 
2173 	memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2174 
2175 	nandc->buf_start += real_len;
2176 }
2177 
2178 /* we support only one external chip for now */
2179 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
2180 {
2181 	struct nand_chip *chip = mtd_to_nand(mtd);
2182 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2183 
2184 	if (chipnr <= 0)
2185 		return;
2186 
2187 	dev_warn(nandc->dev, "invalid chip select\n");
2188 }
2189 
2190 /*
2191  * NAND controller page layout info
2192  *
2193  * Layout with ECC enabled:
2194  *
2195  * |----------------------|  |---------------------------------|
2196  * |           xx.......yy|  |             *********xx.......yy|
2197  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
2198  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
2199  * |           xx.......yy|  |             *********xx.......yy|
2200  * |----------------------|  |---------------------------------|
2201  *     codeword 1,2..n-1                  codeword n
2202  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
2203  *
2204  * n = Number of codewords in the page
2205  * . = ECC bytes
2206  * * = Spare/free bytes
2207  * x = Unused byte(s)
2208  * y = Reserved byte(s)
2209  *
2210  * 2K page: n = 4, spare = 16 bytes
2211  * 4K page: n = 8, spare = 32 bytes
2212  * 8K page: n = 16, spare = 64 bytes
2213  *
2214  * the qcom nand controller operates at a sub page/codeword level. each
2215  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2216  * the number of ECC bytes vary based on the ECC strength and the bus width.
2217  *
2218  * the first n - 1 codewords contains 516 bytes of user data, the remaining
2219  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2220  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2221  *
2222  * When we access a page with ECC enabled, the reserved bytes(s) are not
2223  * accessible at all. When reading, we fill up these unreadable positions
2224  * with 0xffs. When writing, the controller skips writing the inaccessible
2225  * bytes.
2226  *
2227  * Layout with ECC disabled:
2228  *
2229  * |------------------------------|  |---------------------------------------|
2230  * |         yy          xx.......|  |         bb          *********xx.......|
2231  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
2232  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
2233  * |         yy          xx.......|  |         bb          *********xx.......|
2234  * |------------------------------|  |---------------------------------------|
2235  *         codeword 1,2..n-1                        codeword n
2236  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
2237  *
2238  * n = Number of codewords in the page
2239  * . = ECC bytes
2240  * * = Spare/free bytes
2241  * x = Unused byte(s)
2242  * y = Dummy Bad Bock byte(s)
2243  * b = Real Bad Block byte(s)
2244  * size1/size2 = function of codeword size and 'n'
2245  *
2246  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2247  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2248  * Block Markers. In the last codeword, this position contains the real BBM
2249  *
2250  * In order to have a consistent layout between RAW and ECC modes, we assume
2251  * the following OOB layout arrangement:
2252  *
2253  * |-----------|  |--------------------|
2254  * |yyxx.......|  |bb*********xx.......|
2255  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
2256  * |yyxx.......|  |bb*********xx.......|
2257  * |yyxx.......|  |bb*********xx.......|
2258  * |-----------|  |--------------------|
2259  *  first n - 1       nth OOB region
2260  *  OOB regions
2261  *
2262  * n = Number of codewords in the page
2263  * . = ECC bytes
2264  * * = FREE OOB bytes
2265  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2266  * x = Unused byte(s)
2267  * b = Real bad block byte(s) (inaccessible when ECC enabled)
2268  *
2269  * This layout is read as is when ECC is disabled. When ECC is enabled, the
2270  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2271  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2272  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2273  * the sum of the three).
2274  */
2275 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2276 				   struct mtd_oob_region *oobregion)
2277 {
2278 	struct nand_chip *chip = mtd_to_nand(mtd);
2279 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
2280 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2281 
2282 	if (section > 1)
2283 		return -ERANGE;
2284 
2285 	if (!section) {
2286 		oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2287 				    host->bbm_size;
2288 		oobregion->offset = 0;
2289 	} else {
2290 		oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2291 		oobregion->offset = mtd->oobsize - oobregion->length;
2292 	}
2293 
2294 	return 0;
2295 }
2296 
2297 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2298 				     struct mtd_oob_region *oobregion)
2299 {
2300 	struct nand_chip *chip = mtd_to_nand(mtd);
2301 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
2302 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2303 
2304 	if (section)
2305 		return -ERANGE;
2306 
2307 	oobregion->length = ecc->steps * 4;
2308 	oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2309 
2310 	return 0;
2311 }
2312 
2313 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2314 	.ecc = qcom_nand_ooblayout_ecc,
2315 	.free = qcom_nand_ooblayout_free,
2316 };
2317 
2318 static int qcom_nand_host_setup(struct qcom_nand_host *host)
2319 {
2320 	struct nand_chip *chip = &host->chip;
2321 	struct mtd_info *mtd = nand_to_mtd(chip);
2322 	struct nand_ecc_ctrl *ecc = &chip->ecc;
2323 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2324 	int cwperpage, bad_block_byte;
2325 	bool wide_bus;
2326 	int ecc_mode = 1;
2327 
2328 	/*
2329 	 * the controller requires each step consists of 512 bytes of data.
2330 	 * bail out if DT has populated a wrong step size.
2331 	 */
2332 	if (ecc->size != NANDC_STEP_SIZE) {
2333 		dev_err(nandc->dev, "invalid ecc size\n");
2334 		return -EINVAL;
2335 	}
2336 
2337 	wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2338 
2339 	if (ecc->strength >= 8) {
2340 		/* 8 bit ECC defaults to BCH ECC on all platforms */
2341 		host->bch_enabled = true;
2342 		ecc_mode = 1;
2343 
2344 		if (wide_bus) {
2345 			host->ecc_bytes_hw = 14;
2346 			host->spare_bytes = 0;
2347 			host->bbm_size = 2;
2348 		} else {
2349 			host->ecc_bytes_hw = 13;
2350 			host->spare_bytes = 2;
2351 			host->bbm_size = 1;
2352 		}
2353 	} else {
2354 		/*
2355 		 * if the controller supports BCH for 4 bit ECC, the controller
2356 		 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2357 		 * always 10 bytes
2358 		 */
2359 		if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2360 			/* BCH */
2361 			host->bch_enabled = true;
2362 			ecc_mode = 0;
2363 
2364 			if (wide_bus) {
2365 				host->ecc_bytes_hw = 8;
2366 				host->spare_bytes = 2;
2367 				host->bbm_size = 2;
2368 			} else {
2369 				host->ecc_bytes_hw = 7;
2370 				host->spare_bytes = 4;
2371 				host->bbm_size = 1;
2372 			}
2373 		} else {
2374 			/* RS */
2375 			host->ecc_bytes_hw = 10;
2376 
2377 			if (wide_bus) {
2378 				host->spare_bytes = 0;
2379 				host->bbm_size = 2;
2380 			} else {
2381 				host->spare_bytes = 1;
2382 				host->bbm_size = 1;
2383 			}
2384 		}
2385 	}
2386 
2387 	/*
2388 	 * we consider ecc->bytes as the sum of all the non-data content in a
2389 	 * step. It gives us a clean representation of the oob area (even if
2390 	 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2391 	 * ECC and 12 bytes for 4 bit ECC
2392 	 */
2393 	ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2394 
2395 	ecc->read_page		= qcom_nandc_read_page;
2396 	ecc->read_page_raw	= qcom_nandc_read_page_raw;
2397 	ecc->read_oob		= qcom_nandc_read_oob;
2398 	ecc->write_page		= qcom_nandc_write_page;
2399 	ecc->write_page_raw	= qcom_nandc_write_page_raw;
2400 	ecc->write_oob		= qcom_nandc_write_oob;
2401 
2402 	ecc->mode = NAND_ECC_HW;
2403 
2404 	mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2405 
2406 	cwperpage = mtd->writesize / ecc->size;
2407 	nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2408 				     cwperpage);
2409 
2410 	/*
2411 	 * DATA_UD_BYTES varies based on whether the read/write command protects
2412 	 * spare data with ECC too. We protect spare data by default, so we set
2413 	 * it to main + spare data, which are 512 and 4 bytes respectively.
2414 	 */
2415 	host->cw_data = 516;
2416 
2417 	/*
2418 	 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2419 	 * for 8 bit ECC
2420 	 */
2421 	host->cw_size = host->cw_data + ecc->bytes;
2422 
2423 	if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
2424 		dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
2425 		return -EINVAL;
2426 	}
2427 
2428 	bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2429 
2430 	host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2431 				| host->cw_data << UD_SIZE_BYTES
2432 				| 0 << DISABLE_STATUS_AFTER_WRITE
2433 				| 5 << NUM_ADDR_CYCLES
2434 				| host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2435 				| 0 << STATUS_BFR_READ
2436 				| 1 << SET_RD_MODE_AFTER_STATUS
2437 				| host->spare_bytes << SPARE_SIZE_BYTES;
2438 
2439 	host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2440 				| 0 <<  CS_ACTIVE_BSY
2441 				| bad_block_byte << BAD_BLOCK_BYTE_NUM
2442 				| 0 << BAD_BLOCK_IN_SPARE_AREA
2443 				| 2 << WR_RD_BSY_GAP
2444 				| wide_bus << WIDE_FLASH
2445 				| host->bch_enabled << ENABLE_BCH_ECC;
2446 
2447 	host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2448 				| host->cw_size << UD_SIZE_BYTES
2449 				| 5 << NUM_ADDR_CYCLES
2450 				| 0 << SPARE_SIZE_BYTES;
2451 
2452 	host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2453 				| 0 << CS_ACTIVE_BSY
2454 				| 17 << BAD_BLOCK_BYTE_NUM
2455 				| 1 << BAD_BLOCK_IN_SPARE_AREA
2456 				| 2 << WR_RD_BSY_GAP
2457 				| wide_bus << WIDE_FLASH
2458 				| 1 << DEV0_CFG1_ECC_DISABLE;
2459 
2460 	host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2461 				| 0 << ECC_SW_RESET
2462 				| host->cw_data << ECC_NUM_DATA_BYTES
2463 				| 1 << ECC_FORCE_CLK_OPEN
2464 				| ecc_mode << ECC_MODE
2465 				| host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2466 
2467 	host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2468 
2469 	host->clrflashstatus = FS_READY_BSY_N;
2470 	host->clrreadstatus = 0xc0;
2471 	nandc->regs->erased_cw_detect_cfg_clr =
2472 		cpu_to_le32(CLR_ERASED_PAGE_DET);
2473 	nandc->regs->erased_cw_detect_cfg_set =
2474 		cpu_to_le32(SET_ERASED_PAGE_DET);
2475 
2476 	dev_dbg(nandc->dev,
2477 		"cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2478 		host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2479 		host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2480 		cwperpage);
2481 
2482 	return 0;
2483 }
2484 
2485 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2486 {
2487 	int ret;
2488 
2489 	ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2490 	if (ret) {
2491 		dev_err(nandc->dev, "failed to set DMA mask\n");
2492 		return ret;
2493 	}
2494 
2495 	/*
2496 	 * we use the internal buffer for reading ONFI params, reading small
2497 	 * data like ID and status, and preforming read-copy-write operations
2498 	 * when writing to a codeword partially. 532 is the maximum possible
2499 	 * size of a codeword for our nand controller
2500 	 */
2501 	nandc->buf_size = 532;
2502 
2503 	nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2504 					GFP_KERNEL);
2505 	if (!nandc->data_buffer)
2506 		return -ENOMEM;
2507 
2508 	nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2509 					GFP_KERNEL);
2510 	if (!nandc->regs)
2511 		return -ENOMEM;
2512 
2513 	nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2514 				MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2515 				GFP_KERNEL);
2516 	if (!nandc->reg_read_buf)
2517 		return -ENOMEM;
2518 
2519 	if (nandc->props->is_bam) {
2520 		nandc->reg_read_dma =
2521 			dma_map_single(nandc->dev, nandc->reg_read_buf,
2522 				       MAX_REG_RD *
2523 				       sizeof(*nandc->reg_read_buf),
2524 				       DMA_FROM_DEVICE);
2525 		if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2526 			dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2527 			return -EIO;
2528 		}
2529 
2530 		nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
2531 		if (!nandc->tx_chan) {
2532 			dev_err(nandc->dev, "failed to request tx channel\n");
2533 			return -ENODEV;
2534 		}
2535 
2536 		nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
2537 		if (!nandc->rx_chan) {
2538 			dev_err(nandc->dev, "failed to request rx channel\n");
2539 			return -ENODEV;
2540 		}
2541 
2542 		nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
2543 		if (!nandc->cmd_chan) {
2544 			dev_err(nandc->dev, "failed to request cmd channel\n");
2545 			return -ENODEV;
2546 		}
2547 
2548 		/*
2549 		 * Initially allocate BAM transaction to read ONFI param page.
2550 		 * After detecting all the devices, this BAM transaction will
2551 		 * be freed and the next BAM tranasction will be allocated with
2552 		 * maximum codeword size
2553 		 */
2554 		nandc->max_cwperpage = 1;
2555 		nandc->bam_txn = alloc_bam_transaction(nandc);
2556 		if (!nandc->bam_txn) {
2557 			dev_err(nandc->dev,
2558 				"failed to allocate bam transaction\n");
2559 			return -ENOMEM;
2560 		}
2561 	} else {
2562 		nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
2563 		if (!nandc->chan) {
2564 			dev_err(nandc->dev,
2565 				"failed to request slave channel\n");
2566 			return -ENODEV;
2567 		}
2568 	}
2569 
2570 	INIT_LIST_HEAD(&nandc->desc_list);
2571 	INIT_LIST_HEAD(&nandc->host_list);
2572 
2573 	nand_hw_control_init(&nandc->controller);
2574 
2575 	return 0;
2576 }
2577 
2578 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2579 {
2580 	if (nandc->props->is_bam) {
2581 		if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2582 			dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2583 					 MAX_REG_RD *
2584 					 sizeof(*nandc->reg_read_buf),
2585 					 DMA_FROM_DEVICE);
2586 
2587 		if (nandc->tx_chan)
2588 			dma_release_channel(nandc->tx_chan);
2589 
2590 		if (nandc->rx_chan)
2591 			dma_release_channel(nandc->rx_chan);
2592 
2593 		if (nandc->cmd_chan)
2594 			dma_release_channel(nandc->cmd_chan);
2595 	} else {
2596 		if (nandc->chan)
2597 			dma_release_channel(nandc->chan);
2598 	}
2599 }
2600 
2601 /* one time setup of a few nand controller registers */
2602 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2603 {
2604 	u32 nand_ctrl;
2605 
2606 	/* kill onenand */
2607 	nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2608 	nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2609 		    NAND_DEV_CMD_VLD_VAL);
2610 
2611 	/* enable ADM or BAM DMA */
2612 	if (nandc->props->is_bam) {
2613 		nand_ctrl = nandc_read(nandc, NAND_CTRL);
2614 		nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2615 	} else {
2616 		nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2617 	}
2618 
2619 	/* save the original values of these registers */
2620 	nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2621 	nandc->vld = NAND_DEV_CMD_VLD_VAL;
2622 
2623 	return 0;
2624 }
2625 
2626 static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
2627 			       struct qcom_nand_host *host,
2628 			       struct device_node *dn)
2629 {
2630 	struct nand_chip *chip = &host->chip;
2631 	struct mtd_info *mtd = nand_to_mtd(chip);
2632 	struct device *dev = nandc->dev;
2633 	int ret;
2634 
2635 	ret = of_property_read_u32(dn, "reg", &host->cs);
2636 	if (ret) {
2637 		dev_err(dev, "can't get chip-select\n");
2638 		return -ENXIO;
2639 	}
2640 
2641 	nand_set_flash_node(chip, dn);
2642 	mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2643 	if (!mtd->name)
2644 		return -ENOMEM;
2645 
2646 	mtd->owner = THIS_MODULE;
2647 	mtd->dev.parent = dev;
2648 
2649 	chip->cmdfunc		= qcom_nandc_command;
2650 	chip->select_chip	= qcom_nandc_select_chip;
2651 	chip->read_byte		= qcom_nandc_read_byte;
2652 	chip->read_buf		= qcom_nandc_read_buf;
2653 	chip->write_buf		= qcom_nandc_write_buf;
2654 	chip->set_features	= nand_get_set_features_notsupp;
2655 	chip->get_features	= nand_get_set_features_notsupp;
2656 
2657 	/*
2658 	 * the bad block marker is readable only when we read the last codeword
2659 	 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2660 	 * helpers don't allow us to read BB from a nand chip with ECC
2661 	 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2662 	 * and block_markbad helpers until we permanently switch to using
2663 	 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2664 	 */
2665 	chip->block_bad		= qcom_nandc_block_bad;
2666 	chip->block_markbad	= qcom_nandc_block_markbad;
2667 
2668 	chip->controller = &nandc->controller;
2669 	chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2670 			 NAND_SKIP_BBTSCAN;
2671 
2672 	/* set up initial status value */
2673 	host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2674 
2675 	ret = nand_scan_ident(mtd, 1, NULL);
2676 	if (ret)
2677 		return ret;
2678 
2679 	ret = qcom_nand_host_setup(host);
2680 
2681 	return ret;
2682 }
2683 
2684 static int qcom_nand_mtd_register(struct qcom_nand_controller *nandc,
2685 				  struct qcom_nand_host *host,
2686 				  struct device_node *dn)
2687 {
2688 	struct nand_chip *chip = &host->chip;
2689 	struct mtd_info *mtd = nand_to_mtd(chip);
2690 	int ret;
2691 
2692 	ret = nand_scan_tail(mtd);
2693 	if (ret)
2694 		return ret;
2695 
2696 	ret = mtd_device_register(mtd, NULL, 0);
2697 	if (ret)
2698 		nand_cleanup(mtd_to_nand(mtd));
2699 
2700 	return ret;
2701 }
2702 
2703 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2704 {
2705 	struct device *dev = nandc->dev;
2706 	struct device_node *dn = dev->of_node, *child;
2707 	struct qcom_nand_host *host, *tmp;
2708 	int ret;
2709 
2710 	for_each_available_child_of_node(dn, child) {
2711 		host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2712 		if (!host) {
2713 			of_node_put(child);
2714 			return -ENOMEM;
2715 		}
2716 
2717 		ret = qcom_nand_host_init(nandc, host, child);
2718 		if (ret) {
2719 			devm_kfree(dev, host);
2720 			continue;
2721 		}
2722 
2723 		list_add_tail(&host->node, &nandc->host_list);
2724 	}
2725 
2726 	if (list_empty(&nandc->host_list))
2727 		return -ENODEV;
2728 
2729 	if (nandc->props->is_bam) {
2730 		free_bam_transaction(nandc);
2731 		nandc->bam_txn = alloc_bam_transaction(nandc);
2732 		if (!nandc->bam_txn) {
2733 			dev_err(nandc->dev,
2734 				"failed to allocate bam transaction\n");
2735 			return -ENOMEM;
2736 		}
2737 	}
2738 
2739 	list_for_each_entry_safe(host, tmp, &nandc->host_list, node) {
2740 		ret = qcom_nand_mtd_register(nandc, host, child);
2741 		if (ret) {
2742 			list_del(&host->node);
2743 			devm_kfree(dev, host);
2744 		}
2745 	}
2746 
2747 	if (list_empty(&nandc->host_list))
2748 		return -ENODEV;
2749 
2750 	return 0;
2751 }
2752 
2753 /* parse custom DT properties here */
2754 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2755 {
2756 	struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2757 	struct device_node *np = nandc->dev->of_node;
2758 	int ret;
2759 
2760 	if (!nandc->props->is_bam) {
2761 		ret = of_property_read_u32(np, "qcom,cmd-crci",
2762 					   &nandc->cmd_crci);
2763 		if (ret) {
2764 			dev_err(nandc->dev, "command CRCI unspecified\n");
2765 			return ret;
2766 		}
2767 
2768 		ret = of_property_read_u32(np, "qcom,data-crci",
2769 					   &nandc->data_crci);
2770 		if (ret) {
2771 			dev_err(nandc->dev, "data CRCI unspecified\n");
2772 			return ret;
2773 		}
2774 	}
2775 
2776 	return 0;
2777 }
2778 
2779 static int qcom_nandc_probe(struct platform_device *pdev)
2780 {
2781 	struct qcom_nand_controller *nandc;
2782 	const void *dev_data;
2783 	struct device *dev = &pdev->dev;
2784 	struct resource *res;
2785 	int ret;
2786 
2787 	nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2788 	if (!nandc)
2789 		return -ENOMEM;
2790 
2791 	platform_set_drvdata(pdev, nandc);
2792 	nandc->dev = dev;
2793 
2794 	dev_data = of_device_get_match_data(dev);
2795 	if (!dev_data) {
2796 		dev_err(&pdev->dev, "failed to get device data\n");
2797 		return -ENODEV;
2798 	}
2799 
2800 	nandc->props = dev_data;
2801 
2802 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2803 	nandc->base = devm_ioremap_resource(dev, res);
2804 	if (IS_ERR(nandc->base))
2805 		return PTR_ERR(nandc->base);
2806 
2807 	nandc->base_phys = res->start;
2808 	nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2809 
2810 	nandc->core_clk = devm_clk_get(dev, "core");
2811 	if (IS_ERR(nandc->core_clk))
2812 		return PTR_ERR(nandc->core_clk);
2813 
2814 	nandc->aon_clk = devm_clk_get(dev, "aon");
2815 	if (IS_ERR(nandc->aon_clk))
2816 		return PTR_ERR(nandc->aon_clk);
2817 
2818 	ret = qcom_nandc_parse_dt(pdev);
2819 	if (ret)
2820 		return ret;
2821 
2822 	ret = qcom_nandc_alloc(nandc);
2823 	if (ret)
2824 		goto err_core_clk;
2825 
2826 	ret = clk_prepare_enable(nandc->core_clk);
2827 	if (ret)
2828 		goto err_core_clk;
2829 
2830 	ret = clk_prepare_enable(nandc->aon_clk);
2831 	if (ret)
2832 		goto err_aon_clk;
2833 
2834 	ret = qcom_nandc_setup(nandc);
2835 	if (ret)
2836 		goto err_setup;
2837 
2838 	ret = qcom_probe_nand_devices(nandc);
2839 	if (ret)
2840 		goto err_setup;
2841 
2842 	return 0;
2843 
2844 err_setup:
2845 	clk_disable_unprepare(nandc->aon_clk);
2846 err_aon_clk:
2847 	clk_disable_unprepare(nandc->core_clk);
2848 err_core_clk:
2849 	qcom_nandc_unalloc(nandc);
2850 
2851 	return ret;
2852 }
2853 
2854 static int qcom_nandc_remove(struct platform_device *pdev)
2855 {
2856 	struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2857 	struct qcom_nand_host *host;
2858 
2859 	list_for_each_entry(host, &nandc->host_list, node)
2860 		nand_release(nand_to_mtd(&host->chip));
2861 
2862 	qcom_nandc_unalloc(nandc);
2863 
2864 	clk_disable_unprepare(nandc->aon_clk);
2865 	clk_disable_unprepare(nandc->core_clk);
2866 
2867 	return 0;
2868 }
2869 
2870 static const struct qcom_nandc_props ipq806x_nandc_props = {
2871 	.ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
2872 	.is_bam = false,
2873 	.dev_cmd_reg_start = 0x0,
2874 };
2875 
2876 static const struct qcom_nandc_props ipq4019_nandc_props = {
2877 	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2878 	.is_bam = true,
2879 	.dev_cmd_reg_start = 0x0,
2880 };
2881 
2882 static const struct qcom_nandc_props ipq8074_nandc_props = {
2883 	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2884 	.is_bam = true,
2885 	.dev_cmd_reg_start = 0x7000,
2886 };
2887 
2888 /*
2889  * data will hold a struct pointer containing more differences once we support
2890  * more controller variants
2891  */
2892 static const struct of_device_id qcom_nandc_of_match[] = {
2893 	{
2894 		.compatible = "qcom,ipq806x-nand",
2895 		.data = &ipq806x_nandc_props,
2896 	},
2897 	{
2898 		.compatible = "qcom,ipq4019-nand",
2899 		.data = &ipq4019_nandc_props,
2900 	},
2901 	{
2902 		.compatible = "qcom,ipq8074-nand",
2903 		.data = &ipq8074_nandc_props,
2904 	},
2905 	{}
2906 };
2907 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2908 
2909 static struct platform_driver qcom_nandc_driver = {
2910 	.driver = {
2911 		.name = "qcom-nandc",
2912 		.of_match_table = qcom_nandc_of_match,
2913 	},
2914 	.probe   = qcom_nandc_probe,
2915 	.remove  = qcom_nandc_remove,
2916 };
2917 module_platform_driver(qcom_nandc_driver);
2918 
2919 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2920 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2921 MODULE_LICENSE("GPL v2");
2922