xref: /linux/drivers/mtd/nand/raw/cadence-nand-controller.c (revision a3daad8215143340c0870c5489e599fd059037e9)
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Cadence NAND flash controller driver
4  *
5  * Copyright (C) 2019 Cadence
6  *
7  * Author: Piotr Sroka <piotrs@cadence.com>
8  */
9 
10 #include <linux/bitfield.h>
11 #include <linux/clk.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/interrupt.h>
15 #include <linux/module.h>
16 #include <linux/mtd/mtd.h>
17 #include <linux/mtd/rawnand.h>
18 #include <linux/iopoll.h>
19 #include <linux/of.h>
20 #include <linux/platform_device.h>
21 #include <linux/property.h>
22 #include <linux/slab.h>
23 
24 /*
25  * HPNFC can work in 3 modes:
26  * -  PIO - can work in master or slave DMA
27  * -  CDMA - needs Master DMA for accessing command descriptors.
28  * -  Generic mode - can use only slave DMA.
29  * CDMA and PIO modes can be used to execute only base commands.
30  * Generic mode can be used to execute any command
31  * on NAND flash memory. Driver uses CDMA mode for
32  * block erasing, page reading, page programing.
33  * Generic mode is used for executing rest of commands.
34  */
35 
36 #define MAX_ADDRESS_CYC		6
37 #define MAX_ERASE_ADDRESS_CYC	3
38 #define MAX_DATA_SIZE		0xFFFC
39 #define DMA_DATA_SIZE_ALIGN	8
40 
41 /* Register definition. */
42 /*
43  * Command register 0.
44  * Writing data to this register will initiate a new transaction
45  * of the NF controller.
46  */
47 #define CMD_REG0			0x0000
48 /* Command type field mask. */
49 #define		CMD_REG0_CT		GENMASK(31, 30)
50 /* Command type CDMA. */
51 #define		CMD_REG0_CT_CDMA	0uL
52 /* Command type generic. */
53 #define		CMD_REG0_CT_GEN		3uL
54 /* Command thread number field mask. */
55 #define		CMD_REG0_TN		GENMASK(27, 24)
56 
57 /* Command register 2. */
58 #define CMD_REG2			0x0008
59 /* Command register 3. */
60 #define CMD_REG3			0x000C
61 /* Pointer register to select which thread status will be selected. */
62 #define CMD_STATUS_PTR			0x0010
63 /* Command status register for selected thread. */
64 #define CMD_STATUS			0x0014
65 
66 /* Interrupt status register. */
67 #define INTR_STATUS			0x0110
68 #define		INTR_STATUS_SDMA_ERR	BIT(22)
69 #define		INTR_STATUS_SDMA_TRIGG	BIT(21)
70 #define		INTR_STATUS_UNSUPP_CMD	BIT(19)
71 #define		INTR_STATUS_DDMA_TERR	BIT(18)
72 #define		INTR_STATUS_CDMA_TERR	BIT(17)
73 #define		INTR_STATUS_CDMA_IDL	BIT(16)
74 
75 /* Interrupt enable register. */
76 #define INTR_ENABLE				0x0114
77 #define		INTR_ENABLE_INTR_EN		BIT(31)
78 #define		INTR_ENABLE_SDMA_ERR_EN		BIT(22)
79 #define		INTR_ENABLE_SDMA_TRIGG_EN	BIT(21)
80 #define		INTR_ENABLE_UNSUPP_CMD_EN	BIT(19)
81 #define		INTR_ENABLE_DDMA_TERR_EN	BIT(18)
82 #define		INTR_ENABLE_CDMA_TERR_EN	BIT(17)
83 #define		INTR_ENABLE_CDMA_IDLE_EN	BIT(16)
84 
85 /* Controller internal state. */
86 #define CTRL_STATUS				0x0118
87 #define		CTRL_STATUS_INIT_COMP		BIT(9)
88 #define		CTRL_STATUS_CTRL_BUSY		BIT(8)
89 
90 /* Command Engine threads state. */
91 #define TRD_STATUS				0x0120
92 
93 /* Command Engine interrupt thread error status. */
94 #define TRD_ERR_INT_STATUS			0x0128
95 /* Command Engine interrupt thread error enable. */
96 #define TRD_ERR_INT_STATUS_EN			0x0130
97 /* Command Engine interrupt thread complete status. */
98 #define TRD_COMP_INT_STATUS			0x0138
99 
100 /*
101  * Transfer config 0 register.
102  * Configures data transfer parameters.
103  */
104 #define TRAN_CFG_0				0x0400
105 /* Offset value from the beginning of the page. */
106 #define		TRAN_CFG_0_OFFSET		GENMASK(31, 16)
107 /* Numbers of sectors to transfer within singlNF device's page. */
108 #define		TRAN_CFG_0_SEC_CNT		GENMASK(7, 0)
109 
110 /*
111  * Transfer config 1 register.
112  * Configures data transfer parameters.
113  */
114 #define TRAN_CFG_1				0x0404
115 /* Size of last data sector. */
116 #define		TRAN_CFG_1_LAST_SEC_SIZE	GENMASK(31, 16)
117 /* Size of not-last data sector. */
118 #define		TRAN_CFG_1_SECTOR_SIZE		GENMASK(15, 0)
119 
120 /* ECC engine configuration register 0. */
121 #define ECC_CONFIG_0				0x0428
122 /* Correction strength. */
123 #define		ECC_CONFIG_0_CORR_STR		GENMASK(10, 8)
124 /* Enable erased pages detection mechanism. */
125 #define		ECC_CONFIG_0_ERASE_DET_EN	BIT(1)
126 /* Enable controller ECC check bits generation and correction. */
127 #define		ECC_CONFIG_0_ECC_EN		BIT(0)
128 
129 /* ECC engine configuration register 1. */
130 #define ECC_CONFIG_1				0x042C
131 
132 /* Multiplane settings register. */
133 #define MULTIPLANE_CFG				0x0434
134 /* Cache operation settings. */
135 #define CACHE_CFG				0x0438
136 
137 /* DMA settings register. */
138 #define DMA_SETINGS				0x043C
139 /* Enable SDMA error report on access unprepared slave DMA interface. */
140 #define		DMA_SETINGS_SDMA_ERR_RSP	BIT(17)
141 
142 /* Transferred data block size for the slave DMA module. */
143 #define SDMA_SIZE				0x0440
144 
145 /* Thread number associated with transferred data block
146  * for the slave DMA module.
147  */
148 #define SDMA_TRD_NUM				0x0444
149 /* Thread number mask. */
150 #define		SDMA_TRD_NUM_SDMA_TRD		GENMASK(2, 0)
151 
152 #define CONTROL_DATA_CTRL			0x0494
153 /* Thread number mask. */
154 #define		CONTROL_DATA_CTRL_SIZE		GENMASK(15, 0)
155 
156 #define CTRL_VERSION				0x800
157 #define		CTRL_VERSION_REV		GENMASK(7, 0)
158 
159 /* Available hardware features of the controller. */
160 #define CTRL_FEATURES				0x804
161 /* Support for NV-DDR2/3 work mode. */
162 #define		CTRL_FEATURES_NVDDR_2_3		BIT(28)
163 /* Support for NV-DDR work mode. */
164 #define		CTRL_FEATURES_NVDDR		BIT(27)
165 /* Support for asynchronous work mode. */
166 #define		CTRL_FEATURES_ASYNC		BIT(26)
167 /* Support for asynchronous work mode. */
168 #define		CTRL_FEATURES_N_BANKS		GENMASK(25, 24)
169 /* Slave and Master DMA data width. */
170 #define		CTRL_FEATURES_DMA_DWITH64	BIT(21)
171 /* Availability of Control Data feature.*/
172 #define		CTRL_FEATURES_CONTROL_DATA	BIT(10)
173 
174 /* BCH Engine identification register 0 - correction strengths. */
175 #define BCH_CFG_0				0x838
176 #define		BCH_CFG_0_CORR_CAP_0		GENMASK(7, 0)
177 #define		BCH_CFG_0_CORR_CAP_1		GENMASK(15, 8)
178 #define		BCH_CFG_0_CORR_CAP_2		GENMASK(23, 16)
179 #define		BCH_CFG_0_CORR_CAP_3		GENMASK(31, 24)
180 
181 /* BCH Engine identification register 1 - correction strengths. */
182 #define BCH_CFG_1				0x83C
183 #define		BCH_CFG_1_CORR_CAP_4		GENMASK(7, 0)
184 #define		BCH_CFG_1_CORR_CAP_5		GENMASK(15, 8)
185 #define		BCH_CFG_1_CORR_CAP_6		GENMASK(23, 16)
186 #define		BCH_CFG_1_CORR_CAP_7		GENMASK(31, 24)
187 
188 /* BCH Engine identification register 2 - sector sizes. */
189 #define BCH_CFG_2				0x840
190 #define		BCH_CFG_2_SECT_0		GENMASK(15, 0)
191 #define		BCH_CFG_2_SECT_1		GENMASK(31, 16)
192 
193 /* BCH Engine identification register 3. */
194 #define BCH_CFG_3				0x844
195 #define		BCH_CFG_3_METADATA_SIZE		GENMASK(23, 16)
196 
197 /* Ready/Busy# line status. */
198 #define RBN_SETINGS				0x1004
199 
200 /* Common settings. */
201 #define COMMON_SET				0x1008
202 /* 16 bit device connected to the NAND Flash interface. */
203 #define		COMMON_SET_DEVICE_16BIT		BIT(8)
204 
205 /* Skip_bytes registers. */
206 #define SKIP_BYTES_CONF				0x100C
207 #define		SKIP_BYTES_MARKER_VALUE		GENMASK(31, 16)
208 #define		SKIP_BYTES_NUM_OF_BYTES		GENMASK(7, 0)
209 
210 #define SKIP_BYTES_OFFSET			0x1010
211 #define		 SKIP_BYTES_OFFSET_VALUE	GENMASK(23, 0)
212 
213 /* Timings configuration. */
214 #define ASYNC_TOGGLE_TIMINGS			0x101c
215 #define		ASYNC_TOGGLE_TIMINGS_TRH	GENMASK(28, 24)
216 #define		ASYNC_TOGGLE_TIMINGS_TRP	GENMASK(20, 16)
217 #define		ASYNC_TOGGLE_TIMINGS_TWH	GENMASK(12, 8)
218 #define		ASYNC_TOGGLE_TIMINGS_TWP	GENMASK(4, 0)
219 
220 #define	TIMINGS0				0x1024
221 #define		TIMINGS0_TADL			GENMASK(31, 24)
222 #define		TIMINGS0_TCCS			GENMASK(23, 16)
223 #define		TIMINGS0_TWHR			GENMASK(15, 8)
224 #define		TIMINGS0_TRHW			GENMASK(7, 0)
225 
226 #define	TIMINGS1				0x1028
227 #define		TIMINGS1_TRHZ			GENMASK(31, 24)
228 #define		TIMINGS1_TWB			GENMASK(23, 16)
229 #define		TIMINGS1_TVDLY			GENMASK(7, 0)
230 
231 #define	TIMINGS2				0x102c
232 #define		TIMINGS2_TFEAT			GENMASK(25, 16)
233 #define		TIMINGS2_CS_HOLD_TIME		GENMASK(13, 8)
234 #define		TIMINGS2_CS_SETUP_TIME		GENMASK(5, 0)
235 
236 /* Configuration of the resynchronization of slave DLL of PHY. */
237 #define DLL_PHY_CTRL				0x1034
238 #define		DLL_PHY_CTRL_DLL_RST_N		BIT(24)
239 #define		DLL_PHY_CTRL_EXTENDED_WR_MODE	BIT(17)
240 #define		DLL_PHY_CTRL_EXTENDED_RD_MODE	BIT(16)
241 #define		DLL_PHY_CTRL_RS_HIGH_WAIT_CNT	GENMASK(11, 8)
242 #define		DLL_PHY_CTRL_RS_IDLE_CNT	GENMASK(7, 0)
243 
244 /* Register controlling DQ related timing. */
245 #define PHY_DQ_TIMING				0x2000
246 /* Register controlling DSQ related timing.  */
247 #define PHY_DQS_TIMING				0x2004
248 #define		PHY_DQS_TIMING_DQS_SEL_OE_END	GENMASK(3, 0)
249 #define		PHY_DQS_TIMING_PHONY_DQS_SEL	BIT(16)
250 #define		PHY_DQS_TIMING_USE_PHONY_DQS	BIT(20)
251 
252 /* Register controlling the gate and loopback control related timing. */
253 #define PHY_GATE_LPBK_CTRL			0x2008
254 #define		PHY_GATE_LPBK_CTRL_RDS		GENMASK(24, 19)
255 
256 /* Register holds the control for the master DLL logic. */
257 #define PHY_DLL_MASTER_CTRL			0x200C
258 #define		PHY_DLL_MASTER_CTRL_BYPASS_MODE	BIT(23)
259 
260 /* Register holds the control for the slave DLL logic. */
261 #define PHY_DLL_SLAVE_CTRL			0x2010
262 
263 /* This register handles the global control settings for the PHY. */
264 #define PHY_CTRL				0x2080
265 #define		PHY_CTRL_SDR_DQS		BIT(14)
266 #define		PHY_CTRL_PHONY_DQS		GENMASK(9, 4)
267 
268 /*
269  * This register handles the global control settings
270  * for the termination selects for reads.
271  */
272 #define PHY_TSEL				0x2084
273 
274 /* Generic command layout. */
275 #define GCMD_LAY_CS			GENMASK_ULL(11, 8)
276 /*
277  * This bit informs the minicotroller if it has to wait for tWB
278  * after sending the last CMD/ADDR/DATA in the sequence.
279  */
280 #define GCMD_LAY_TWB			BIT_ULL(6)
281 /* Type of generic instruction. */
282 #define GCMD_LAY_INSTR			GENMASK_ULL(5, 0)
283 
284 /* Generic CMD sequence type. */
285 #define		GCMD_LAY_INSTR_CMD	0
286 /* Generic ADDR sequence type. */
287 #define		GCMD_LAY_INSTR_ADDR	1
288 /* Generic data transfer sequence type. */
289 #define		GCMD_LAY_INSTR_DATA	2
290 
291 /* Input part of generic command type of input is command. */
292 #define GCMD_LAY_INPUT_CMD		GENMASK_ULL(23, 16)
293 
294 /* Generic command address sequence - address fields. */
295 #define GCMD_LAY_INPUT_ADDR		GENMASK_ULL(63, 16)
296 /* Generic command address sequence - address size. */
297 #define GCMD_LAY_INPUT_ADDR_SIZE	GENMASK_ULL(13, 11)
298 
299 /* Transfer direction field of generic command data sequence. */
300 #define GCMD_DIR			BIT_ULL(11)
301 /* Read transfer direction of generic command data sequence. */
302 #define		GCMD_DIR_READ		0
303 /* Write transfer direction of generic command data sequence. */
304 #define		GCMD_DIR_WRITE		1
305 
306 /* ECC enabled flag of generic command data sequence - ECC enabled. */
307 #define GCMD_ECC_EN			BIT_ULL(12)
308 /* Generic command data sequence - sector size. */
309 #define GCMD_SECT_SIZE			GENMASK_ULL(31, 16)
310 /* Generic command data sequence - sector count. */
311 #define GCMD_SECT_CNT			GENMASK_ULL(39, 32)
312 /* Generic command data sequence - last sector size. */
313 #define GCMD_LAST_SIZE			GENMASK_ULL(55, 40)
314 
315 /* CDMA descriptor fields. */
316 /* Erase command type of CDMA descriptor. */
317 #define CDMA_CT_ERASE		0x1000
318 /* Program page command type of CDMA descriptor. */
319 #define CDMA_CT_WR		0x2100
320 /* Read page command type of CDMA descriptor. */
321 #define CDMA_CT_RD		0x2200
322 
323 /* Flash pointer memory shift. */
324 #define CDMA_CFPTR_MEM_SHIFT	24
325 /* Flash pointer memory mask. */
326 #define CDMA_CFPTR_MEM		GENMASK(26, 24)
327 
328 /*
329  * Command DMA descriptor flags. If set causes issue interrupt after
330  * the completion of descriptor processing.
331  */
332 #define CDMA_CF_INT		BIT(8)
333 /*
334  * Command DMA descriptor flags - the next descriptor
335  * address field is valid and descriptor processing should continue.
336  */
337 #define CDMA_CF_CONT		BIT(9)
338 /* DMA master flag of command DMA descriptor. */
339 #define CDMA_CF_DMA_MASTER	BIT(10)
340 
341 /* Operation complete status of command descriptor. */
342 #define CDMA_CS_COMP		BIT(15)
343 /* Operation complete status of command descriptor. */
344 /* Command descriptor status - operation fail. */
345 #define CDMA_CS_FAIL		BIT(14)
346 /* Command descriptor status - page erased. */
347 #define CDMA_CS_ERP		BIT(11)
348 /* Command descriptor status - timeout occurred. */
349 #define CDMA_CS_TOUT		BIT(10)
350 /*
351  * Maximum amount of correction applied to one ECC sector.
352  * It is part of command descriptor status.
353  */
354 #define CDMA_CS_MAXERR		GENMASK(9, 2)
355 /* Command descriptor status - uncorrectable ECC error. */
356 #define CDMA_CS_UNCE		BIT(1)
357 /* Command descriptor status - descriptor error. */
358 #define CDMA_CS_ERR		BIT(0)
359 
360 /* Status of operation - OK. */
361 #define STAT_OK			0
362 /* Status of operation - FAIL. */
363 #define STAT_FAIL		2
364 /* Status of operation - uncorrectable ECC error. */
365 #define STAT_ECC_UNCORR		3
366 /* Status of operation - page erased. */
367 #define STAT_ERASED		5
368 /* Status of operation - correctable ECC error. */
369 #define STAT_ECC_CORR		6
370 /* Status of operation - unsuspected state. */
371 #define STAT_UNKNOWN		7
372 /* Status of operation - operation is not completed yet. */
373 #define STAT_BUSY		0xFF
374 
375 #define BCH_MAX_NUM_CORR_CAPS		8
376 #define BCH_MAX_NUM_SECTOR_SIZES	2
377 
378 struct cadence_nand_timings {
379 	u32 async_toggle_timings;
380 	u32 timings0;
381 	u32 timings1;
382 	u32 timings2;
383 	u32 dll_phy_ctrl;
384 	u32 phy_ctrl;
385 	u32 phy_dqs_timing;
386 	u32 phy_gate_lpbk_ctrl;
387 };
388 
389 /* Command DMA descriptor. */
390 struct cadence_nand_cdma_desc {
391 	/* Next descriptor address. */
392 	u64 next_pointer;
393 
394 	/* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
395 	u32 flash_pointer;
396 	/*field appears in HPNFC version 13*/
397 	u16 bank;
398 	u16 rsvd0;
399 
400 	/* Operation the controller needs to perform. */
401 	u16 command_type;
402 	u16 rsvd1;
403 	/* Flags for operation of this command. */
404 	u16 command_flags;
405 	u16 rsvd2;
406 
407 	/* System/host memory address required for data DMA commands. */
408 	u64 memory_pointer;
409 
410 	/* Status of operation. */
411 	u32 status;
412 	u32 rsvd3;
413 
414 	/* Address pointer to sync buffer location. */
415 	u64 sync_flag_pointer;
416 
417 	/* Controls the buffer sync mechanism. */
418 	u32 sync_arguments;
419 	u32 rsvd4;
420 
421 	/* Control data pointer. */
422 	u64 ctrl_data_ptr;
423 };
424 
425 /* Interrupt status. */
426 struct cadence_nand_irq_status {
427 	/* Thread operation complete status. */
428 	u32 trd_status;
429 	/* Thread operation error. */
430 	u32 trd_error;
431 	/* Controller status. */
432 	u32 status;
433 };
434 
435 /* Cadence NAND flash controller capabilities get from driver data. */
436 struct cadence_nand_dt_devdata {
437 	/* Skew value of the output signals of the NAND Flash interface. */
438 	u32 if_skew;
439 	/* It informs if slave DMA interface is connected to DMA engine. */
440 	unsigned int has_dma:1;
441 };
442 
443 /* Cadence NAND flash controller capabilities read from registers. */
444 struct cdns_nand_caps {
445 	/* Maximum number of banks supported by hardware. */
446 	u8 max_banks;
447 	/* Slave and Master DMA data width in bytes (4 or 8). */
448 	u8 data_dma_width;
449 	/* Control Data feature supported. */
450 	bool data_control_supp;
451 	/* Is PHY type DLL. */
452 	bool is_phy_type_dll;
453 };
454 
455 struct cdns_nand_ctrl {
456 	struct device *dev;
457 	struct nand_controller controller;
458 	struct cadence_nand_cdma_desc *cdma_desc;
459 	/* IP capability. */
460 	const struct cadence_nand_dt_devdata *caps1;
461 	struct cdns_nand_caps caps2;
462 	u8 ctrl_rev;
463 	dma_addr_t dma_cdma_desc;
464 	u8 *buf;
465 	u32 buf_size;
466 	u8 curr_corr_str_idx;
467 
468 	/* Register interface. */
469 	void __iomem *reg;
470 
471 	struct {
472 		void __iomem *virt;
473 		dma_addr_t dma;
474 		dma_addr_t iova_dma;
475 		u32 size;
476 	} io;
477 
478 	int irq;
479 	/* Interrupts that have happened. */
480 	struct cadence_nand_irq_status irq_status;
481 	/* Interrupts we are waiting for. */
482 	struct cadence_nand_irq_status irq_mask;
483 	struct completion complete;
484 	/* Protect irq_mask and irq_status. */
485 	spinlock_t irq_lock;
486 
487 	int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
488 	struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
489 	struct nand_ecc_caps ecc_caps;
490 
491 	int curr_trans_type;
492 
493 	struct dma_chan *dmac;
494 
495 	u32 nf_clk_rate;
496 	/*
497 	 * Estimated Board delay. The value includes the total
498 	 * round trip delay for the signals and is used for deciding on values
499 	 * associated with data read capture.
500 	 */
501 	u32 board_delay;
502 
503 	struct nand_chip *selected_chip;
504 
505 	unsigned long assigned_cs;
506 	struct list_head chips;
507 	u8 bch_metadata_size;
508 };
509 
510 struct cdns_nand_chip {
511 	struct cadence_nand_timings timings;
512 	struct nand_chip chip;
513 	u8 nsels;
514 	struct list_head node;
515 
516 	/*
517 	 * part of oob area of NAND flash memory page.
518 	 * This part is available for user to read or write.
519 	 */
520 	u32 avail_oob_size;
521 
522 	/* Sector size. There are few sectors per mtd->writesize */
523 	u32 sector_size;
524 	u32 sector_count;
525 
526 	/* Offset of BBM. */
527 	u8 bbm_offs;
528 	/* Number of bytes reserved for BBM. */
529 	u8 bbm_len;
530 	/* ECC strength index. */
531 	u8 corr_str_idx;
532 
533 	u8 cs[] __counted_by(nsels);
534 };
535 
536 static inline struct
to_cdns_nand_chip(struct nand_chip * chip)537 cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
538 {
539 	return container_of(chip, struct cdns_nand_chip, chip);
540 }
541 
542 static inline struct
to_cdns_nand_ctrl(struct nand_controller * controller)543 cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
544 {
545 	return container_of(controller, struct cdns_nand_ctrl, controller);
546 }
547 
548 static bool
cadence_nand_dma_buf_ok(struct cdns_nand_ctrl * cdns_ctrl,const void * buf,u32 buf_len)549 cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
550 			u32 buf_len)
551 {
552 	u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
553 
554 	return buf && virt_addr_valid(buf) &&
555 		likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
556 		likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
557 }
558 
cadence_nand_wait_for_value(struct cdns_nand_ctrl * cdns_ctrl,u32 reg_offset,u32 timeout_us,u32 mask,bool is_clear)559 static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
560 				       u32 reg_offset, u32 timeout_us,
561 				       u32 mask, bool is_clear)
562 {
563 	u32 val;
564 	int ret;
565 
566 	ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
567 					 val, !(val & mask) == is_clear,
568 					 10, timeout_us);
569 
570 	if (ret < 0) {
571 		dev_err(cdns_ctrl->dev,
572 			"Timeout while waiting for reg %x with mask %x is clear %d\n",
573 			reg_offset, mask, is_clear);
574 	}
575 
576 	return ret;
577 }
578 
cadence_nand_set_ecc_enable(struct cdns_nand_ctrl * cdns_ctrl,bool enable)579 static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
580 				       bool enable)
581 {
582 	u32 reg;
583 
584 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
585 					1000000,
586 					CTRL_STATUS_CTRL_BUSY, true))
587 		return -ETIMEDOUT;
588 
589 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
590 
591 	if (enable)
592 		reg |= ECC_CONFIG_0_ECC_EN;
593 	else
594 		reg &= ~ECC_CONFIG_0_ECC_EN;
595 
596 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
597 
598 	return 0;
599 }
600 
cadence_nand_set_ecc_strength(struct cdns_nand_ctrl * cdns_ctrl,u8 corr_str_idx)601 static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
602 					  u8 corr_str_idx)
603 {
604 	u32 reg;
605 
606 	if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
607 		return;
608 
609 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
610 	reg &= ~ECC_CONFIG_0_CORR_STR;
611 	reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
612 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
613 
614 	cdns_ctrl->curr_corr_str_idx = corr_str_idx;
615 }
616 
cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl * cdns_ctrl,u8 strength)617 static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
618 					     u8 strength)
619 {
620 	int i, corr_str_idx = -1;
621 
622 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
623 		if (cdns_ctrl->ecc_strengths[i] == strength) {
624 			corr_str_idx = i;
625 			break;
626 		}
627 	}
628 
629 	return corr_str_idx;
630 }
631 
cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl * cdns_ctrl,u16 marker_value)632 static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
633 					    u16 marker_value)
634 {
635 	u32 reg;
636 
637 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
638 					1000000,
639 					CTRL_STATUS_CTRL_BUSY, true))
640 		return -ETIMEDOUT;
641 
642 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
643 	reg &= ~SKIP_BYTES_MARKER_VALUE;
644 	reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
645 			  marker_value);
646 
647 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
648 
649 	return 0;
650 }
651 
cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl * cdns_ctrl,u8 num_of_bytes,u32 offset_value,int enable)652 static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
653 					    u8 num_of_bytes,
654 					    u32 offset_value,
655 					    int enable)
656 {
657 	u32 reg, skip_bytes_offset;
658 
659 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
660 					1000000,
661 					CTRL_STATUS_CTRL_BUSY, true))
662 		return -ETIMEDOUT;
663 
664 	if (!enable) {
665 		num_of_bytes = 0;
666 		offset_value = 0;
667 	}
668 
669 	reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
670 	reg &= ~SKIP_BYTES_NUM_OF_BYTES;
671 	reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
672 			  num_of_bytes);
673 	skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
674 				       offset_value);
675 
676 	writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
677 	writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
678 
679 	return 0;
680 }
681 
682 /* Functions enables/disables hardware detection of erased data */
cadence_nand_set_erase_detection(struct cdns_nand_ctrl * cdns_ctrl,bool enable,u8 bitflips_threshold)683 static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
684 					     bool enable,
685 					     u8 bitflips_threshold)
686 {
687 	u32 reg;
688 
689 	reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
690 
691 	if (enable)
692 		reg |= ECC_CONFIG_0_ERASE_DET_EN;
693 	else
694 		reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
695 
696 	writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
697 	writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
698 }
699 
cadence_nand_set_access_width16(struct cdns_nand_ctrl * cdns_ctrl,bool bit_bus16)700 static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
701 					   bool bit_bus16)
702 {
703 	u32 reg;
704 
705 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
706 					1000000,
707 					CTRL_STATUS_CTRL_BUSY, true))
708 		return -ETIMEDOUT;
709 
710 	reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
711 
712 	if (!bit_bus16)
713 		reg &= ~COMMON_SET_DEVICE_16BIT;
714 	else
715 		reg |= COMMON_SET_DEVICE_16BIT;
716 	writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
717 
718 	return 0;
719 }
720 
721 static void
cadence_nand_clear_interrupt(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_status)722 cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
723 			     struct cadence_nand_irq_status *irq_status)
724 {
725 	writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
726 	writel_relaxed(irq_status->trd_status,
727 		       cdns_ctrl->reg + TRD_COMP_INT_STATUS);
728 	writel_relaxed(irq_status->trd_error,
729 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS);
730 }
731 
732 static void
cadence_nand_read_int_status(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_status)733 cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
734 			     struct cadence_nand_irq_status *irq_status)
735 {
736 	irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
737 	irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
738 					       + TRD_COMP_INT_STATUS);
739 	irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
740 					      + TRD_ERR_INT_STATUS);
741 }
742 
irq_detected(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_status)743 static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
744 			struct cadence_nand_irq_status *irq_status)
745 {
746 	cadence_nand_read_int_status(cdns_ctrl, irq_status);
747 
748 	return irq_status->status || irq_status->trd_status ||
749 		irq_status->trd_error;
750 }
751 
cadence_nand_reset_irq(struct cdns_nand_ctrl * cdns_ctrl)752 static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
753 {
754 	unsigned long flags;
755 
756 	spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
757 	memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
758 	memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
759 	spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
760 }
761 
762 /*
763  * This is the interrupt service routine. It handles all interrupts
764  * sent to this device.
765  */
cadence_nand_isr(int irq,void * dev_id)766 static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
767 {
768 	struct cdns_nand_ctrl *cdns_ctrl = dev_id;
769 	struct cadence_nand_irq_status irq_status;
770 	irqreturn_t result = IRQ_NONE;
771 
772 	spin_lock(&cdns_ctrl->irq_lock);
773 
774 	if (irq_detected(cdns_ctrl, &irq_status)) {
775 		/* Handle interrupt. */
776 		/* First acknowledge it. */
777 		cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
778 		/* Status in the device context for someone to read. */
779 		cdns_ctrl->irq_status.status |= irq_status.status;
780 		cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
781 		cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
782 		/* Notify anyone who cares that it happened. */
783 		complete(&cdns_ctrl->complete);
784 		/* Tell the OS that we've handled this. */
785 		result = IRQ_HANDLED;
786 	}
787 	spin_unlock(&cdns_ctrl->irq_lock);
788 
789 	return result;
790 }
791 
cadence_nand_set_irq_mask(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_mask)792 static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
793 				      struct cadence_nand_irq_status *irq_mask)
794 {
795 	writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
796 		       cdns_ctrl->reg + INTR_ENABLE);
797 
798 	writel_relaxed(irq_mask->trd_error,
799 		       cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
800 }
801 
802 static void
cadence_nand_wait_for_irq(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_mask,struct cadence_nand_irq_status * irq_status)803 cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
804 			  struct cadence_nand_irq_status *irq_mask,
805 			  struct cadence_nand_irq_status *irq_status)
806 {
807 	unsigned long timeout = msecs_to_jiffies(10000);
808 	unsigned long time_left;
809 
810 	time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
811 						timeout);
812 
813 	*irq_status = cdns_ctrl->irq_status;
814 	if (time_left == 0) {
815 		/* Timeout error. */
816 		dev_err(cdns_ctrl->dev, "timeout occurred:\n");
817 		dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
818 			irq_status->status, irq_mask->status);
819 		dev_err(cdns_ctrl->dev,
820 			"\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
821 			irq_status->trd_status, irq_mask->trd_status);
822 		dev_err(cdns_ctrl->dev,
823 			"\t trd_error = 0x%x, trd_error mask = 0x%x\n",
824 			irq_status->trd_error, irq_mask->trd_error);
825 	}
826 }
827 
828 /* Execute generic command on NAND controller. */
cadence_nand_generic_cmd_send(struct cdns_nand_ctrl * cdns_ctrl,u8 chip_nr,u64 mini_ctrl_cmd)829 static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
830 					 u8 chip_nr,
831 					 u64 mini_ctrl_cmd)
832 {
833 	u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
834 
835 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
836 	mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
837 	mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
838 
839 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
840 					1000000,
841 					CTRL_STATUS_CTRL_BUSY, true))
842 		return -ETIMEDOUT;
843 
844 	cadence_nand_reset_irq(cdns_ctrl);
845 
846 	writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
847 	writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
848 
849 	/* Select generic command. */
850 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
851 	/* Thread number. */
852 	reg |= FIELD_PREP(CMD_REG0_TN, 0);
853 
854 	/* Issue command. */
855 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
856 
857 	return 0;
858 }
859 
860 /* Wait for data on slave DMA interface. */
cadence_nand_wait_on_sdma(struct cdns_nand_ctrl * cdns_ctrl,u8 * out_sdma_trd,u32 * out_sdma_size)861 static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
862 				     u8 *out_sdma_trd,
863 				     u32 *out_sdma_size)
864 {
865 	struct cadence_nand_irq_status irq_mask, irq_status;
866 
867 	irq_mask.trd_status = 0;
868 	irq_mask.trd_error = 0;
869 	irq_mask.status = INTR_STATUS_SDMA_TRIGG
870 		| INTR_STATUS_SDMA_ERR
871 		| INTR_STATUS_UNSUPP_CMD;
872 
873 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
874 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
875 	if (irq_status.status == 0) {
876 		dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
877 		return -ETIMEDOUT;
878 	}
879 
880 	if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
881 		*out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
882 		*out_sdma_trd  = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
883 		*out_sdma_trd =
884 			FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
885 	} else {
886 		dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
887 			irq_status.status);
888 		return -EIO;
889 	}
890 
891 	return 0;
892 }
893 
cadence_nand_get_caps(struct cdns_nand_ctrl * cdns_ctrl)894 static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
895 {
896 	u32  reg;
897 
898 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
899 
900 	cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
901 
902 	if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
903 		cdns_ctrl->caps2.data_dma_width = 8;
904 	else
905 		cdns_ctrl->caps2.data_dma_width = 4;
906 
907 	if (reg & CTRL_FEATURES_CONTROL_DATA)
908 		cdns_ctrl->caps2.data_control_supp = true;
909 
910 	if (reg & (CTRL_FEATURES_NVDDR_2_3
911 		   | CTRL_FEATURES_NVDDR))
912 		cdns_ctrl->caps2.is_phy_type_dll = true;
913 }
914 
915 /* Prepare CDMA descriptor. */
916 static void
cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl * cdns_ctrl,char nf_mem,u32 flash_ptr,dma_addr_t mem_ptr,dma_addr_t ctrl_data_ptr,u16 ctype)917 cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
918 			       char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
919 				   dma_addr_t ctrl_data_ptr, u16 ctype)
920 {
921 	struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
922 
923 	memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
924 
925 	/* Set fields for one descriptor. */
926 	cdma_desc->flash_pointer = flash_ptr;
927 	if (cdns_ctrl->ctrl_rev >= 13)
928 		cdma_desc->bank = nf_mem;
929 	else
930 		cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
931 
932 	cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
933 	cdma_desc->command_flags  |= CDMA_CF_INT;
934 
935 	cdma_desc->memory_pointer = mem_ptr;
936 	cdma_desc->status = 0;
937 	cdma_desc->sync_flag_pointer = 0;
938 	cdma_desc->sync_arguments = 0;
939 
940 	cdma_desc->command_type = ctype;
941 	cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
942 }
943 
cadence_nand_check_desc_error(struct cdns_nand_ctrl * cdns_ctrl,u32 desc_status)944 static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
945 					u32 desc_status)
946 {
947 	if (desc_status & CDMA_CS_ERP)
948 		return STAT_ERASED;
949 
950 	if (desc_status & CDMA_CS_UNCE)
951 		return STAT_ECC_UNCORR;
952 
953 	if (desc_status & CDMA_CS_ERR) {
954 		dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
955 		return STAT_FAIL;
956 	}
957 
958 	if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
959 		return STAT_ECC_CORR;
960 
961 	return STAT_FAIL;
962 }
963 
cadence_nand_cdma_finish(struct cdns_nand_ctrl * cdns_ctrl)964 static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
965 {
966 	struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
967 	u8 status = STAT_BUSY;
968 
969 	if (desc_ptr->status & CDMA_CS_FAIL) {
970 		status = cadence_nand_check_desc_error(cdns_ctrl,
971 						       desc_ptr->status);
972 		dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
973 	} else if (desc_ptr->status & CDMA_CS_COMP) {
974 		/* Descriptor finished with no errors. */
975 		if (desc_ptr->command_flags & CDMA_CF_CONT) {
976 			dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
977 			status = STAT_UNKNOWN;
978 		} else {
979 			/* Last descriptor.  */
980 			status = STAT_OK;
981 		}
982 	}
983 
984 	return status;
985 }
986 
cadence_nand_cdma_send(struct cdns_nand_ctrl * cdns_ctrl,u8 thread)987 static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
988 				  u8 thread)
989 {
990 	u32 reg;
991 	int status;
992 
993 	/* Wait for thread ready. */
994 	status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
995 					     1000000,
996 					     BIT(thread), true);
997 	if (status)
998 		return status;
999 
1000 	cadence_nand_reset_irq(cdns_ctrl);
1001 	reinit_completion(&cdns_ctrl->complete);
1002 
1003 	writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1004 		       cdns_ctrl->reg + CMD_REG2);
1005 	writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1006 
1007 	/* Select CDMA mode. */
1008 	reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1009 	/* Thread number. */
1010 	reg |= FIELD_PREP(CMD_REG0_TN, thread);
1011 	/* Issue command. */
1012 	writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1013 
1014 	return 0;
1015 }
1016 
1017 /* Send SDMA command and wait for finish. */
1018 static u32
cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl * cdns_ctrl,u8 thread)1019 cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1020 				u8 thread)
1021 {
1022 	struct cadence_nand_irq_status irq_mask, irq_status = {0};
1023 	int status;
1024 
1025 	irq_mask.trd_status = BIT(thread);
1026 	irq_mask.trd_error = BIT(thread);
1027 	irq_mask.status = INTR_STATUS_CDMA_TERR;
1028 
1029 	cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1030 
1031 	status = cadence_nand_cdma_send(cdns_ctrl, thread);
1032 	if (status)
1033 		return status;
1034 
1035 	cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1036 
1037 	if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1038 	    irq_status.trd_error == 0) {
1039 		dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1040 		return -ETIMEDOUT;
1041 	}
1042 	if (irq_status.status & irq_mask.status) {
1043 		dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1044 		return -EIO;
1045 	}
1046 
1047 	return 0;
1048 }
1049 
1050 /*
1051  * ECC size depends on configured ECC strength and on maximum supported
1052  * ECC step size.
1053  */
cadence_nand_calc_ecc_bytes(int max_step_size,int strength)1054 static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1055 {
1056 	int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1057 
1058 	return ALIGN(nbytes, 2);
1059 }
1060 
1061 #define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1062 	static int \
1063 	cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1064 						    int strength)\
1065 	{\
1066 		return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1067 	}
1068 
1069 CADENCE_NAND_CALC_ECC_BYTES(256)
1070 CADENCE_NAND_CALC_ECC_BYTES(512)
1071 CADENCE_NAND_CALC_ECC_BYTES(1024)
1072 CADENCE_NAND_CALC_ECC_BYTES(2048)
1073 CADENCE_NAND_CALC_ECC_BYTES(4096)
1074 
1075 /* Function reads BCH capabilities. */
cadence_nand_read_bch_caps(struct cdns_nand_ctrl * cdns_ctrl)1076 static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1077 {
1078 	struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1079 	int max_step_size = 0, nstrengths, i;
1080 	u32 reg;
1081 
1082 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1083 	cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1084 	if (cdns_ctrl->bch_metadata_size < 4) {
1085 		dev_err(cdns_ctrl->dev,
1086 			"Driver needs at least 4 bytes of BCH meta data\n");
1087 		return -EIO;
1088 	}
1089 
1090 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1091 	cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1092 	cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1093 	cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1094 	cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1095 
1096 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1097 	cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1098 	cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1099 	cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1100 	cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1101 
1102 	reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1103 	cdns_ctrl->ecc_stepinfos[0].stepsize =
1104 		FIELD_GET(BCH_CFG_2_SECT_0, reg);
1105 
1106 	cdns_ctrl->ecc_stepinfos[1].stepsize =
1107 		FIELD_GET(BCH_CFG_2_SECT_1, reg);
1108 
1109 	nstrengths = 0;
1110 	for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1111 		if (cdns_ctrl->ecc_strengths[i] != 0)
1112 			nstrengths++;
1113 	}
1114 
1115 	ecc_caps->nstepinfos = 0;
1116 	for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1117 		/* ECC strengths are common for all step infos. */
1118 		cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1119 		cdns_ctrl->ecc_stepinfos[i].strengths =
1120 			cdns_ctrl->ecc_strengths;
1121 
1122 		if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1123 			ecc_caps->nstepinfos++;
1124 
1125 		if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1126 			max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1127 	}
1128 	ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1129 
1130 	switch (max_step_size) {
1131 	case 256:
1132 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1133 		break;
1134 	case 512:
1135 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1136 		break;
1137 	case 1024:
1138 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1139 		break;
1140 	case 2048:
1141 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1142 		break;
1143 	case 4096:
1144 		ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1145 		break;
1146 	default:
1147 		dev_err(cdns_ctrl->dev,
1148 			"Unsupported sector size(ecc step size) %d\n",
1149 			max_step_size);
1150 		return -EIO;
1151 	}
1152 
1153 	return 0;
1154 }
1155 
1156 /* Hardware initialization. */
cadence_nand_hw_init(struct cdns_nand_ctrl * cdns_ctrl)1157 static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1158 {
1159 	int status;
1160 	u32 reg;
1161 
1162 	status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1163 					     1000000,
1164 					     CTRL_STATUS_INIT_COMP, false);
1165 	if (status)
1166 		return status;
1167 
1168 	reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1169 	cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1170 
1171 	dev_info(cdns_ctrl->dev,
1172 		 "%s: cadence nand controller version reg %x\n",
1173 		 __func__, reg);
1174 
1175 	/* Disable cache and multiplane. */
1176 	writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1177 	writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1178 
1179 	/* Clear all interrupts. */
1180 	writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1181 
1182 	cadence_nand_get_caps(cdns_ctrl);
1183 	if (cadence_nand_read_bch_caps(cdns_ctrl))
1184 		return -EIO;
1185 
1186 #ifndef CONFIG_64BIT
1187 	if (cdns_ctrl->caps2.data_dma_width == 8) {
1188 		dev_err(cdns_ctrl->dev,
1189 			"cannot access 64-bit dma on !64-bit architectures");
1190 		return -EIO;
1191 	}
1192 #endif
1193 
1194 	/*
1195 	 * Set IO width access to 8.
1196 	 * It is because during SW device discovering width access
1197 	 * is expected to be 8.
1198 	 */
1199 	status = cadence_nand_set_access_width16(cdns_ctrl, false);
1200 
1201 	return status;
1202 }
1203 
1204 #define TT_MAIN_OOB_AREAS	2
1205 #define TT_RAW_PAGE		3
1206 #define TT_BBM			4
1207 #define TT_MAIN_OOB_AREA_EXT	5
1208 
1209 /* Prepare size of data to transfer. */
1210 static void
cadence_nand_prepare_data_size(struct nand_chip * chip,int transfer_type)1211 cadence_nand_prepare_data_size(struct nand_chip *chip,
1212 			       int transfer_type)
1213 {
1214 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1215 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1216 	struct mtd_info *mtd = nand_to_mtd(chip);
1217 	u32 sec_size = 0, offset = 0, sec_cnt = 1;
1218 	u32 last_sec_size = cdns_chip->sector_size;
1219 	u32 data_ctrl_size = 0;
1220 	u32 reg = 0;
1221 
1222 	if (cdns_ctrl->curr_trans_type == transfer_type)
1223 		return;
1224 
1225 	switch (transfer_type) {
1226 	case TT_MAIN_OOB_AREA_EXT:
1227 		sec_cnt = cdns_chip->sector_count;
1228 		sec_size = cdns_chip->sector_size;
1229 		data_ctrl_size = cdns_chip->avail_oob_size;
1230 		break;
1231 	case TT_MAIN_OOB_AREAS:
1232 		sec_cnt = cdns_chip->sector_count;
1233 		last_sec_size = cdns_chip->sector_size
1234 			+ cdns_chip->avail_oob_size;
1235 		sec_size = cdns_chip->sector_size;
1236 		break;
1237 	case TT_RAW_PAGE:
1238 		last_sec_size = mtd->writesize + mtd->oobsize;
1239 		break;
1240 	case TT_BBM:
1241 		offset = mtd->writesize + cdns_chip->bbm_offs;
1242 		last_sec_size = 8;
1243 		break;
1244 	}
1245 
1246 	reg = 0;
1247 	reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1248 	reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1249 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1250 
1251 	reg = 0;
1252 	reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1253 	reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1254 	writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1255 
1256 	if (cdns_ctrl->caps2.data_control_supp) {
1257 		reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1258 		reg &= ~CONTROL_DATA_CTRL_SIZE;
1259 		reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1260 		writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1261 	}
1262 
1263 	cdns_ctrl->curr_trans_type = transfer_type;
1264 }
1265 
1266 static int
cadence_nand_cdma_transfer(struct cdns_nand_ctrl * cdns_ctrl,u8 chip_nr,int page,void * buf,void * ctrl_dat,u32 buf_size,u32 ctrl_dat_size,enum dma_data_direction dir,bool with_ecc)1267 cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1268 			   int page, void *buf, void *ctrl_dat, u32 buf_size,
1269 			   u32 ctrl_dat_size, enum dma_data_direction dir,
1270 			   bool with_ecc)
1271 {
1272 	dma_addr_t dma_buf, dma_ctrl_dat = 0;
1273 	u8 thread_nr = chip_nr;
1274 	int status;
1275 	u16 ctype;
1276 
1277 	if (dir == DMA_FROM_DEVICE)
1278 		ctype = CDMA_CT_RD;
1279 	else
1280 		ctype = CDMA_CT_WR;
1281 
1282 	cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1283 
1284 	dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1285 	if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1286 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1287 		return -EIO;
1288 	}
1289 
1290 	if (ctrl_dat && ctrl_dat_size) {
1291 		dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1292 					      ctrl_dat_size, dir);
1293 		if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1294 			dma_unmap_single(cdns_ctrl->dev, dma_buf,
1295 					 buf_size, dir);
1296 			dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1297 			return -EIO;
1298 		}
1299 	}
1300 
1301 	cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1302 				       dma_buf, dma_ctrl_dat, ctype);
1303 
1304 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1305 
1306 	dma_unmap_single(cdns_ctrl->dev, dma_buf,
1307 			 buf_size, dir);
1308 
1309 	if (ctrl_dat && ctrl_dat_size)
1310 		dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1311 				 ctrl_dat_size, dir);
1312 	if (status)
1313 		return status;
1314 
1315 	return cadence_nand_cdma_finish(cdns_ctrl);
1316 }
1317 
cadence_nand_set_timings(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_timings * t)1318 static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1319 				     struct cadence_nand_timings *t)
1320 {
1321 	writel_relaxed(t->async_toggle_timings,
1322 		       cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1323 	writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1324 	writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1325 	writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1326 
1327 	if (cdns_ctrl->caps2.is_phy_type_dll)
1328 		writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1329 
1330 	writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1331 
1332 	if (cdns_ctrl->caps2.is_phy_type_dll) {
1333 		writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1334 		writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1335 		writel_relaxed(t->phy_dqs_timing,
1336 			       cdns_ctrl->reg + PHY_DQS_TIMING);
1337 		writel_relaxed(t->phy_gate_lpbk_ctrl,
1338 			       cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1339 		writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1340 			       cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1341 		writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1342 	}
1343 }
1344 
cadence_nand_select_target(struct nand_chip * chip)1345 static int cadence_nand_select_target(struct nand_chip *chip)
1346 {
1347 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1348 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1349 
1350 	if (chip == cdns_ctrl->selected_chip)
1351 		return 0;
1352 
1353 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1354 					1000000,
1355 					CTRL_STATUS_CTRL_BUSY, true))
1356 		return -ETIMEDOUT;
1357 
1358 	cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1359 
1360 	cadence_nand_set_ecc_strength(cdns_ctrl,
1361 				      cdns_chip->corr_str_idx);
1362 
1363 	cadence_nand_set_erase_detection(cdns_ctrl, true,
1364 					 chip->ecc.strength);
1365 
1366 	cdns_ctrl->curr_trans_type = -1;
1367 	cdns_ctrl->selected_chip = chip;
1368 
1369 	return 0;
1370 }
1371 
cadence_nand_erase(struct nand_chip * chip,u32 page)1372 static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1373 {
1374 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1375 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1376 	int status;
1377 	u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1378 
1379 	cadence_nand_cdma_desc_prepare(cdns_ctrl,
1380 				       cdns_chip->cs[chip->cur_cs],
1381 				       page, 0, 0,
1382 				       CDMA_CT_ERASE);
1383 	status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1384 	if (status) {
1385 		dev_err(cdns_ctrl->dev, "erase operation failed\n");
1386 		return -EIO;
1387 	}
1388 
1389 	status = cadence_nand_cdma_finish(cdns_ctrl);
1390 	if (status)
1391 		return status;
1392 
1393 	return 0;
1394 }
1395 
cadence_nand_read_bbm(struct nand_chip * chip,int page,u8 * buf)1396 static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1397 {
1398 	int status;
1399 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1400 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1401 	struct mtd_info *mtd = nand_to_mtd(chip);
1402 
1403 	cadence_nand_prepare_data_size(chip, TT_BBM);
1404 
1405 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1406 
1407 	/*
1408 	 * Read only bad block marker from offset
1409 	 * defined by a memory manufacturer.
1410 	 */
1411 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1412 					    cdns_chip->cs[chip->cur_cs],
1413 					    page, cdns_ctrl->buf, NULL,
1414 					    mtd->oobsize,
1415 					    0, DMA_FROM_DEVICE, false);
1416 	if (status) {
1417 		dev_err(cdns_ctrl->dev, "read BBM failed\n");
1418 		return -EIO;
1419 	}
1420 
1421 	memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1422 
1423 	return 0;
1424 }
1425 
cadence_nand_write_page(struct nand_chip * chip,const u8 * buf,int oob_required,int page)1426 static int cadence_nand_write_page(struct nand_chip *chip,
1427 				   const u8 *buf, int oob_required,
1428 				   int page)
1429 {
1430 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1431 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1432 	struct mtd_info *mtd = nand_to_mtd(chip);
1433 	int status;
1434 	u16 marker_val = 0xFFFF;
1435 
1436 	status = cadence_nand_select_target(chip);
1437 	if (status)
1438 		return status;
1439 
1440 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1441 					 mtd->writesize
1442 					 + cdns_chip->bbm_offs,
1443 					 1);
1444 
1445 	if (oob_required) {
1446 		marker_val = *(u16 *)(chip->oob_poi
1447 				      + cdns_chip->bbm_offs);
1448 	} else {
1449 		/* Set oob data to 0xFF. */
1450 		memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1451 		       cdns_chip->avail_oob_size);
1452 	}
1453 
1454 	cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1455 
1456 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1457 
1458 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1459 	    cdns_ctrl->caps2.data_control_supp) {
1460 		u8 *oob;
1461 
1462 		if (oob_required)
1463 			oob = chip->oob_poi;
1464 		else
1465 			oob = cdns_ctrl->buf + mtd->writesize;
1466 
1467 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1468 						    cdns_chip->cs[chip->cur_cs],
1469 						    page, (void *)buf, oob,
1470 						    mtd->writesize,
1471 						    cdns_chip->avail_oob_size,
1472 						    DMA_TO_DEVICE, true);
1473 		if (status) {
1474 			dev_err(cdns_ctrl->dev, "write page failed\n");
1475 			return -EIO;
1476 		}
1477 
1478 		return 0;
1479 	}
1480 
1481 	if (oob_required) {
1482 		/* Transfer the data to the oob area. */
1483 		memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1484 		       cdns_chip->avail_oob_size);
1485 	}
1486 
1487 	memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1488 
1489 	cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1490 
1491 	return cadence_nand_cdma_transfer(cdns_ctrl,
1492 					  cdns_chip->cs[chip->cur_cs],
1493 					  page, cdns_ctrl->buf, NULL,
1494 					  mtd->writesize
1495 					  + cdns_chip->avail_oob_size,
1496 					  0, DMA_TO_DEVICE, true);
1497 }
1498 
cadence_nand_write_oob(struct nand_chip * chip,int page)1499 static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1500 {
1501 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1502 	struct mtd_info *mtd = nand_to_mtd(chip);
1503 
1504 	memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1505 
1506 	return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1507 }
1508 
cadence_nand_write_page_raw(struct nand_chip * chip,const u8 * buf,int oob_required,int page)1509 static int cadence_nand_write_page_raw(struct nand_chip *chip,
1510 				       const u8 *buf, int oob_required,
1511 				       int page)
1512 {
1513 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1514 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1515 	struct mtd_info *mtd = nand_to_mtd(chip);
1516 	int writesize = mtd->writesize;
1517 	int oobsize = mtd->oobsize;
1518 	int ecc_steps = chip->ecc.steps;
1519 	int ecc_size = chip->ecc.size;
1520 	int ecc_bytes = chip->ecc.bytes;
1521 	void *tmp_buf = cdns_ctrl->buf;
1522 	int oob_skip = cdns_chip->bbm_len;
1523 	size_t size = writesize + oobsize;
1524 	int i, pos, len;
1525 	int status = 0;
1526 
1527 	status = cadence_nand_select_target(chip);
1528 	if (status)
1529 		return status;
1530 
1531 	/*
1532 	 * Fill the buffer with 0xff first except the full page transfer.
1533 	 * This simplifies the logic.
1534 	 */
1535 	if (!buf || !oob_required)
1536 		memset(tmp_buf, 0xff, size);
1537 
1538 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1539 
1540 	/* Arrange the buffer for syndrome payload/ecc layout. */
1541 	if (buf) {
1542 		for (i = 0; i < ecc_steps; i++) {
1543 			pos = i * (ecc_size + ecc_bytes);
1544 			len = ecc_size;
1545 
1546 			if (pos >= writesize)
1547 				pos += oob_skip;
1548 			else if (pos + len > writesize)
1549 				len = writesize - pos;
1550 
1551 			memcpy(tmp_buf + pos, buf, len);
1552 			buf += len;
1553 			if (len < ecc_size) {
1554 				len = ecc_size - len;
1555 				memcpy(tmp_buf + writesize + oob_skip, buf,
1556 				       len);
1557 				buf += len;
1558 			}
1559 		}
1560 	}
1561 
1562 	if (oob_required) {
1563 		const u8 *oob = chip->oob_poi;
1564 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1565 			(cdns_chip->sector_size + chip->ecc.bytes)
1566 			+ cdns_chip->sector_size + oob_skip;
1567 
1568 		/* BBM at the beginning of the OOB area. */
1569 		memcpy(tmp_buf + writesize, oob, oob_skip);
1570 
1571 		/* OOB free. */
1572 		memcpy(tmp_buf + oob_data_offset, oob,
1573 		       cdns_chip->avail_oob_size);
1574 		oob += cdns_chip->avail_oob_size;
1575 
1576 		/* OOB ECC. */
1577 		for (i = 0; i < ecc_steps; i++) {
1578 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1579 			if (i == (ecc_steps - 1))
1580 				pos += cdns_chip->avail_oob_size;
1581 
1582 			len = ecc_bytes;
1583 
1584 			if (pos >= writesize)
1585 				pos += oob_skip;
1586 			else if (pos + len > writesize)
1587 				len = writesize - pos;
1588 
1589 			memcpy(tmp_buf + pos, oob, len);
1590 			oob += len;
1591 			if (len < ecc_bytes) {
1592 				len = ecc_bytes - len;
1593 				memcpy(tmp_buf + writesize + oob_skip, oob,
1594 				       len);
1595 				oob += len;
1596 			}
1597 		}
1598 	}
1599 
1600 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1601 
1602 	return cadence_nand_cdma_transfer(cdns_ctrl,
1603 					  cdns_chip->cs[chip->cur_cs],
1604 					  page, cdns_ctrl->buf, NULL,
1605 					  mtd->writesize +
1606 					  mtd->oobsize,
1607 					  0, DMA_TO_DEVICE, false);
1608 }
1609 
cadence_nand_write_oob_raw(struct nand_chip * chip,int page)1610 static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1611 				      int page)
1612 {
1613 	return cadence_nand_write_page_raw(chip, NULL, true, page);
1614 }
1615 
cadence_nand_read_page(struct nand_chip * chip,u8 * buf,int oob_required,int page)1616 static int cadence_nand_read_page(struct nand_chip *chip,
1617 				  u8 *buf, int oob_required, int page)
1618 {
1619 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1620 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1621 	struct mtd_info *mtd = nand_to_mtd(chip);
1622 	int status = 0;
1623 	int ecc_err_count = 0;
1624 
1625 	status = cadence_nand_select_target(chip);
1626 	if (status)
1627 		return status;
1628 
1629 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1630 					 mtd->writesize
1631 					 + cdns_chip->bbm_offs, 1);
1632 
1633 	/*
1634 	 * If data buffer can be accessed by DMA and data_control feature
1635 	 * is supported then transfer data and oob directly.
1636 	 */
1637 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1638 	    cdns_ctrl->caps2.data_control_supp) {
1639 		u8 *oob;
1640 
1641 		if (oob_required)
1642 			oob = chip->oob_poi;
1643 		else
1644 			oob = cdns_ctrl->buf + mtd->writesize;
1645 
1646 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1647 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1648 						    cdns_chip->cs[chip->cur_cs],
1649 						    page, buf, oob,
1650 						    mtd->writesize,
1651 						    cdns_chip->avail_oob_size,
1652 						    DMA_FROM_DEVICE, true);
1653 	/* Otherwise use bounce buffer. */
1654 	} else {
1655 		cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1656 		status = cadence_nand_cdma_transfer(cdns_ctrl,
1657 						    cdns_chip->cs[chip->cur_cs],
1658 						    page, cdns_ctrl->buf,
1659 						    NULL, mtd->writesize
1660 						    + cdns_chip->avail_oob_size,
1661 						    0, DMA_FROM_DEVICE, true);
1662 
1663 		memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1664 		if (oob_required)
1665 			memcpy(chip->oob_poi,
1666 			       cdns_ctrl->buf + mtd->writesize,
1667 			       mtd->oobsize);
1668 	}
1669 
1670 	switch (status) {
1671 	case STAT_ECC_UNCORR:
1672 		mtd->ecc_stats.failed++;
1673 		ecc_err_count++;
1674 		break;
1675 	case STAT_ECC_CORR:
1676 		ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1677 					  cdns_ctrl->cdma_desc->status);
1678 		mtd->ecc_stats.corrected += ecc_err_count;
1679 		break;
1680 	case STAT_ERASED:
1681 	case STAT_OK:
1682 		break;
1683 	default:
1684 		dev_err(cdns_ctrl->dev, "read page failed\n");
1685 		return -EIO;
1686 	}
1687 
1688 	if (oob_required)
1689 		if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1690 			return -EIO;
1691 
1692 	return ecc_err_count;
1693 }
1694 
1695 /* Reads OOB data from the device. */
cadence_nand_read_oob(struct nand_chip * chip,int page)1696 static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1697 {
1698 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1699 
1700 	return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1701 }
1702 
cadence_nand_read_page_raw(struct nand_chip * chip,u8 * buf,int oob_required,int page)1703 static int cadence_nand_read_page_raw(struct nand_chip *chip,
1704 				      u8 *buf, int oob_required, int page)
1705 {
1706 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1707 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1708 	struct mtd_info *mtd = nand_to_mtd(chip);
1709 	int oob_skip = cdns_chip->bbm_len;
1710 	int writesize = mtd->writesize;
1711 	int ecc_steps = chip->ecc.steps;
1712 	int ecc_size = chip->ecc.size;
1713 	int ecc_bytes = chip->ecc.bytes;
1714 	void *tmp_buf = cdns_ctrl->buf;
1715 	int i, pos, len;
1716 	int status = 0;
1717 
1718 	status = cadence_nand_select_target(chip);
1719 	if (status)
1720 		return status;
1721 
1722 	cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1723 
1724 	cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1725 	status = cadence_nand_cdma_transfer(cdns_ctrl,
1726 					    cdns_chip->cs[chip->cur_cs],
1727 					    page, cdns_ctrl->buf, NULL,
1728 					    mtd->writesize
1729 					    + mtd->oobsize,
1730 					    0, DMA_FROM_DEVICE, false);
1731 
1732 	switch (status) {
1733 	case STAT_ERASED:
1734 	case STAT_OK:
1735 		break;
1736 	default:
1737 		dev_err(cdns_ctrl->dev, "read raw page failed\n");
1738 		return -EIO;
1739 	}
1740 
1741 	/* Arrange the buffer for syndrome payload/ecc layout. */
1742 	if (buf) {
1743 		for (i = 0; i < ecc_steps; i++) {
1744 			pos = i * (ecc_size + ecc_bytes);
1745 			len = ecc_size;
1746 
1747 			if (pos >= writesize)
1748 				pos += oob_skip;
1749 			else if (pos + len > writesize)
1750 				len = writesize - pos;
1751 
1752 			memcpy(buf, tmp_buf + pos, len);
1753 			buf += len;
1754 			if (len < ecc_size) {
1755 				len = ecc_size - len;
1756 				memcpy(buf, tmp_buf + writesize + oob_skip,
1757 				       len);
1758 				buf += len;
1759 			}
1760 		}
1761 	}
1762 
1763 	if (oob_required) {
1764 		u8 *oob = chip->oob_poi;
1765 		u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1766 			(cdns_chip->sector_size + chip->ecc.bytes)
1767 			+ cdns_chip->sector_size + oob_skip;
1768 
1769 		/* OOB free. */
1770 		memcpy(oob, tmp_buf + oob_data_offset,
1771 		       cdns_chip->avail_oob_size);
1772 
1773 		/* BBM at the beginning of the OOB area. */
1774 		memcpy(oob, tmp_buf + writesize, oob_skip);
1775 
1776 		oob += cdns_chip->avail_oob_size;
1777 
1778 		/* OOB ECC */
1779 		for (i = 0; i < ecc_steps; i++) {
1780 			pos = ecc_size + i * (ecc_size + ecc_bytes);
1781 			len = ecc_bytes;
1782 
1783 			if (i == (ecc_steps - 1))
1784 				pos += cdns_chip->avail_oob_size;
1785 
1786 			if (pos >= writesize)
1787 				pos += oob_skip;
1788 			else if (pos + len > writesize)
1789 				len = writesize - pos;
1790 
1791 			memcpy(oob, tmp_buf + pos, len);
1792 			oob += len;
1793 			if (len < ecc_bytes) {
1794 				len = ecc_bytes - len;
1795 				memcpy(oob, tmp_buf + writesize + oob_skip,
1796 				       len);
1797 				oob += len;
1798 			}
1799 		}
1800 	}
1801 
1802 	return 0;
1803 }
1804 
cadence_nand_read_oob_raw(struct nand_chip * chip,int page)1805 static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1806 				     int page)
1807 {
1808 	return cadence_nand_read_page_raw(chip, NULL, true, page);
1809 }
1810 
cadence_nand_slave_dma_transfer_finished(void * data)1811 static void cadence_nand_slave_dma_transfer_finished(void *data)
1812 {
1813 	struct completion *finished = data;
1814 
1815 	complete(finished);
1816 }
1817 
cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl * cdns_ctrl,void * buf,dma_addr_t dev_dma,size_t len,enum dma_data_direction dir)1818 static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1819 					   void *buf,
1820 					   dma_addr_t dev_dma, size_t len,
1821 					   enum dma_data_direction dir)
1822 {
1823 	DECLARE_COMPLETION_ONSTACK(finished);
1824 	struct dma_chan *chan;
1825 	struct dma_device *dma_dev;
1826 	dma_addr_t src_dma, dst_dma, buf_dma;
1827 	struct dma_async_tx_descriptor *tx;
1828 	dma_cookie_t cookie;
1829 
1830 	chan = cdns_ctrl->dmac;
1831 	dma_dev = chan->device;
1832 
1833 	buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1834 	if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1835 		dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1836 		goto err;
1837 	}
1838 
1839 	if (dir == DMA_FROM_DEVICE) {
1840 		src_dma = cdns_ctrl->io.iova_dma;
1841 		dst_dma = buf_dma;
1842 	} else {
1843 		src_dma = buf_dma;
1844 		dst_dma = cdns_ctrl->io.iova_dma;
1845 	}
1846 
1847 	tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1848 				       DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1849 	if (!tx) {
1850 		dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1851 		goto err_unmap;
1852 	}
1853 
1854 	tx->callback = cadence_nand_slave_dma_transfer_finished;
1855 	tx->callback_param = &finished;
1856 
1857 	cookie = dmaengine_submit(tx);
1858 	if (dma_submit_error(cookie)) {
1859 		dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1860 		goto err_unmap;
1861 	}
1862 
1863 	dma_async_issue_pending(cdns_ctrl->dmac);
1864 	wait_for_completion(&finished);
1865 
1866 	dma_unmap_single(dma_dev->dev, buf_dma, len, dir);
1867 
1868 	return 0;
1869 
1870 err_unmap:
1871 	dma_unmap_single(dma_dev->dev, buf_dma, len, dir);
1872 
1873 err:
1874 	dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1875 
1876 	return -EIO;
1877 }
1878 
cadence_nand_read_buf(struct cdns_nand_ctrl * cdns_ctrl,u8 * buf,int len)1879 static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1880 				 u8 *buf, int len)
1881 {
1882 	u8 thread_nr = 0;
1883 	u32 sdma_size;
1884 	int status;
1885 
1886 	/* Wait until slave DMA interface is ready to data transfer. */
1887 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1888 	if (status)
1889 		return status;
1890 
1891 	if (!cdns_ctrl->caps1->has_dma) {
1892 		u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1893 
1894 		int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1895 
1896 		/* read alignment data */
1897 		if (data_dma_width == 4)
1898 			ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1899 #ifdef CONFIG_64BIT
1900 		else
1901 			readsq(cdns_ctrl->io.virt, buf, len_in_words);
1902 #endif
1903 
1904 		if (sdma_size > len) {
1905 			int read_bytes = (data_dma_width == 4) ?
1906 				len_in_words << 2 : len_in_words << 3;
1907 
1908 			/* read rest data from slave DMA interface if any */
1909 			if (data_dma_width == 4)
1910 				ioread32_rep(cdns_ctrl->io.virt,
1911 					     cdns_ctrl->buf,
1912 					     sdma_size / 4 - len_in_words);
1913 #ifdef CONFIG_64BIT
1914 			else
1915 				readsq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1916 				       sdma_size / 8 - len_in_words);
1917 #endif
1918 
1919 			/* copy rest of data */
1920 			memcpy(buf + read_bytes, cdns_ctrl->buf,
1921 			       len - read_bytes);
1922 		}
1923 		return 0;
1924 	}
1925 
1926 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1927 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1928 							 cdns_ctrl->io.dma,
1929 							 len, DMA_FROM_DEVICE);
1930 		if (status == 0)
1931 			return 0;
1932 
1933 		dev_warn(cdns_ctrl->dev,
1934 			 "Slave DMA transfer failed. Try again using bounce buffer.");
1935 	}
1936 
1937 	/* If DMA transfer is not possible or failed then use bounce buffer. */
1938 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1939 						 cdns_ctrl->io.dma,
1940 						 sdma_size, DMA_FROM_DEVICE);
1941 
1942 	if (status) {
1943 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1944 		return status;
1945 	}
1946 
1947 	memcpy(buf, cdns_ctrl->buf, len);
1948 
1949 	return 0;
1950 }
1951 
cadence_nand_write_buf(struct cdns_nand_ctrl * cdns_ctrl,const u8 * buf,int len)1952 static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1953 				  const u8 *buf, int len)
1954 {
1955 	u8 thread_nr = 0;
1956 	u32 sdma_size;
1957 	int status;
1958 
1959 	/* Wait until slave DMA interface is ready to data transfer. */
1960 	status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1961 	if (status)
1962 		return status;
1963 
1964 	if (!cdns_ctrl->caps1->has_dma) {
1965 		u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1966 
1967 		int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1968 
1969 		if (data_dma_width == 4)
1970 			iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1971 #ifdef CONFIG_64BIT
1972 		else
1973 			writesq(cdns_ctrl->io.virt, buf, len_in_words);
1974 #endif
1975 
1976 		if (sdma_size > len) {
1977 			int written_bytes = (data_dma_width == 4) ?
1978 				len_in_words << 2 : len_in_words << 3;
1979 
1980 			/* copy rest of data */
1981 			memcpy(cdns_ctrl->buf, buf + written_bytes,
1982 			       len - written_bytes);
1983 
1984 			/* write all expected by nand controller data */
1985 			if (data_dma_width == 4)
1986 				iowrite32_rep(cdns_ctrl->io.virt,
1987 					      cdns_ctrl->buf,
1988 					      sdma_size / 4 - len_in_words);
1989 #ifdef CONFIG_64BIT
1990 			else
1991 				writesq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1992 					sdma_size / 8 - len_in_words);
1993 #endif
1994 		}
1995 
1996 		return 0;
1997 	}
1998 
1999 	if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
2000 		status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
2001 							 cdns_ctrl->io.dma,
2002 							 len, DMA_TO_DEVICE);
2003 		if (status == 0)
2004 			return 0;
2005 
2006 		dev_warn(cdns_ctrl->dev,
2007 			 "Slave DMA transfer failed. Try again using bounce buffer.");
2008 	}
2009 
2010 	/* If DMA transfer is not possible or failed then use bounce buffer. */
2011 	memcpy(cdns_ctrl->buf, buf, len);
2012 
2013 	status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
2014 						 cdns_ctrl->io.dma,
2015 						 sdma_size, DMA_TO_DEVICE);
2016 
2017 	if (status)
2018 		dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
2019 
2020 	return status;
2021 }
2022 
cadence_nand_force_byte_access(struct nand_chip * chip,bool force_8bit)2023 static int cadence_nand_force_byte_access(struct nand_chip *chip,
2024 					  bool force_8bit)
2025 {
2026 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2027 
2028 	/*
2029 	 * Callers of this function do not verify if the NAND is using a 16-bit
2030 	 * an 8-bit bus for normal operations, so we need to take care of that
2031 	 * here by leaving the configuration unchanged if the NAND does not have
2032 	 * the NAND_BUSWIDTH_16 flag set.
2033 	 */
2034 	if (!(chip->options & NAND_BUSWIDTH_16))
2035 		return 0;
2036 
2037 	return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
2038 }
2039 
cadence_nand_cmd_opcode(struct nand_chip * chip,const struct nand_subop * subop)2040 static int cadence_nand_cmd_opcode(struct nand_chip *chip,
2041 				   const struct nand_subop *subop)
2042 {
2043 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2044 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2045 	const struct nand_op_instr *instr;
2046 	unsigned int op_id = 0;
2047 	u64 mini_ctrl_cmd = 0;
2048 	int ret;
2049 
2050 	instr = &subop->instrs[op_id];
2051 
2052 	if (instr->delay_ns > 0)
2053 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2054 
2055 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2056 				    GCMD_LAY_INSTR_CMD);
2057 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2058 				    instr->ctx.cmd.opcode);
2059 
2060 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2061 					    cdns_chip->cs[chip->cur_cs],
2062 					    mini_ctrl_cmd);
2063 	if (ret)
2064 		dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2065 			instr->ctx.cmd.opcode);
2066 
2067 	return ret;
2068 }
2069 
cadence_nand_cmd_address(struct nand_chip * chip,const struct nand_subop * subop)2070 static int cadence_nand_cmd_address(struct nand_chip *chip,
2071 				    const struct nand_subop *subop)
2072 {
2073 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2074 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2075 	const struct nand_op_instr *instr;
2076 	unsigned int op_id = 0;
2077 	u64 mini_ctrl_cmd = 0;
2078 	unsigned int offset, naddrs;
2079 	u64 address = 0;
2080 	const u8 *addrs;
2081 	int ret;
2082 	int i;
2083 
2084 	instr = &subop->instrs[op_id];
2085 
2086 	if (instr->delay_ns > 0)
2087 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2088 
2089 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2090 				    GCMD_LAY_INSTR_ADDR);
2091 
2092 	offset = nand_subop_get_addr_start_off(subop, op_id);
2093 	naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2094 	addrs = &instr->ctx.addr.addrs[offset];
2095 
2096 	for (i = 0; i < naddrs; i++)
2097 		address |= (u64)addrs[i] << (8 * i);
2098 
2099 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2100 				    address);
2101 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2102 				    naddrs - 1);
2103 
2104 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2105 					    cdns_chip->cs[chip->cur_cs],
2106 					    mini_ctrl_cmd);
2107 	if (ret)
2108 		dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2109 
2110 	return ret;
2111 }
2112 
cadence_nand_cmd_erase(struct nand_chip * chip,const struct nand_subop * subop)2113 static int cadence_nand_cmd_erase(struct nand_chip *chip,
2114 				  const struct nand_subop *subop)
2115 {
2116 	unsigned int op_id;
2117 
2118 	if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2119 		int i;
2120 		const struct nand_op_instr *instr = NULL;
2121 		unsigned int offset, naddrs;
2122 		const u8 *addrs;
2123 		u32 page = 0;
2124 
2125 		instr = &subop->instrs[1];
2126 		offset = nand_subop_get_addr_start_off(subop, 1);
2127 		naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2128 		addrs = &instr->ctx.addr.addrs[offset];
2129 
2130 		for (i = 0; i < naddrs; i++)
2131 			page |= (u32)addrs[i] << (8 * i);
2132 
2133 		return cadence_nand_erase(chip, page);
2134 	}
2135 
2136 	/*
2137 	 * If it is not an erase operation then handle operation
2138 	 * by calling exec_op function.
2139 	 */
2140 	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2141 		int ret;
2142 		const struct nand_operation nand_op = {
2143 			.cs = chip->cur_cs,
2144 			.instrs =  &subop->instrs[op_id],
2145 			.ninstrs = 1};
2146 		ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2147 		if (ret)
2148 			return ret;
2149 	}
2150 
2151 	return 0;
2152 }
2153 
cadence_nand_cmd_data(struct nand_chip * chip,const struct nand_subop * subop)2154 static int cadence_nand_cmd_data(struct nand_chip *chip,
2155 				 const struct nand_subop *subop)
2156 {
2157 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2158 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2159 	const struct nand_op_instr *instr;
2160 	unsigned int offset, op_id = 0;
2161 	u64 mini_ctrl_cmd = 0;
2162 	int len = 0;
2163 	int ret;
2164 
2165 	instr = &subop->instrs[op_id];
2166 
2167 	if (instr->delay_ns > 0)
2168 		mini_ctrl_cmd |= GCMD_LAY_TWB;
2169 
2170 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2171 				    GCMD_LAY_INSTR_DATA);
2172 
2173 	if (instr->type == NAND_OP_DATA_OUT_INSTR)
2174 		mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2175 					    GCMD_DIR_WRITE);
2176 
2177 	len = nand_subop_get_data_len(subop, op_id);
2178 	offset = nand_subop_get_data_start_off(subop, op_id);
2179 	mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2180 	mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2181 	if (instr->ctx.data.force_8bit) {
2182 		ret = cadence_nand_force_byte_access(chip, true);
2183 		if (ret) {
2184 			dev_err(cdns_ctrl->dev,
2185 				"cannot change byte access generic data cmd failed\n");
2186 			return ret;
2187 		}
2188 	}
2189 
2190 	ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2191 					    cdns_chip->cs[chip->cur_cs],
2192 					    mini_ctrl_cmd);
2193 	if (ret) {
2194 		dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2195 		return ret;
2196 	}
2197 
2198 	if (instr->type == NAND_OP_DATA_IN_INSTR) {
2199 		void *buf = instr->ctx.data.buf.in + offset;
2200 
2201 		ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2202 	} else {
2203 		const void *buf = instr->ctx.data.buf.out + offset;
2204 
2205 		ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2206 	}
2207 
2208 	if (ret) {
2209 		dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2210 		return ret;
2211 	}
2212 
2213 	if (instr->ctx.data.force_8bit) {
2214 		ret = cadence_nand_force_byte_access(chip, false);
2215 		if (ret) {
2216 			dev_err(cdns_ctrl->dev,
2217 				"cannot change byte access generic data cmd failed\n");
2218 		}
2219 	}
2220 
2221 	return ret;
2222 }
2223 
cadence_nand_cmd_waitrdy(struct nand_chip * chip,const struct nand_subop * subop)2224 static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2225 				    const struct nand_subop *subop)
2226 {
2227 	int status;
2228 	unsigned int op_id = 0;
2229 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2230 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2231 	const struct nand_op_instr *instr = &subop->instrs[op_id];
2232 	u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2233 
2234 	status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2235 					     timeout_us,
2236 					     BIT(cdns_chip->cs[chip->cur_cs]),
2237 					     false);
2238 	return status;
2239 }
2240 
2241 static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2242 	NAND_OP_PARSER_PATTERN(
2243 		cadence_nand_cmd_erase,
2244 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2245 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2246 		NAND_OP_PARSER_PAT_CMD_ELEM(false),
2247 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2248 	NAND_OP_PARSER_PATTERN(
2249 		cadence_nand_cmd_opcode,
2250 		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2251 	NAND_OP_PARSER_PATTERN(
2252 		cadence_nand_cmd_address,
2253 		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2254 	NAND_OP_PARSER_PATTERN(
2255 		cadence_nand_cmd_data,
2256 		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2257 	NAND_OP_PARSER_PATTERN(
2258 		cadence_nand_cmd_data,
2259 		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2260 	NAND_OP_PARSER_PATTERN(
2261 		cadence_nand_cmd_waitrdy,
2262 		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2263 	);
2264 
cadence_nand_exec_op(struct nand_chip * chip,const struct nand_operation * op,bool check_only)2265 static int cadence_nand_exec_op(struct nand_chip *chip,
2266 				const struct nand_operation *op,
2267 				bool check_only)
2268 {
2269 	if (!check_only) {
2270 		int status = cadence_nand_select_target(chip);
2271 
2272 		if (status)
2273 			return status;
2274 	}
2275 
2276 	return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2277 				      check_only);
2278 }
2279 
cadence_nand_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)2280 static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2281 				       struct mtd_oob_region *oobregion)
2282 {
2283 	struct nand_chip *chip = mtd_to_nand(mtd);
2284 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2285 
2286 	if (section)
2287 		return -ERANGE;
2288 
2289 	oobregion->offset = cdns_chip->bbm_len;
2290 	oobregion->length = cdns_chip->avail_oob_size
2291 		- cdns_chip->bbm_len;
2292 
2293 	return 0;
2294 }
2295 
cadence_nand_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)2296 static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2297 				      struct mtd_oob_region *oobregion)
2298 {
2299 	struct nand_chip *chip = mtd_to_nand(mtd);
2300 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2301 
2302 	if (section)
2303 		return -ERANGE;
2304 
2305 	oobregion->offset = cdns_chip->avail_oob_size;
2306 	oobregion->length = chip->ecc.total;
2307 
2308 	return 0;
2309 }
2310 
2311 static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2312 	.free = cadence_nand_ooblayout_free,
2313 	.ecc = cadence_nand_ooblayout_ecc,
2314 };
2315 
calc_cycl(u32 timing,u32 clock)2316 static int calc_cycl(u32 timing, u32 clock)
2317 {
2318 	if (timing == 0 || clock == 0)
2319 		return 0;
2320 
2321 	if ((timing % clock) > 0)
2322 		return timing / clock;
2323 	else
2324 		return timing / clock - 1;
2325 }
2326 
2327 /* Calculate max data valid window. */
calc_tdvw_max(u32 trp_cnt,u32 clk_period,u32 trhoh_min,u32 board_delay_skew_min,u32 ext_mode)2328 static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2329 				u32 board_delay_skew_min, u32 ext_mode)
2330 {
2331 	if (ext_mode == 0)
2332 		clk_period /= 2;
2333 
2334 	return (trp_cnt + 1) * clk_period + trhoh_min +
2335 		board_delay_skew_min;
2336 }
2337 
2338 /* Calculate data valid window. */
calc_tdvw(u32 trp_cnt,u32 clk_period,u32 trhoh_min,u32 trea_max,u32 ext_mode)2339 static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2340 			    u32 trea_max, u32 ext_mode)
2341 {
2342 	if (ext_mode == 0)
2343 		clk_period /= 2;
2344 
2345 	return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2346 }
2347 
2348 static int
cadence_nand_setup_interface(struct nand_chip * chip,int chipnr,const struct nand_interface_config * conf)2349 cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2350 			     const struct nand_interface_config *conf)
2351 {
2352 	const struct nand_sdr_timings *sdr;
2353 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2354 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2355 	struct cadence_nand_timings *t = &cdns_chip->timings;
2356 	u32 reg;
2357 	u32 board_delay = cdns_ctrl->board_delay;
2358 	u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2359 					    cdns_ctrl->nf_clk_rate);
2360 	u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2361 	u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2362 	u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2363 	u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2364 	u32 if_skew = cdns_ctrl->caps1->if_skew;
2365 	u32 board_delay_skew_min = board_delay - if_skew;
2366 	u32 board_delay_skew_max = board_delay + if_skew;
2367 	u32 dqs_sampl_res, phony_dqs_mod;
2368 	u32 tdvw, tdvw_min, tdvw_max;
2369 	u32 ext_rd_mode, ext_wr_mode;
2370 	u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2371 	u32 sampling_point;
2372 
2373 	sdr = nand_get_sdr_timings(conf);
2374 	if (IS_ERR(sdr))
2375 		return PTR_ERR(sdr);
2376 
2377 	memset(t, 0, sizeof(*t));
2378 	/* Sampling point calculation. */
2379 
2380 	if (cdns_ctrl->caps2.is_phy_type_dll)
2381 		phony_dqs_mod = 2;
2382 	else
2383 		phony_dqs_mod = 1;
2384 
2385 	dqs_sampl_res = clk_period / phony_dqs_mod;
2386 
2387 	tdvw_min = sdr->tREA_max + board_delay_skew_max;
2388 	/*
2389 	 * The idea of those calculation is to get the optimum value
2390 	 * for tRP and tRH timings. If it is NOT possible to sample data
2391 	 * with optimal tRP/tRH settings, the parameters will be extended.
2392 	 * If clk_period is 50ns (the lowest value) this condition is met
2393 	 * for SDR timing modes 1, 2, 3, 4 and 5.
2394 	 * If clk_period is 20ns the condition is met only for SDR timing
2395 	 * mode 5.
2396 	 */
2397 	if (sdr->tRC_min <= clk_period &&
2398 	    sdr->tRP_min <= (clk_period / 2) &&
2399 	    sdr->tREH_min <= (clk_period / 2)) {
2400 		/* Performance mode. */
2401 		ext_rd_mode = 0;
2402 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2403 				 sdr->tREA_max, ext_rd_mode);
2404 		tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2405 					 board_delay_skew_min,
2406 					 ext_rd_mode);
2407 		/*
2408 		 * Check if data valid window and sampling point can be found
2409 		 * and is not on the edge (ie. we have hold margin).
2410 		 * If not extend the tRP timings.
2411 		 */
2412 		if (tdvw > 0) {
2413 			if (tdvw_max <= tdvw_min ||
2414 			    (tdvw_max % dqs_sampl_res) == 0) {
2415 				/*
2416 				 * No valid sampling point so the RE pulse need
2417 				 * to be widen widening by half clock cycle.
2418 				 */
2419 				ext_rd_mode = 1;
2420 			}
2421 		} else {
2422 			/*
2423 			 * There is no valid window
2424 			 * to be able to sample data the tRP need to be widen.
2425 			 * Very safe calculations are performed here.
2426 			 */
2427 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2428 				   + dqs_sampl_res) / clk_period;
2429 			ext_rd_mode = 1;
2430 		}
2431 
2432 	} else {
2433 		/* Extended read mode. */
2434 		u32 trh;
2435 
2436 		ext_rd_mode = 1;
2437 		trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2438 		trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2439 		if (sdr->tREH_min >= trh)
2440 			trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2441 		else
2442 			trh_cnt = calc_cycl(trh, clk_period);
2443 
2444 		tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2445 				 sdr->tREA_max, ext_rd_mode);
2446 		/*
2447 		 * Check if data valid window and sampling point can be found
2448 		 * or if it is at the edge check if previous is valid
2449 		 * - if not extend the tRP timings.
2450 		 */
2451 		if (tdvw > 0) {
2452 			tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2453 						 sdr->tRHOH_min,
2454 						 board_delay_skew_min,
2455 						 ext_rd_mode);
2456 
2457 			if ((((tdvw_max / dqs_sampl_res)
2458 			      * dqs_sampl_res) <= tdvw_min) ||
2459 			    (((tdvw_max % dqs_sampl_res) == 0) &&
2460 			     (((tdvw_max / dqs_sampl_res - 1)
2461 			       * dqs_sampl_res) <= tdvw_min))) {
2462 				/*
2463 				 * Data valid window width is lower than
2464 				 * sampling resolution and do not hit any
2465 				 * sampling point to be sure the sampling point
2466 				 * will be found the RE low pulse width will be
2467 				 *  extended by one clock cycle.
2468 				 */
2469 				trp_cnt = trp_cnt + 1;
2470 			}
2471 		} else {
2472 			/*
2473 			 * There is no valid window to be able to sample data.
2474 			 * The tRP need to be widen.
2475 			 * Very safe calculations are performed here.
2476 			 */
2477 			trp_cnt = (sdr->tREA_max + board_delay_skew_max
2478 				   + dqs_sampl_res) / clk_period;
2479 		}
2480 	}
2481 
2482 	tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2483 				 sdr->tRHOH_min,
2484 				 board_delay_skew_min, ext_rd_mode);
2485 
2486 	if (sdr->tWC_min <= clk_period &&
2487 	    (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2488 	    (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2489 		ext_wr_mode = 0;
2490 	} else {
2491 		u32 twh;
2492 
2493 		ext_wr_mode = 1;
2494 		twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2495 		if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2496 			twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2497 					    clk_period);
2498 
2499 		twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2500 		if (sdr->tWH_min >= twh)
2501 			twh = sdr->tWH_min;
2502 
2503 		twh_cnt = calc_cycl(twh + if_skew, clk_period);
2504 	}
2505 
2506 	reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2507 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2508 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2509 	reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2510 	t->async_toggle_timings = reg;
2511 	dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2512 
2513 	tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2514 	tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2515 	twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2516 	trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2517 	reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2518 
2519 	/*
2520 	 * If timing exceeds delay field in timing register
2521 	 * then use maximum value.
2522 	 */
2523 	if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2524 		reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2525 	else
2526 		reg |= TIMINGS0_TCCS;
2527 
2528 	reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2529 	reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2530 	t->timings0 = reg;
2531 	dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2532 
2533 	/* The following is related to single signal so skew is not needed. */
2534 	trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2535 	trhz_cnt = trhz_cnt + 1;
2536 	twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2537 	/*
2538 	 * Because of the two stage syncflop the value must be increased by 3
2539 	 * first value is related with sync, second value is related
2540 	 * with output if delay.
2541 	 */
2542 	twb_cnt = twb_cnt + 3 + 5;
2543 	/*
2544 	 * The following is related to the we edge of the random data input
2545 	 * sequence so skew is not needed.
2546 	 */
2547 	tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2548 	reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2549 	reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2550 	reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2551 	t->timings1 = reg;
2552 	dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2553 
2554 	tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2555 	if (tfeat_cnt < twb_cnt)
2556 		tfeat_cnt = twb_cnt;
2557 
2558 	tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2559 	tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2560 
2561 	reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2562 	reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2563 	reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2564 	t->timings2 = reg;
2565 	dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2566 
2567 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2568 		reg = DLL_PHY_CTRL_DLL_RST_N;
2569 		if (ext_wr_mode)
2570 			reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2571 		if (ext_rd_mode)
2572 			reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2573 
2574 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2575 		reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2576 		t->dll_phy_ctrl = reg;
2577 		dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2578 	}
2579 
2580 	/* Sampling point calculation. */
2581 	if ((tdvw_max % dqs_sampl_res) > 0)
2582 		sampling_point = tdvw_max / dqs_sampl_res;
2583 	else
2584 		sampling_point = (tdvw_max / dqs_sampl_res - 1);
2585 
2586 	if (sampling_point * dqs_sampl_res > tdvw_min) {
2587 		dll_phy_dqs_timing =
2588 			FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2589 		dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2590 		phony_dqs_timing = sampling_point / phony_dqs_mod;
2591 
2592 		if ((sampling_point % 2) > 0) {
2593 			dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2594 			if ((tdvw_max % dqs_sampl_res) == 0)
2595 				/*
2596 				 * Calculation for sampling point at the edge
2597 				 * of data and being odd number.
2598 				 */
2599 				phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2600 					/ phony_dqs_mod - 1;
2601 
2602 			if (!cdns_ctrl->caps2.is_phy_type_dll)
2603 				phony_dqs_timing--;
2604 
2605 		} else {
2606 			phony_dqs_timing--;
2607 		}
2608 		rd_del_sel = phony_dqs_timing + 3;
2609 	} else {
2610 		dev_warn(cdns_ctrl->dev,
2611 			 "ERROR : cannot find valid sampling point\n");
2612 	}
2613 
2614 	reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2615 	if (cdns_ctrl->caps2.is_phy_type_dll)
2616 		reg  |= PHY_CTRL_SDR_DQS;
2617 	t->phy_ctrl = reg;
2618 	dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2619 
2620 	if (cdns_ctrl->caps2.is_phy_type_dll) {
2621 		dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2622 		dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2623 		dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2624 			dll_phy_dqs_timing);
2625 		t->phy_dqs_timing = dll_phy_dqs_timing;
2626 
2627 		reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2628 		dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2629 			reg);
2630 		t->phy_gate_lpbk_ctrl = reg;
2631 
2632 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2633 			PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2634 		dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2635 	}
2636 
2637 	return 0;
2638 }
2639 
cadence_nand_attach_chip(struct nand_chip * chip)2640 static int cadence_nand_attach_chip(struct nand_chip *chip)
2641 {
2642 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2643 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2644 	u32 ecc_size;
2645 	struct mtd_info *mtd = nand_to_mtd(chip);
2646 	int ret;
2647 
2648 	if (chip->options & NAND_BUSWIDTH_16) {
2649 		ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2650 		if (ret)
2651 			return ret;
2652 	}
2653 
2654 	chip->bbt_options |= NAND_BBT_USE_FLASH;
2655 	chip->bbt_options |= NAND_BBT_NO_OOB;
2656 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2657 
2658 	chip->options |= NAND_NO_SUBPAGE_WRITE;
2659 
2660 	cdns_chip->bbm_offs = chip->badblockpos;
2661 	cdns_chip->bbm_offs &= ~0x01;
2662 	/* this value should be even number */
2663 	cdns_chip->bbm_len = 2;
2664 
2665 	ret = nand_ecc_choose_conf(chip,
2666 				   &cdns_ctrl->ecc_caps,
2667 				   mtd->oobsize - cdns_chip->bbm_len);
2668 	if (ret) {
2669 		dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2670 		return ret;
2671 	}
2672 
2673 	dev_dbg(cdns_ctrl->dev,
2674 		"chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2675 		chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2676 
2677 	/* Error correction configuration. */
2678 	cdns_chip->sector_size = chip->ecc.size;
2679 	cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2680 	ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2681 
2682 	cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2683 
2684 	if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2685 		cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2686 
2687 	if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2688 	    > mtd->oobsize)
2689 		cdns_chip->avail_oob_size -= 4;
2690 
2691 	ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2692 	if (ret < 0)
2693 		return -EINVAL;
2694 
2695 	cdns_chip->corr_str_idx = (u8)ret;
2696 
2697 	if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2698 					1000000,
2699 					CTRL_STATUS_CTRL_BUSY, true))
2700 		return -ETIMEDOUT;
2701 
2702 	cadence_nand_set_ecc_strength(cdns_ctrl,
2703 				      cdns_chip->corr_str_idx);
2704 
2705 	cadence_nand_set_erase_detection(cdns_ctrl, true,
2706 					 chip->ecc.strength);
2707 
2708 	/* Override the default read operations. */
2709 	chip->ecc.read_page = cadence_nand_read_page;
2710 	chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2711 	chip->ecc.write_page = cadence_nand_write_page;
2712 	chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2713 	chip->ecc.read_oob = cadence_nand_read_oob;
2714 	chip->ecc.write_oob = cadence_nand_write_oob;
2715 	chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2716 	chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2717 
2718 	if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2719 		cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2720 
2721 	/* Is 32-bit DMA supported? */
2722 	ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2723 	if (ret) {
2724 		dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2725 		return ret;
2726 	}
2727 
2728 	mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2729 
2730 	return 0;
2731 }
2732 
2733 static const struct nand_controller_ops cadence_nand_controller_ops = {
2734 	.attach_chip = cadence_nand_attach_chip,
2735 	.exec_op = cadence_nand_exec_op,
2736 	.setup_interface = cadence_nand_setup_interface,
2737 };
2738 
cadence_nand_chip_init(struct cdns_nand_ctrl * cdns_ctrl,struct device_node * np)2739 static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2740 				  struct device_node *np)
2741 {
2742 	struct cdns_nand_chip *cdns_chip;
2743 	struct mtd_info *mtd;
2744 	struct nand_chip *chip;
2745 	int nsels, ret, i;
2746 	u32 cs;
2747 
2748 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2749 	if (nsels <= 0) {
2750 		dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2751 		return -EINVAL;
2752 	}
2753 
2754 	/* Allocate the nand chip structure. */
2755 	cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2756 				 (nsels * sizeof(u8)),
2757 				 GFP_KERNEL);
2758 	if (!cdns_chip) {
2759 		dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2760 		return -ENOMEM;
2761 	}
2762 
2763 	cdns_chip->nsels = nsels;
2764 
2765 	for (i = 0; i < nsels; i++) {
2766 		/* Retrieve CS id. */
2767 		ret = of_property_read_u32_index(np, "reg", i, &cs);
2768 		if (ret) {
2769 			dev_err(cdns_ctrl->dev,
2770 				"could not retrieve reg property: %d\n",
2771 				ret);
2772 			return ret;
2773 		}
2774 
2775 		if (cs >= cdns_ctrl->caps2.max_banks) {
2776 			dev_err(cdns_ctrl->dev,
2777 				"invalid reg value: %u (max CS = %d)\n",
2778 				cs, cdns_ctrl->caps2.max_banks);
2779 			return -EINVAL;
2780 		}
2781 
2782 		if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2783 			dev_err(cdns_ctrl->dev,
2784 				"CS %d already assigned\n", cs);
2785 			return -EINVAL;
2786 		}
2787 
2788 		cdns_chip->cs[i] = cs;
2789 	}
2790 
2791 	chip = &cdns_chip->chip;
2792 	chip->controller = &cdns_ctrl->controller;
2793 	nand_set_flash_node(chip, np);
2794 
2795 	mtd = nand_to_mtd(chip);
2796 	mtd->dev.parent = cdns_ctrl->dev;
2797 
2798 	/*
2799 	 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2800 	 * in the DT node, this entry will be overwritten in nand_scan_ident().
2801 	 */
2802 	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2803 
2804 	ret = nand_scan(chip, cdns_chip->nsels);
2805 	if (ret) {
2806 		dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2807 		return ret;
2808 	}
2809 
2810 	ret = mtd_device_register(mtd, NULL, 0);
2811 	if (ret) {
2812 		dev_err(cdns_ctrl->dev,
2813 			"failed to register mtd device: %d\n", ret);
2814 		nand_cleanup(chip);
2815 		return ret;
2816 	}
2817 
2818 	list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2819 
2820 	return 0;
2821 }
2822 
cadence_nand_chips_cleanup(struct cdns_nand_ctrl * cdns_ctrl)2823 static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2824 {
2825 	struct cdns_nand_chip *entry, *temp;
2826 	struct nand_chip *chip;
2827 	int ret;
2828 
2829 	list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2830 		chip = &entry->chip;
2831 		ret = mtd_device_unregister(nand_to_mtd(chip));
2832 		WARN_ON(ret);
2833 		nand_cleanup(chip);
2834 		list_del(&entry->node);
2835 	}
2836 }
2837 
cadence_nand_chips_init(struct cdns_nand_ctrl * cdns_ctrl)2838 static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2839 {
2840 	struct device_node *np = cdns_ctrl->dev->of_node;
2841 	int max_cs = cdns_ctrl->caps2.max_banks;
2842 	int nchips, ret;
2843 
2844 	nchips = of_get_child_count(np);
2845 
2846 	if (nchips > max_cs) {
2847 		dev_err(cdns_ctrl->dev,
2848 			"too many NAND chips: %d (max = %d CS)\n",
2849 			nchips, max_cs);
2850 		return -EINVAL;
2851 	}
2852 
2853 	for_each_child_of_node_scoped(np, nand_np) {
2854 		ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2855 		if (ret) {
2856 			cadence_nand_chips_cleanup(cdns_ctrl);
2857 			return ret;
2858 		}
2859 	}
2860 
2861 	return 0;
2862 }
2863 
2864 static void
cadence_nand_irq_cleanup(int irqnum,struct cdns_nand_ctrl * cdns_ctrl)2865 cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2866 {
2867 	/* Disable interrupts. */
2868 	writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2869 }
2870 
cadence_nand_init(struct cdns_nand_ctrl * cdns_ctrl)2871 static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2872 {
2873 	dma_cap_mask_t mask;
2874 	struct dma_device *dma_dev = cdns_ctrl->dmac->device;
2875 	int ret;
2876 
2877 	cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2878 						  sizeof(*cdns_ctrl->cdma_desc),
2879 						  &cdns_ctrl->dma_cdma_desc,
2880 						  GFP_KERNEL);
2881 	if (!cdns_ctrl->dma_cdma_desc)
2882 		return -ENOMEM;
2883 
2884 	cdns_ctrl->buf_size = SZ_16K;
2885 	cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2886 	if (!cdns_ctrl->buf) {
2887 		ret = -ENOMEM;
2888 		goto free_buf_desc;
2889 	}
2890 
2891 	if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2892 			     IRQF_SHARED, "cadence-nand-controller",
2893 			     cdns_ctrl)) {
2894 		dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2895 		ret = -ENODEV;
2896 		goto free_buf;
2897 	}
2898 
2899 	spin_lock_init(&cdns_ctrl->irq_lock);
2900 	init_completion(&cdns_ctrl->complete);
2901 
2902 	ret = cadence_nand_hw_init(cdns_ctrl);
2903 	if (ret)
2904 		goto disable_irq;
2905 
2906 	dma_cap_zero(mask);
2907 	dma_cap_set(DMA_MEMCPY, mask);
2908 
2909 	if (cdns_ctrl->caps1->has_dma) {
2910 		cdns_ctrl->dmac = dma_request_chan_by_mask(&mask);
2911 		if (IS_ERR(cdns_ctrl->dmac)) {
2912 			ret = dev_err_probe(cdns_ctrl->dev, PTR_ERR(cdns_ctrl->dmac),
2913 					    "%d: Failed to get a DMA channel\n", ret);
2914 			goto disable_irq;
2915 		}
2916 	}
2917 
2918 	cdns_ctrl->io.iova_dma = dma_map_resource(dma_dev->dev, cdns_ctrl->io.dma,
2919 						  cdns_ctrl->io.size,
2920 						  DMA_BIDIRECTIONAL, 0);
2921 
2922 	ret = dma_mapping_error(dma_dev->dev, cdns_ctrl->io.iova_dma);
2923 	if (ret) {
2924 		dev_err(cdns_ctrl->dev, "Failed to map I/O resource to DMA\n");
2925 		goto dma_release_chnl;
2926 	}
2927 
2928 	nand_controller_init(&cdns_ctrl->controller);
2929 	INIT_LIST_HEAD(&cdns_ctrl->chips);
2930 
2931 	cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2932 	cdns_ctrl->curr_corr_str_idx = 0xFF;
2933 
2934 	ret = cadence_nand_chips_init(cdns_ctrl);
2935 	if (ret) {
2936 		dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2937 			ret);
2938 		goto unmap_dma_resource;
2939 	}
2940 
2941 	kfree(cdns_ctrl->buf);
2942 	cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2943 	if (!cdns_ctrl->buf) {
2944 		ret = -ENOMEM;
2945 		goto unmap_dma_resource;
2946 	}
2947 
2948 	return 0;
2949 
2950 unmap_dma_resource:
2951 	dma_unmap_resource(dma_dev->dev, cdns_ctrl->io.iova_dma,
2952 			   cdns_ctrl->io.size, DMA_BIDIRECTIONAL, 0);
2953 
2954 dma_release_chnl:
2955 	if (cdns_ctrl->dmac)
2956 		dma_release_channel(cdns_ctrl->dmac);
2957 
2958 disable_irq:
2959 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2960 
2961 free_buf:
2962 	kfree(cdns_ctrl->buf);
2963 
2964 free_buf_desc:
2965 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2966 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2967 
2968 	return ret;
2969 }
2970 
2971 /* Driver exit point. */
cadence_nand_remove(struct cdns_nand_ctrl * cdns_ctrl)2972 static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2973 {
2974 	cadence_nand_chips_cleanup(cdns_ctrl);
2975 	if (cdns_ctrl->dmac)
2976 		dma_unmap_resource(cdns_ctrl->dmac->device->dev,
2977 				   cdns_ctrl->io.iova_dma, cdns_ctrl->io.size,
2978 				   DMA_BIDIRECTIONAL, 0);
2979 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2980 	kfree(cdns_ctrl->buf);
2981 	dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2982 			  cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2983 
2984 	if (cdns_ctrl->dmac)
2985 		dma_release_channel(cdns_ctrl->dmac);
2986 }
2987 
2988 struct cadence_nand_dt {
2989 	struct cdns_nand_ctrl cdns_ctrl;
2990 	struct clk *clk;
2991 };
2992 
2993 static const struct cadence_nand_dt_devdata cadence_nand_default = {
2994 	.if_skew = 0,
2995 	.has_dma = 1,
2996 };
2997 
2998 static const struct of_device_id cadence_nand_dt_ids[] = {
2999 	{
3000 		.compatible = "cdns,hp-nfc",
3001 		.data = &cadence_nand_default
3002 	}, {}
3003 };
3004 
3005 MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
3006 
cadence_nand_dt_probe(struct platform_device * ofdev)3007 static int cadence_nand_dt_probe(struct platform_device *ofdev)
3008 {
3009 	struct resource *res;
3010 	struct cadence_nand_dt *dt;
3011 	struct cdns_nand_ctrl *cdns_ctrl;
3012 	int ret;
3013 	const struct cadence_nand_dt_devdata *devdata;
3014 	u32 val;
3015 
3016 	devdata = device_get_match_data(&ofdev->dev);
3017 	if (!devdata) {
3018 		pr_err("Failed to find the right device id.\n");
3019 		return -ENOMEM;
3020 	}
3021 
3022 	dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
3023 	if (!dt)
3024 		return -ENOMEM;
3025 
3026 	cdns_ctrl = &dt->cdns_ctrl;
3027 	cdns_ctrl->caps1 = devdata;
3028 
3029 	cdns_ctrl->dev = &ofdev->dev;
3030 	cdns_ctrl->irq = platform_get_irq(ofdev, 0);
3031 	if (cdns_ctrl->irq < 0)
3032 		return cdns_ctrl->irq;
3033 
3034 	dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
3035 
3036 	cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
3037 	if (IS_ERR(cdns_ctrl->reg))
3038 		return PTR_ERR(cdns_ctrl->reg);
3039 
3040 	cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
3041 	if (IS_ERR(cdns_ctrl->io.virt))
3042 		return PTR_ERR(cdns_ctrl->io.virt);
3043 
3044 	cdns_ctrl->io.dma = res->start;
3045 	cdns_ctrl->io.size = resource_size(res);
3046 
3047 	dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
3048 	if (IS_ERR(dt->clk))
3049 		return PTR_ERR(dt->clk);
3050 
3051 	cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
3052 
3053 	ret = of_property_read_u32(ofdev->dev.of_node,
3054 				   "cdns,board-delay-ps", &val);
3055 	if (ret) {
3056 		val = 4830;
3057 		dev_info(cdns_ctrl->dev,
3058 			 "missing cdns,board-delay-ps property, %d was set\n",
3059 			 val);
3060 	}
3061 	cdns_ctrl->board_delay = val;
3062 
3063 	ret = cadence_nand_init(cdns_ctrl);
3064 	if (ret)
3065 		return ret;
3066 
3067 	platform_set_drvdata(ofdev, dt);
3068 	return 0;
3069 }
3070 
cadence_nand_dt_remove(struct platform_device * ofdev)3071 static void cadence_nand_dt_remove(struct platform_device *ofdev)
3072 {
3073 	struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3074 
3075 	cadence_nand_remove(&dt->cdns_ctrl);
3076 }
3077 
3078 static struct platform_driver cadence_nand_dt_driver = {
3079 	.probe		= cadence_nand_dt_probe,
3080 	.remove		= cadence_nand_dt_remove,
3081 	.driver		= {
3082 		.name	= "cadence-nand-controller",
3083 		.of_match_table = cadence_nand_dt_ids,
3084 	},
3085 };
3086 
3087 module_platform_driver(cadence_nand_dt_driver);
3088 
3089 MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3090 MODULE_LICENSE("GPL v2");
3091 MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3092 
3093