xref: /linux/drivers/spi/spi-cadence-quadspi.c (revision f858cc9eed5b05cbe38d7ffd2787c21e3718eb7d)
1 // SPDX-License-Identifier: GPL-2.0-only
2 //
3 // Driver for Cadence QSPI Controller
4 //
5 // Copyright Altera Corporation (C) 2012-2014. All rights reserved.
6 // Copyright Intel Corporation (C) 2019-2020. All rights reserved.
7 // Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com
8 
9 #include <linux/clk.h>
10 #include <linux/completion.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/err.h>
15 #include <linux/errno.h>
16 #include <linux/firmware/xlnx-zynqmp.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/iopoll.h>
20 #include <linux/jiffies.h>
21 #include <linux/kernel.h>
22 #include <linux/log2.h>
23 #include <linux/module.h>
24 #include <linux/of.h>
25 #include <linux/platform_device.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/reset.h>
28 #include <linux/sched.h>
29 #include <linux/spi/spi.h>
30 #include <linux/spi/spi-mem.h>
31 #include <linux/timer.h>
32 
33 #define CQSPI_NAME			"cadence-qspi"
34 #define CQSPI_MAX_CHIPSELECT		4
35 
36 static_assert(CQSPI_MAX_CHIPSELECT <= SPI_CS_CNT_MAX);
37 
38 /* Quirks */
39 #define CQSPI_NEEDS_WR_DELAY		BIT(0)
40 #define CQSPI_DISABLE_DAC_MODE		BIT(1)
41 #define CQSPI_SUPPORT_EXTERNAL_DMA	BIT(2)
42 #define CQSPI_NO_SUPPORT_WR_COMPLETION	BIT(3)
43 #define CQSPI_SLOW_SRAM		BIT(4)
44 #define CQSPI_NEEDS_APB_AHB_HAZARD_WAR	BIT(5)
45 #define CQSPI_RD_NO_IRQ			BIT(6)
46 
47 /* Capabilities */
48 #define CQSPI_SUPPORTS_OCTAL		BIT(0)
49 
50 #define CQSPI_OP_WIDTH(part) ((part).nbytes ? ilog2((part).buswidth) : 0)
51 
52 enum {
53 	CLK_QSPI_APB = 0,
54 	CLK_QSPI_AHB,
55 	CLK_QSPI_NUM,
56 };
57 
58 struct cqspi_st;
59 
60 struct cqspi_flash_pdata {
61 	struct cqspi_st	*cqspi;
62 	u32		clk_rate;
63 	u32		read_delay;
64 	u32		tshsl_ns;
65 	u32		tsd2d_ns;
66 	u32		tchsh_ns;
67 	u32		tslch_ns;
68 	u8		cs;
69 };
70 
71 struct cqspi_st {
72 	struct platform_device	*pdev;
73 	struct spi_controller	*host;
74 	struct clk		*clk;
75 	struct clk		*clks[CLK_QSPI_NUM];
76 	unsigned int		sclk;
77 
78 	void __iomem		*iobase;
79 	void __iomem		*ahb_base;
80 	resource_size_t		ahb_size;
81 	struct completion	transfer_complete;
82 
83 	struct dma_chan		*rx_chan;
84 	struct completion	rx_dma_complete;
85 	dma_addr_t		mmap_phys_base;
86 
87 	int			current_cs;
88 	unsigned long		master_ref_clk_hz;
89 	bool			is_decoded_cs;
90 	u32			fifo_depth;
91 	u32			fifo_width;
92 	u32			num_chipselect;
93 	bool			rclk_en;
94 	u32			trigger_address;
95 	u32			wr_delay;
96 	bool			use_direct_mode;
97 	bool			use_direct_mode_wr;
98 	struct cqspi_flash_pdata f_pdata[CQSPI_MAX_CHIPSELECT];
99 	bool			use_dma_read;
100 	u32			pd_dev_id;
101 	bool			wr_completion;
102 	bool			slow_sram;
103 	bool			apb_ahb_hazard;
104 
105 	bool			is_jh7110; /* Flag for StarFive JH7110 SoC */
106 
107 	const struct cqspi_driver_platdata *ddata;
108 };
109 
110 struct cqspi_driver_platdata {
111 	u32 hwcaps_mask;
112 	u8 quirks;
113 	int (*indirect_read_dma)(struct cqspi_flash_pdata *f_pdata,
114 				 u_char *rxbuf, loff_t from_addr, size_t n_rx);
115 	u32 (*get_dma_status)(struct cqspi_st *cqspi);
116 	int (*jh7110_clk_init)(struct platform_device *pdev,
117 			       struct cqspi_st *cqspi);
118 };
119 
120 /* Operation timeout value */
121 #define CQSPI_TIMEOUT_MS			500
122 #define CQSPI_READ_TIMEOUT_MS			10
123 #define CQSPI_BUSYWAIT_TIMEOUT_US		500
124 
125 /* Runtime_pm autosuspend delay */
126 #define CQSPI_AUTOSUSPEND_TIMEOUT		2000
127 
128 #define CQSPI_DUMMY_CLKS_PER_BYTE		8
129 #define CQSPI_DUMMY_BYTES_MAX			4
130 #define CQSPI_DUMMY_CLKS_MAX			31
131 
132 #define CQSPI_STIG_DATA_LEN_MAX			8
133 
134 /* Register map */
135 #define CQSPI_REG_CONFIG			0x00
136 #define CQSPI_REG_CONFIG_ENABLE_MASK		BIT(0)
137 #define CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL	BIT(7)
138 #define CQSPI_REG_CONFIG_DECODE_MASK		BIT(9)
139 #define CQSPI_REG_CONFIG_CHIPSELECT_LSB		10
140 #define CQSPI_REG_CONFIG_DMA_MASK		BIT(15)
141 #define CQSPI_REG_CONFIG_BAUD_LSB		19
142 #define CQSPI_REG_CONFIG_DTR_PROTO		BIT(24)
143 #define CQSPI_REG_CONFIG_DUAL_OPCODE		BIT(30)
144 #define CQSPI_REG_CONFIG_IDLE_LSB		31
145 #define CQSPI_REG_CONFIG_CHIPSELECT_MASK	0xF
146 #define CQSPI_REG_CONFIG_BAUD_MASK		0xF
147 
148 #define CQSPI_REG_RD_INSTR			0x04
149 #define CQSPI_REG_RD_INSTR_OPCODE_LSB		0
150 #define CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB	8
151 #define CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB	12
152 #define CQSPI_REG_RD_INSTR_TYPE_DATA_LSB	16
153 #define CQSPI_REG_RD_INSTR_MODE_EN_LSB		20
154 #define CQSPI_REG_RD_INSTR_DUMMY_LSB		24
155 #define CQSPI_REG_RD_INSTR_TYPE_INSTR_MASK	0x3
156 #define CQSPI_REG_RD_INSTR_TYPE_ADDR_MASK	0x3
157 #define CQSPI_REG_RD_INSTR_TYPE_DATA_MASK	0x3
158 #define CQSPI_REG_RD_INSTR_DUMMY_MASK		0x1F
159 
160 #define CQSPI_REG_WR_INSTR			0x08
161 #define CQSPI_REG_WR_INSTR_OPCODE_LSB		0
162 #define CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB	12
163 #define CQSPI_REG_WR_INSTR_TYPE_DATA_LSB	16
164 
165 #define CQSPI_REG_DELAY				0x0C
166 #define CQSPI_REG_DELAY_TSLCH_LSB		0
167 #define CQSPI_REG_DELAY_TCHSH_LSB		8
168 #define CQSPI_REG_DELAY_TSD2D_LSB		16
169 #define CQSPI_REG_DELAY_TSHSL_LSB		24
170 #define CQSPI_REG_DELAY_TSLCH_MASK		0xFF
171 #define CQSPI_REG_DELAY_TCHSH_MASK		0xFF
172 #define CQSPI_REG_DELAY_TSD2D_MASK		0xFF
173 #define CQSPI_REG_DELAY_TSHSL_MASK		0xFF
174 
175 #define CQSPI_REG_READCAPTURE			0x10
176 #define CQSPI_REG_READCAPTURE_BYPASS_LSB	0
177 #define CQSPI_REG_READCAPTURE_DELAY_LSB		1
178 #define CQSPI_REG_READCAPTURE_DELAY_MASK	0xF
179 
180 #define CQSPI_REG_SIZE				0x14
181 #define CQSPI_REG_SIZE_ADDRESS_LSB		0
182 #define CQSPI_REG_SIZE_PAGE_LSB			4
183 #define CQSPI_REG_SIZE_BLOCK_LSB		16
184 #define CQSPI_REG_SIZE_ADDRESS_MASK		0xF
185 #define CQSPI_REG_SIZE_PAGE_MASK		0xFFF
186 #define CQSPI_REG_SIZE_BLOCK_MASK		0x3F
187 
188 #define CQSPI_REG_SRAMPARTITION			0x18
189 #define CQSPI_REG_INDIRECTTRIGGER		0x1C
190 
191 #define CQSPI_REG_DMA				0x20
192 #define CQSPI_REG_DMA_SINGLE_LSB		0
193 #define CQSPI_REG_DMA_BURST_LSB			8
194 #define CQSPI_REG_DMA_SINGLE_MASK		0xFF
195 #define CQSPI_REG_DMA_BURST_MASK		0xFF
196 
197 #define CQSPI_REG_REMAP				0x24
198 #define CQSPI_REG_MODE_BIT			0x28
199 
200 #define CQSPI_REG_SDRAMLEVEL			0x2C
201 #define CQSPI_REG_SDRAMLEVEL_RD_LSB		0
202 #define CQSPI_REG_SDRAMLEVEL_WR_LSB		16
203 #define CQSPI_REG_SDRAMLEVEL_RD_MASK		0xFFFF
204 #define CQSPI_REG_SDRAMLEVEL_WR_MASK		0xFFFF
205 
206 #define CQSPI_REG_WR_COMPLETION_CTRL		0x38
207 #define CQSPI_REG_WR_DISABLE_AUTO_POLL		BIT(14)
208 
209 #define CQSPI_REG_IRQSTATUS			0x40
210 #define CQSPI_REG_IRQMASK			0x44
211 
212 #define CQSPI_REG_INDIRECTRD			0x60
213 #define CQSPI_REG_INDIRECTRD_START_MASK		BIT(0)
214 #define CQSPI_REG_INDIRECTRD_CANCEL_MASK	BIT(1)
215 #define CQSPI_REG_INDIRECTRD_DONE_MASK		BIT(5)
216 
217 #define CQSPI_REG_INDIRECTRDWATERMARK		0x64
218 #define CQSPI_REG_INDIRECTRDSTARTADDR		0x68
219 #define CQSPI_REG_INDIRECTRDBYTES		0x6C
220 
221 #define CQSPI_REG_CMDCTRL			0x90
222 #define CQSPI_REG_CMDCTRL_EXECUTE_MASK		BIT(0)
223 #define CQSPI_REG_CMDCTRL_INPROGRESS_MASK	BIT(1)
224 #define CQSPI_REG_CMDCTRL_DUMMY_LSB		7
225 #define CQSPI_REG_CMDCTRL_WR_BYTES_LSB		12
226 #define CQSPI_REG_CMDCTRL_WR_EN_LSB		15
227 #define CQSPI_REG_CMDCTRL_ADD_BYTES_LSB		16
228 #define CQSPI_REG_CMDCTRL_ADDR_EN_LSB		19
229 #define CQSPI_REG_CMDCTRL_RD_BYTES_LSB		20
230 #define CQSPI_REG_CMDCTRL_RD_EN_LSB		23
231 #define CQSPI_REG_CMDCTRL_OPCODE_LSB		24
232 #define CQSPI_REG_CMDCTRL_WR_BYTES_MASK		0x7
233 #define CQSPI_REG_CMDCTRL_ADD_BYTES_MASK	0x3
234 #define CQSPI_REG_CMDCTRL_RD_BYTES_MASK		0x7
235 #define CQSPI_REG_CMDCTRL_DUMMY_MASK		0x1F
236 
237 #define CQSPI_REG_INDIRECTWR			0x70
238 #define CQSPI_REG_INDIRECTWR_START_MASK		BIT(0)
239 #define CQSPI_REG_INDIRECTWR_CANCEL_MASK	BIT(1)
240 #define CQSPI_REG_INDIRECTWR_DONE_MASK		BIT(5)
241 
242 #define CQSPI_REG_INDIRECTWRWATERMARK		0x74
243 #define CQSPI_REG_INDIRECTWRSTARTADDR		0x78
244 #define CQSPI_REG_INDIRECTWRBYTES		0x7C
245 
246 #define CQSPI_REG_INDTRIG_ADDRRANGE		0x80
247 
248 #define CQSPI_REG_CMDADDRESS			0x94
249 #define CQSPI_REG_CMDREADDATALOWER		0xA0
250 #define CQSPI_REG_CMDREADDATAUPPER		0xA4
251 #define CQSPI_REG_CMDWRITEDATALOWER		0xA8
252 #define CQSPI_REG_CMDWRITEDATAUPPER		0xAC
253 
254 #define CQSPI_REG_POLLING_STATUS		0xB0
255 #define CQSPI_REG_POLLING_STATUS_DUMMY_LSB	16
256 
257 #define CQSPI_REG_OP_EXT_LOWER			0xE0
258 #define CQSPI_REG_OP_EXT_READ_LSB		24
259 #define CQSPI_REG_OP_EXT_WRITE_LSB		16
260 #define CQSPI_REG_OP_EXT_STIG_LSB		0
261 
262 #define CQSPI_REG_VERSAL_DMA_SRC_ADDR		0x1000
263 
264 #define CQSPI_REG_VERSAL_DMA_DST_ADDR		0x1800
265 #define CQSPI_REG_VERSAL_DMA_DST_SIZE		0x1804
266 
267 #define CQSPI_REG_VERSAL_DMA_DST_CTRL		0x180C
268 
269 #define CQSPI_REG_VERSAL_DMA_DST_I_STS		0x1814
270 #define CQSPI_REG_VERSAL_DMA_DST_I_EN		0x1818
271 #define CQSPI_REG_VERSAL_DMA_DST_I_DIS		0x181C
272 #define CQSPI_REG_VERSAL_DMA_DST_DONE_MASK	BIT(1)
273 
274 #define CQSPI_REG_VERSAL_DMA_DST_ADDR_MSB	0x1828
275 
276 #define CQSPI_REG_VERSAL_DMA_DST_CTRL_VAL	0xF43FFA00
277 #define CQSPI_REG_VERSAL_ADDRRANGE_WIDTH_VAL	0x6
278 
279 /* Interrupt status bits */
280 #define CQSPI_REG_IRQ_MODE_ERR			BIT(0)
281 #define CQSPI_REG_IRQ_UNDERFLOW			BIT(1)
282 #define CQSPI_REG_IRQ_IND_COMP			BIT(2)
283 #define CQSPI_REG_IRQ_IND_RD_REJECT		BIT(3)
284 #define CQSPI_REG_IRQ_WR_PROTECTED_ERR		BIT(4)
285 #define CQSPI_REG_IRQ_ILLEGAL_AHB_ERR		BIT(5)
286 #define CQSPI_REG_IRQ_WATERMARK			BIT(6)
287 #define CQSPI_REG_IRQ_IND_SRAM_FULL		BIT(12)
288 
289 #define CQSPI_IRQ_MASK_RD		(CQSPI_REG_IRQ_WATERMARK	| \
290 					 CQSPI_REG_IRQ_IND_SRAM_FULL	| \
291 					 CQSPI_REG_IRQ_IND_COMP)
292 
293 #define CQSPI_IRQ_MASK_WR		(CQSPI_REG_IRQ_IND_COMP		| \
294 					 CQSPI_REG_IRQ_WATERMARK	| \
295 					 CQSPI_REG_IRQ_UNDERFLOW)
296 
297 #define CQSPI_IRQ_STATUS_MASK		0x1FFFF
298 #define CQSPI_DMA_UNALIGN		0x3
299 
300 #define CQSPI_REG_VERSAL_DMA_VAL		0x602
301 
302 static int cqspi_wait_for_bit(const struct cqspi_driver_platdata *ddata,
303 			      void __iomem *reg, const u32 mask, bool clr,
304 			      bool busywait)
305 {
306 	u64 timeout_us = CQSPI_TIMEOUT_MS * USEC_PER_MSEC;
307 	u32 val;
308 
309 	if (busywait) {
310 		int ret = readl_relaxed_poll_timeout(reg, val,
311 						     (((clr ? ~val : val) & mask) == mask),
312 						     0, CQSPI_BUSYWAIT_TIMEOUT_US);
313 
314 		if (ret != -ETIMEDOUT)
315 			return ret;
316 
317 		timeout_us -= CQSPI_BUSYWAIT_TIMEOUT_US;
318 	}
319 
320 	return readl_relaxed_poll_timeout(reg, val,
321 					  (((clr ? ~val : val) & mask) == mask),
322 					  10, timeout_us);
323 }
324 
325 static bool cqspi_is_idle(struct cqspi_st *cqspi)
326 {
327 	u32 reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
328 
329 	return reg & (1UL << CQSPI_REG_CONFIG_IDLE_LSB);
330 }
331 
332 static u32 cqspi_get_rd_sram_level(struct cqspi_st *cqspi)
333 {
334 	u32 reg = readl(cqspi->iobase + CQSPI_REG_SDRAMLEVEL);
335 
336 	reg >>= CQSPI_REG_SDRAMLEVEL_RD_LSB;
337 	return reg & CQSPI_REG_SDRAMLEVEL_RD_MASK;
338 }
339 
340 static u32 cqspi_get_versal_dma_status(struct cqspi_st *cqspi)
341 {
342 	u32 dma_status;
343 
344 	dma_status = readl(cqspi->iobase +
345 					   CQSPI_REG_VERSAL_DMA_DST_I_STS);
346 	writel(dma_status, cqspi->iobase +
347 		   CQSPI_REG_VERSAL_DMA_DST_I_STS);
348 
349 	return dma_status & CQSPI_REG_VERSAL_DMA_DST_DONE_MASK;
350 }
351 
352 static irqreturn_t cqspi_irq_handler(int this_irq, void *dev)
353 {
354 	struct cqspi_st *cqspi = dev;
355 	const struct cqspi_driver_platdata *ddata = cqspi->ddata;
356 	unsigned int irq_status;
357 
358 	/* Read interrupt status */
359 	irq_status = readl(cqspi->iobase + CQSPI_REG_IRQSTATUS);
360 
361 	/* Clear interrupt */
362 	writel(irq_status, cqspi->iobase + CQSPI_REG_IRQSTATUS);
363 
364 	if (cqspi->use_dma_read && ddata && ddata->get_dma_status) {
365 		if (ddata->get_dma_status(cqspi)) {
366 			complete(&cqspi->transfer_complete);
367 			return IRQ_HANDLED;
368 		}
369 	}
370 
371 	else if (!cqspi->slow_sram)
372 		irq_status &= CQSPI_IRQ_MASK_RD | CQSPI_IRQ_MASK_WR;
373 	else
374 		irq_status &= CQSPI_REG_IRQ_WATERMARK | CQSPI_IRQ_MASK_WR;
375 
376 	if (irq_status)
377 		complete(&cqspi->transfer_complete);
378 
379 	return IRQ_HANDLED;
380 }
381 
382 static unsigned int cqspi_calc_rdreg(const struct spi_mem_op *op)
383 {
384 	u32 rdreg = 0;
385 
386 	rdreg |= CQSPI_OP_WIDTH(op->cmd) << CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB;
387 	rdreg |= CQSPI_OP_WIDTH(op->addr) << CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB;
388 	rdreg |= CQSPI_OP_WIDTH(op->data) << CQSPI_REG_RD_INSTR_TYPE_DATA_LSB;
389 
390 	return rdreg;
391 }
392 
393 static unsigned int cqspi_calc_dummy(const struct spi_mem_op *op)
394 {
395 	unsigned int dummy_clk;
396 
397 	if (!op->dummy.nbytes)
398 		return 0;
399 
400 	dummy_clk = op->dummy.nbytes * (8 / op->dummy.buswidth);
401 	if (op->cmd.dtr)
402 		dummy_clk /= 2;
403 
404 	return dummy_clk;
405 }
406 
407 static int cqspi_wait_idle(struct cqspi_st *cqspi)
408 {
409 	const unsigned int poll_idle_retry = 3;
410 	unsigned int count = 0;
411 	unsigned long timeout;
412 
413 	timeout = jiffies + msecs_to_jiffies(CQSPI_TIMEOUT_MS);
414 	while (1) {
415 		/*
416 		 * Read few times in succession to ensure the controller
417 		 * is indeed idle, that is, the bit does not transition
418 		 * low again.
419 		 */
420 		if (cqspi_is_idle(cqspi))
421 			count++;
422 		else
423 			count = 0;
424 
425 		if (count >= poll_idle_retry)
426 			return 0;
427 
428 		if (time_after(jiffies, timeout)) {
429 			/* Timeout, in busy mode. */
430 			dev_err(&cqspi->pdev->dev,
431 				"QSPI is still busy after %dms timeout.\n",
432 				CQSPI_TIMEOUT_MS);
433 			return -ETIMEDOUT;
434 		}
435 
436 		cpu_relax();
437 	}
438 }
439 
440 static int cqspi_exec_flash_cmd(struct cqspi_st *cqspi, unsigned int reg)
441 {
442 	void __iomem *reg_base = cqspi->iobase;
443 	int ret;
444 
445 	/* Write the CMDCTRL without start execution. */
446 	writel(reg, reg_base + CQSPI_REG_CMDCTRL);
447 	/* Start execute */
448 	reg |= CQSPI_REG_CMDCTRL_EXECUTE_MASK;
449 	writel(reg, reg_base + CQSPI_REG_CMDCTRL);
450 
451 	/* Polling for completion. */
452 	ret = cqspi_wait_for_bit(cqspi->ddata, reg_base + CQSPI_REG_CMDCTRL,
453 				 CQSPI_REG_CMDCTRL_INPROGRESS_MASK, 1, true);
454 	if (ret) {
455 		dev_err(&cqspi->pdev->dev,
456 			"Flash command execution timed out.\n");
457 		return ret;
458 	}
459 
460 	/* Polling QSPI idle status. */
461 	return cqspi_wait_idle(cqspi);
462 }
463 
464 static int cqspi_setup_opcode_ext(struct cqspi_flash_pdata *f_pdata,
465 				  const struct spi_mem_op *op,
466 				  unsigned int shift)
467 {
468 	struct cqspi_st *cqspi = f_pdata->cqspi;
469 	void __iomem *reg_base = cqspi->iobase;
470 	unsigned int reg;
471 	u8 ext;
472 
473 	if (op->cmd.nbytes != 2)
474 		return -EINVAL;
475 
476 	/* Opcode extension is the LSB. */
477 	ext = op->cmd.opcode & 0xff;
478 
479 	reg = readl(reg_base + CQSPI_REG_OP_EXT_LOWER);
480 	reg &= ~(0xff << shift);
481 	reg |= ext << shift;
482 	writel(reg, reg_base + CQSPI_REG_OP_EXT_LOWER);
483 
484 	return 0;
485 }
486 
487 static int cqspi_enable_dtr(struct cqspi_flash_pdata *f_pdata,
488 			    const struct spi_mem_op *op, unsigned int shift)
489 {
490 	struct cqspi_st *cqspi = f_pdata->cqspi;
491 	void __iomem *reg_base = cqspi->iobase;
492 	unsigned int reg;
493 	int ret;
494 
495 	reg = readl(reg_base + CQSPI_REG_CONFIG);
496 
497 	/*
498 	 * We enable dual byte opcode here. The callers have to set up the
499 	 * extension opcode based on which type of operation it is.
500 	 */
501 	if (op->cmd.dtr) {
502 		reg |= CQSPI_REG_CONFIG_DTR_PROTO;
503 		reg |= CQSPI_REG_CONFIG_DUAL_OPCODE;
504 
505 		/* Set up command opcode extension. */
506 		ret = cqspi_setup_opcode_ext(f_pdata, op, shift);
507 		if (ret)
508 			return ret;
509 	} else {
510 		unsigned int mask = CQSPI_REG_CONFIG_DTR_PROTO | CQSPI_REG_CONFIG_DUAL_OPCODE;
511 		/* Shortcut if DTR is already disabled. */
512 		if ((reg & mask) == 0)
513 			return 0;
514 		reg &= ~mask;
515 	}
516 
517 	writel(reg, reg_base + CQSPI_REG_CONFIG);
518 
519 	return cqspi_wait_idle(cqspi);
520 }
521 
522 static int cqspi_command_read(struct cqspi_flash_pdata *f_pdata,
523 			      const struct spi_mem_op *op)
524 {
525 	struct cqspi_st *cqspi = f_pdata->cqspi;
526 	void __iomem *reg_base = cqspi->iobase;
527 	u8 *rxbuf = op->data.buf.in;
528 	u8 opcode;
529 	size_t n_rx = op->data.nbytes;
530 	unsigned int rdreg;
531 	unsigned int reg;
532 	unsigned int dummy_clk;
533 	size_t read_len;
534 	int status;
535 
536 	status = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_STIG_LSB);
537 	if (status)
538 		return status;
539 
540 	if (!n_rx || n_rx > CQSPI_STIG_DATA_LEN_MAX || !rxbuf) {
541 		dev_err(&cqspi->pdev->dev,
542 			"Invalid input argument, len %zu rxbuf 0x%p\n",
543 			n_rx, rxbuf);
544 		return -EINVAL;
545 	}
546 
547 	if (op->cmd.dtr)
548 		opcode = op->cmd.opcode >> 8;
549 	else
550 		opcode = op->cmd.opcode;
551 
552 	reg = opcode << CQSPI_REG_CMDCTRL_OPCODE_LSB;
553 
554 	rdreg = cqspi_calc_rdreg(op);
555 	writel(rdreg, reg_base + CQSPI_REG_RD_INSTR);
556 
557 	dummy_clk = cqspi_calc_dummy(op);
558 	if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
559 		return -EOPNOTSUPP;
560 
561 	if (dummy_clk)
562 		reg |= (dummy_clk & CQSPI_REG_CMDCTRL_DUMMY_MASK)
563 		     << CQSPI_REG_CMDCTRL_DUMMY_LSB;
564 
565 	reg |= (0x1 << CQSPI_REG_CMDCTRL_RD_EN_LSB);
566 
567 	/* 0 means 1 byte. */
568 	reg |= (((n_rx - 1) & CQSPI_REG_CMDCTRL_RD_BYTES_MASK)
569 		<< CQSPI_REG_CMDCTRL_RD_BYTES_LSB);
570 
571 	/* setup ADDR BIT field */
572 	if (op->addr.nbytes) {
573 		reg |= (0x1 << CQSPI_REG_CMDCTRL_ADDR_EN_LSB);
574 		reg |= ((op->addr.nbytes - 1) &
575 			CQSPI_REG_CMDCTRL_ADD_BYTES_MASK)
576 			<< CQSPI_REG_CMDCTRL_ADD_BYTES_LSB;
577 
578 		writel(op->addr.val, reg_base + CQSPI_REG_CMDADDRESS);
579 	}
580 
581 	status = cqspi_exec_flash_cmd(cqspi, reg);
582 	if (status)
583 		return status;
584 
585 	reg = readl(reg_base + CQSPI_REG_CMDREADDATALOWER);
586 
587 	/* Put the read value into rx_buf */
588 	read_len = (n_rx > 4) ? 4 : n_rx;
589 	memcpy(rxbuf, &reg, read_len);
590 	rxbuf += read_len;
591 
592 	if (n_rx > 4) {
593 		reg = readl(reg_base + CQSPI_REG_CMDREADDATAUPPER);
594 
595 		read_len = n_rx - read_len;
596 		memcpy(rxbuf, &reg, read_len);
597 	}
598 
599 	/* Reset CMD_CTRL Reg once command read completes */
600 	writel(0, reg_base + CQSPI_REG_CMDCTRL);
601 
602 	return 0;
603 }
604 
605 static int cqspi_command_write(struct cqspi_flash_pdata *f_pdata,
606 			       const struct spi_mem_op *op)
607 {
608 	struct cqspi_st *cqspi = f_pdata->cqspi;
609 	void __iomem *reg_base = cqspi->iobase;
610 	u8 opcode;
611 	const u8 *txbuf = op->data.buf.out;
612 	size_t n_tx = op->data.nbytes;
613 	unsigned int reg;
614 	unsigned int data;
615 	size_t write_len;
616 	int ret;
617 
618 	ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_STIG_LSB);
619 	if (ret)
620 		return ret;
621 
622 	if (n_tx > CQSPI_STIG_DATA_LEN_MAX || (n_tx && !txbuf)) {
623 		dev_err(&cqspi->pdev->dev,
624 			"Invalid input argument, cmdlen %zu txbuf 0x%p\n",
625 			n_tx, txbuf);
626 		return -EINVAL;
627 	}
628 
629 	reg = cqspi_calc_rdreg(op);
630 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
631 
632 	if (op->cmd.dtr)
633 		opcode = op->cmd.opcode >> 8;
634 	else
635 		opcode = op->cmd.opcode;
636 
637 	reg = opcode << CQSPI_REG_CMDCTRL_OPCODE_LSB;
638 
639 	if (op->addr.nbytes) {
640 		reg |= (0x1 << CQSPI_REG_CMDCTRL_ADDR_EN_LSB);
641 		reg |= ((op->addr.nbytes - 1) &
642 			CQSPI_REG_CMDCTRL_ADD_BYTES_MASK)
643 			<< CQSPI_REG_CMDCTRL_ADD_BYTES_LSB;
644 
645 		writel(op->addr.val, reg_base + CQSPI_REG_CMDADDRESS);
646 	}
647 
648 	if (n_tx) {
649 		reg |= (0x1 << CQSPI_REG_CMDCTRL_WR_EN_LSB);
650 		reg |= ((n_tx - 1) & CQSPI_REG_CMDCTRL_WR_BYTES_MASK)
651 			<< CQSPI_REG_CMDCTRL_WR_BYTES_LSB;
652 		data = 0;
653 		write_len = (n_tx > 4) ? 4 : n_tx;
654 		memcpy(&data, txbuf, write_len);
655 		txbuf += write_len;
656 		writel(data, reg_base + CQSPI_REG_CMDWRITEDATALOWER);
657 
658 		if (n_tx > 4) {
659 			data = 0;
660 			write_len = n_tx - 4;
661 			memcpy(&data, txbuf, write_len);
662 			writel(data, reg_base + CQSPI_REG_CMDWRITEDATAUPPER);
663 		}
664 	}
665 
666 	ret = cqspi_exec_flash_cmd(cqspi, reg);
667 
668 	/* Reset CMD_CTRL Reg once command write completes */
669 	writel(0, reg_base + CQSPI_REG_CMDCTRL);
670 
671 	return ret;
672 }
673 
674 static int cqspi_read_setup(struct cqspi_flash_pdata *f_pdata,
675 			    const struct spi_mem_op *op)
676 {
677 	struct cqspi_st *cqspi = f_pdata->cqspi;
678 	void __iomem *reg_base = cqspi->iobase;
679 	unsigned int dummy_clk = 0;
680 	unsigned int reg;
681 	int ret;
682 	u8 opcode;
683 
684 	ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_READ_LSB);
685 	if (ret)
686 		return ret;
687 
688 	if (op->cmd.dtr)
689 		opcode = op->cmd.opcode >> 8;
690 	else
691 		opcode = op->cmd.opcode;
692 
693 	reg = opcode << CQSPI_REG_RD_INSTR_OPCODE_LSB;
694 	reg |= cqspi_calc_rdreg(op);
695 
696 	/* Setup dummy clock cycles */
697 	dummy_clk = cqspi_calc_dummy(op);
698 
699 	if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
700 		return -EOPNOTSUPP;
701 
702 	if (dummy_clk)
703 		reg |= (dummy_clk & CQSPI_REG_RD_INSTR_DUMMY_MASK)
704 		       << CQSPI_REG_RD_INSTR_DUMMY_LSB;
705 
706 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
707 
708 	/* Set address width */
709 	reg = readl(reg_base + CQSPI_REG_SIZE);
710 	reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
711 	reg |= (op->addr.nbytes - 1);
712 	writel(reg, reg_base + CQSPI_REG_SIZE);
713 	return 0;
714 }
715 
716 static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata,
717 				       u8 *rxbuf, loff_t from_addr,
718 				       const size_t n_rx)
719 {
720 	struct cqspi_st *cqspi = f_pdata->cqspi;
721 	bool use_irq = !(cqspi->ddata && cqspi->ddata->quirks & CQSPI_RD_NO_IRQ);
722 	struct device *dev = &cqspi->pdev->dev;
723 	void __iomem *reg_base = cqspi->iobase;
724 	void __iomem *ahb_base = cqspi->ahb_base;
725 	unsigned int remaining = n_rx;
726 	unsigned int mod_bytes = n_rx % 4;
727 	unsigned int bytes_to_read = 0;
728 	u8 *rxbuf_end = rxbuf + n_rx;
729 	int ret = 0;
730 
731 	writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
732 	writel(remaining, reg_base + CQSPI_REG_INDIRECTRDBYTES);
733 
734 	/* Clear all interrupts. */
735 	writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
736 
737 	/*
738 	 * On SoCFPGA platform reading the SRAM is slow due to
739 	 * hardware limitation and causing read interrupt storm to CPU,
740 	 * so enabling only watermark interrupt to disable all read
741 	 * interrupts later as we want to run "bytes to read" loop with
742 	 * all the read interrupts disabled for max performance.
743 	 */
744 
745 	if (use_irq && cqspi->slow_sram)
746 		writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK);
747 	else if (use_irq)
748 		writel(CQSPI_IRQ_MASK_RD, reg_base + CQSPI_REG_IRQMASK);
749 	else
750 		writel(0, reg_base + CQSPI_REG_IRQMASK);
751 
752 	reinit_completion(&cqspi->transfer_complete);
753 	writel(CQSPI_REG_INDIRECTRD_START_MASK,
754 	       reg_base + CQSPI_REG_INDIRECTRD);
755 
756 	while (remaining > 0) {
757 		if (use_irq &&
758 		    !wait_for_completion_timeout(&cqspi->transfer_complete,
759 						 msecs_to_jiffies(CQSPI_READ_TIMEOUT_MS)))
760 			ret = -ETIMEDOUT;
761 
762 		/*
763 		 * Disable all read interrupts until
764 		 * we are out of "bytes to read"
765 		 */
766 		if (cqspi->slow_sram)
767 			writel(0x0, reg_base + CQSPI_REG_IRQMASK);
768 
769 		bytes_to_read = cqspi_get_rd_sram_level(cqspi);
770 
771 		if (ret && bytes_to_read == 0) {
772 			dev_err(dev, "Indirect read timeout, no bytes\n");
773 			goto failrd;
774 		}
775 
776 		while (bytes_to_read != 0) {
777 			unsigned int word_remain = round_down(remaining, 4);
778 
779 			bytes_to_read *= cqspi->fifo_width;
780 			bytes_to_read = bytes_to_read > remaining ?
781 					remaining : bytes_to_read;
782 			bytes_to_read = round_down(bytes_to_read, 4);
783 			/* Read 4 byte word chunks then single bytes */
784 			if (bytes_to_read) {
785 				ioread32_rep(ahb_base, rxbuf,
786 					     (bytes_to_read / 4));
787 			} else if (!word_remain && mod_bytes) {
788 				unsigned int temp = ioread32(ahb_base);
789 
790 				bytes_to_read = mod_bytes;
791 				memcpy(rxbuf, &temp, min((unsigned int)
792 							 (rxbuf_end - rxbuf),
793 							 bytes_to_read));
794 			}
795 			rxbuf += bytes_to_read;
796 			remaining -= bytes_to_read;
797 			bytes_to_read = cqspi_get_rd_sram_level(cqspi);
798 		}
799 
800 		if (use_irq && remaining > 0) {
801 			reinit_completion(&cqspi->transfer_complete);
802 			if (cqspi->slow_sram)
803 				writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK);
804 		}
805 	}
806 
807 	/* Check indirect done status */
808 	ret = cqspi_wait_for_bit(cqspi->ddata, reg_base + CQSPI_REG_INDIRECTRD,
809 				 CQSPI_REG_INDIRECTRD_DONE_MASK, 0, true);
810 	if (ret) {
811 		dev_err(dev, "Indirect read completion error (%i)\n", ret);
812 		goto failrd;
813 	}
814 
815 	/* Disable interrupt */
816 	writel(0, reg_base + CQSPI_REG_IRQMASK);
817 
818 	/* Clear indirect completion status */
819 	writel(CQSPI_REG_INDIRECTRD_DONE_MASK, reg_base + CQSPI_REG_INDIRECTRD);
820 
821 	return 0;
822 
823 failrd:
824 	/* Disable interrupt */
825 	writel(0, reg_base + CQSPI_REG_IRQMASK);
826 
827 	/* Cancel the indirect read */
828 	writel(CQSPI_REG_INDIRECTRD_CANCEL_MASK,
829 	       reg_base + CQSPI_REG_INDIRECTRD);
830 	return ret;
831 }
832 
833 static void cqspi_controller_enable(struct cqspi_st *cqspi, bool enable)
834 {
835 	void __iomem *reg_base = cqspi->iobase;
836 	unsigned int reg;
837 
838 	reg = readl(reg_base + CQSPI_REG_CONFIG);
839 
840 	if (enable)
841 		reg |= CQSPI_REG_CONFIG_ENABLE_MASK;
842 	else
843 		reg &= ~CQSPI_REG_CONFIG_ENABLE_MASK;
844 
845 	writel(reg, reg_base + CQSPI_REG_CONFIG);
846 }
847 
848 static int cqspi_versal_indirect_read_dma(struct cqspi_flash_pdata *f_pdata,
849 					  u_char *rxbuf, loff_t from_addr,
850 					  size_t n_rx)
851 {
852 	struct cqspi_st *cqspi = f_pdata->cqspi;
853 	struct device *dev = &cqspi->pdev->dev;
854 	void __iomem *reg_base = cqspi->iobase;
855 	u32 reg, bytes_to_dma;
856 	loff_t addr = from_addr;
857 	void *buf = rxbuf;
858 	dma_addr_t dma_addr;
859 	u8 bytes_rem;
860 	int ret = 0;
861 
862 	bytes_rem = n_rx % 4;
863 	bytes_to_dma = (n_rx - bytes_rem);
864 
865 	if (!bytes_to_dma)
866 		goto nondmard;
867 
868 	ret = zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id, PM_OSPI_MUX_SEL_DMA);
869 	if (ret)
870 		return ret;
871 
872 	cqspi_controller_enable(cqspi, 0);
873 
874 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
875 	reg |= CQSPI_REG_CONFIG_DMA_MASK;
876 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
877 
878 	cqspi_controller_enable(cqspi, 1);
879 
880 	dma_addr = dma_map_single(dev, rxbuf, bytes_to_dma, DMA_FROM_DEVICE);
881 	if (dma_mapping_error(dev, dma_addr)) {
882 		dev_err(dev, "dma mapping failed\n");
883 		return -ENOMEM;
884 	}
885 
886 	writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
887 	writel(bytes_to_dma, reg_base + CQSPI_REG_INDIRECTRDBYTES);
888 	writel(CQSPI_REG_VERSAL_ADDRRANGE_WIDTH_VAL,
889 	       reg_base + CQSPI_REG_INDTRIG_ADDRRANGE);
890 
891 	/* Clear all interrupts. */
892 	writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
893 
894 	/* Enable DMA done interrupt */
895 	writel(CQSPI_REG_VERSAL_DMA_DST_DONE_MASK,
896 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_I_EN);
897 
898 	/* Default DMA periph configuration */
899 	writel(CQSPI_REG_VERSAL_DMA_VAL, reg_base + CQSPI_REG_DMA);
900 
901 	/* Configure DMA Dst address */
902 	writel(lower_32_bits(dma_addr),
903 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_ADDR);
904 	writel(upper_32_bits(dma_addr),
905 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_ADDR_MSB);
906 
907 	/* Configure DMA Src address */
908 	writel(cqspi->trigger_address, reg_base +
909 	       CQSPI_REG_VERSAL_DMA_SRC_ADDR);
910 
911 	/* Set DMA destination size */
912 	writel(bytes_to_dma, reg_base + CQSPI_REG_VERSAL_DMA_DST_SIZE);
913 
914 	/* Set DMA destination control */
915 	writel(CQSPI_REG_VERSAL_DMA_DST_CTRL_VAL,
916 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_CTRL);
917 
918 	writel(CQSPI_REG_INDIRECTRD_START_MASK,
919 	       reg_base + CQSPI_REG_INDIRECTRD);
920 
921 	reinit_completion(&cqspi->transfer_complete);
922 
923 	if (!wait_for_completion_timeout(&cqspi->transfer_complete,
924 					 msecs_to_jiffies(max_t(size_t, bytes_to_dma, 500)))) {
925 		ret = -ETIMEDOUT;
926 		goto failrd;
927 	}
928 
929 	/* Disable DMA interrupt */
930 	writel(0x0, cqspi->iobase + CQSPI_REG_VERSAL_DMA_DST_I_DIS);
931 
932 	/* Clear indirect completion status */
933 	writel(CQSPI_REG_INDIRECTRD_DONE_MASK,
934 	       cqspi->iobase + CQSPI_REG_INDIRECTRD);
935 	dma_unmap_single(dev, dma_addr, bytes_to_dma, DMA_FROM_DEVICE);
936 
937 	cqspi_controller_enable(cqspi, 0);
938 
939 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
940 	reg &= ~CQSPI_REG_CONFIG_DMA_MASK;
941 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
942 
943 	cqspi_controller_enable(cqspi, 1);
944 
945 	ret = zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id,
946 					PM_OSPI_MUX_SEL_LINEAR);
947 	if (ret)
948 		return ret;
949 
950 nondmard:
951 	if (bytes_rem) {
952 		addr += bytes_to_dma;
953 		buf += bytes_to_dma;
954 		ret = cqspi_indirect_read_execute(f_pdata, buf, addr,
955 						  bytes_rem);
956 		if (ret)
957 			return ret;
958 	}
959 
960 	return 0;
961 
962 failrd:
963 	/* Disable DMA interrupt */
964 	writel(0x0, reg_base + CQSPI_REG_VERSAL_DMA_DST_I_DIS);
965 
966 	/* Cancel the indirect read */
967 	writel(CQSPI_REG_INDIRECTWR_CANCEL_MASK,
968 	       reg_base + CQSPI_REG_INDIRECTRD);
969 
970 	dma_unmap_single(dev, dma_addr, bytes_to_dma, DMA_FROM_DEVICE);
971 
972 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
973 	reg &= ~CQSPI_REG_CONFIG_DMA_MASK;
974 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
975 
976 	zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id, PM_OSPI_MUX_SEL_LINEAR);
977 
978 	return ret;
979 }
980 
981 static int cqspi_write_setup(struct cqspi_flash_pdata *f_pdata,
982 			     const struct spi_mem_op *op)
983 {
984 	unsigned int reg;
985 	int ret;
986 	struct cqspi_st *cqspi = f_pdata->cqspi;
987 	void __iomem *reg_base = cqspi->iobase;
988 	u8 opcode;
989 
990 	ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_WRITE_LSB);
991 	if (ret)
992 		return ret;
993 
994 	if (op->cmd.dtr)
995 		opcode = op->cmd.opcode >> 8;
996 	else
997 		opcode = op->cmd.opcode;
998 
999 	/* Set opcode. */
1000 	reg = opcode << CQSPI_REG_WR_INSTR_OPCODE_LSB;
1001 	reg |= CQSPI_OP_WIDTH(op->data) << CQSPI_REG_WR_INSTR_TYPE_DATA_LSB;
1002 	reg |= CQSPI_OP_WIDTH(op->addr) << CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB;
1003 	writel(reg, reg_base + CQSPI_REG_WR_INSTR);
1004 	reg = cqspi_calc_rdreg(op);
1005 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
1006 
1007 	/*
1008 	 * SPI NAND flashes require the address of the status register to be
1009 	 * passed in the Read SR command. Also, some SPI NOR flashes like the
1010 	 * cypress Semper flash expect a 4-byte dummy address in the Read SR
1011 	 * command in DTR mode.
1012 	 *
1013 	 * But this controller does not support address phase in the Read SR
1014 	 * command when doing auto-HW polling. So, disable write completion
1015 	 * polling on the controller's side. spinand and spi-nor will take
1016 	 * care of polling the status register.
1017 	 */
1018 	if (cqspi->wr_completion) {
1019 		reg = readl(reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
1020 		reg |= CQSPI_REG_WR_DISABLE_AUTO_POLL;
1021 		writel(reg, reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
1022 		/*
1023 		 * DAC mode require auto polling as flash needs to be polled
1024 		 * for write completion in case of bubble in SPI transaction
1025 		 * due to slow CPU/DMA master.
1026 		 */
1027 		cqspi->use_direct_mode_wr = false;
1028 	}
1029 
1030 	reg = readl(reg_base + CQSPI_REG_SIZE);
1031 	reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
1032 	reg |= (op->addr.nbytes - 1);
1033 	writel(reg, reg_base + CQSPI_REG_SIZE);
1034 	return 0;
1035 }
1036 
1037 static int cqspi_indirect_write_execute(struct cqspi_flash_pdata *f_pdata,
1038 					loff_t to_addr, const u8 *txbuf,
1039 					const size_t n_tx)
1040 {
1041 	struct cqspi_st *cqspi = f_pdata->cqspi;
1042 	struct device *dev = &cqspi->pdev->dev;
1043 	void __iomem *reg_base = cqspi->iobase;
1044 	unsigned int remaining = n_tx;
1045 	unsigned int write_bytes;
1046 	int ret;
1047 
1048 	writel(to_addr, reg_base + CQSPI_REG_INDIRECTWRSTARTADDR);
1049 	writel(remaining, reg_base + CQSPI_REG_INDIRECTWRBYTES);
1050 
1051 	/* Clear all interrupts. */
1052 	writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
1053 
1054 	writel(CQSPI_IRQ_MASK_WR, reg_base + CQSPI_REG_IRQMASK);
1055 
1056 	reinit_completion(&cqspi->transfer_complete);
1057 	writel(CQSPI_REG_INDIRECTWR_START_MASK,
1058 	       reg_base + CQSPI_REG_INDIRECTWR);
1059 	/*
1060 	 * As per 66AK2G02 TRM SPRUHY8F section 11.15.5.3 Indirect Access
1061 	 * Controller programming sequence, couple of cycles of
1062 	 * QSPI_REF_CLK delay is required for the above bit to
1063 	 * be internally synchronized by the QSPI module. Provide 5
1064 	 * cycles of delay.
1065 	 */
1066 	if (cqspi->wr_delay)
1067 		ndelay(cqspi->wr_delay);
1068 
1069 	/*
1070 	 * If a hazard exists between the APB and AHB interfaces, perform a
1071 	 * dummy readback from the controller to ensure synchronization.
1072 	 */
1073 	if (cqspi->apb_ahb_hazard)
1074 		readl(reg_base + CQSPI_REG_INDIRECTWR);
1075 
1076 	while (remaining > 0) {
1077 		size_t write_words, mod_bytes;
1078 
1079 		write_bytes = remaining;
1080 		write_words = write_bytes / 4;
1081 		mod_bytes = write_bytes % 4;
1082 		/* Write 4 bytes at a time then single bytes. */
1083 		if (write_words) {
1084 			iowrite32_rep(cqspi->ahb_base, txbuf, write_words);
1085 			txbuf += (write_words * 4);
1086 		}
1087 		if (mod_bytes) {
1088 			unsigned int temp = 0xFFFFFFFF;
1089 
1090 			memcpy(&temp, txbuf, mod_bytes);
1091 			iowrite32(temp, cqspi->ahb_base);
1092 			txbuf += mod_bytes;
1093 		}
1094 
1095 		if (!wait_for_completion_timeout(&cqspi->transfer_complete,
1096 						 msecs_to_jiffies(CQSPI_TIMEOUT_MS))) {
1097 			dev_err(dev, "Indirect write timeout\n");
1098 			ret = -ETIMEDOUT;
1099 			goto failwr;
1100 		}
1101 
1102 		remaining -= write_bytes;
1103 
1104 		if (remaining > 0)
1105 			reinit_completion(&cqspi->transfer_complete);
1106 	}
1107 
1108 	/* Check indirect done status */
1109 	ret = cqspi_wait_for_bit(cqspi->ddata, reg_base + CQSPI_REG_INDIRECTWR,
1110 				 CQSPI_REG_INDIRECTWR_DONE_MASK, 0, false);
1111 	if (ret) {
1112 		dev_err(dev, "Indirect write completion error (%i)\n", ret);
1113 		goto failwr;
1114 	}
1115 
1116 	/* Disable interrupt. */
1117 	writel(0, reg_base + CQSPI_REG_IRQMASK);
1118 
1119 	/* Clear indirect completion status */
1120 	writel(CQSPI_REG_INDIRECTWR_DONE_MASK, reg_base + CQSPI_REG_INDIRECTWR);
1121 
1122 	cqspi_wait_idle(cqspi);
1123 
1124 	return 0;
1125 
1126 failwr:
1127 	/* Disable interrupt. */
1128 	writel(0, reg_base + CQSPI_REG_IRQMASK);
1129 
1130 	/* Cancel the indirect write */
1131 	writel(CQSPI_REG_INDIRECTWR_CANCEL_MASK,
1132 	       reg_base + CQSPI_REG_INDIRECTWR);
1133 	return ret;
1134 }
1135 
1136 static void cqspi_chipselect(struct cqspi_flash_pdata *f_pdata)
1137 {
1138 	struct cqspi_st *cqspi = f_pdata->cqspi;
1139 	void __iomem *reg_base = cqspi->iobase;
1140 	unsigned int chip_select = f_pdata->cs;
1141 	unsigned int reg;
1142 
1143 	reg = readl(reg_base + CQSPI_REG_CONFIG);
1144 	if (cqspi->is_decoded_cs) {
1145 		reg |= CQSPI_REG_CONFIG_DECODE_MASK;
1146 	} else {
1147 		reg &= ~CQSPI_REG_CONFIG_DECODE_MASK;
1148 
1149 		/* Convert CS if without decoder.
1150 		 * CS0 to 4b'1110
1151 		 * CS1 to 4b'1101
1152 		 * CS2 to 4b'1011
1153 		 * CS3 to 4b'0111
1154 		 */
1155 		chip_select = 0xF & ~(1 << chip_select);
1156 	}
1157 
1158 	reg &= ~(CQSPI_REG_CONFIG_CHIPSELECT_MASK
1159 		 << CQSPI_REG_CONFIG_CHIPSELECT_LSB);
1160 	reg |= (chip_select & CQSPI_REG_CONFIG_CHIPSELECT_MASK)
1161 	    << CQSPI_REG_CONFIG_CHIPSELECT_LSB;
1162 	writel(reg, reg_base + CQSPI_REG_CONFIG);
1163 }
1164 
1165 static unsigned int calculate_ticks_for_ns(const unsigned int ref_clk_hz,
1166 					   const unsigned int ns_val)
1167 {
1168 	unsigned int ticks;
1169 
1170 	ticks = ref_clk_hz / 1000;	/* kHz */
1171 	ticks = DIV_ROUND_UP(ticks * ns_val, 1000000);
1172 
1173 	return ticks;
1174 }
1175 
1176 static void cqspi_delay(struct cqspi_flash_pdata *f_pdata)
1177 {
1178 	struct cqspi_st *cqspi = f_pdata->cqspi;
1179 	void __iomem *iobase = cqspi->iobase;
1180 	const unsigned int ref_clk_hz = cqspi->master_ref_clk_hz;
1181 	unsigned int tshsl, tchsh, tslch, tsd2d;
1182 	unsigned int reg;
1183 	unsigned int tsclk;
1184 
1185 	/* calculate the number of ref ticks for one sclk tick */
1186 	tsclk = DIV_ROUND_UP(ref_clk_hz, cqspi->sclk);
1187 
1188 	tshsl = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tshsl_ns);
1189 	/* this particular value must be at least one sclk */
1190 	if (tshsl < tsclk)
1191 		tshsl = tsclk;
1192 
1193 	tchsh = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tchsh_ns);
1194 	tslch = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tslch_ns);
1195 	tsd2d = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tsd2d_ns);
1196 
1197 	reg = (tshsl & CQSPI_REG_DELAY_TSHSL_MASK)
1198 	       << CQSPI_REG_DELAY_TSHSL_LSB;
1199 	reg |= (tchsh & CQSPI_REG_DELAY_TCHSH_MASK)
1200 		<< CQSPI_REG_DELAY_TCHSH_LSB;
1201 	reg |= (tslch & CQSPI_REG_DELAY_TSLCH_MASK)
1202 		<< CQSPI_REG_DELAY_TSLCH_LSB;
1203 	reg |= (tsd2d & CQSPI_REG_DELAY_TSD2D_MASK)
1204 		<< CQSPI_REG_DELAY_TSD2D_LSB;
1205 	writel(reg, iobase + CQSPI_REG_DELAY);
1206 }
1207 
1208 static void cqspi_config_baudrate_div(struct cqspi_st *cqspi)
1209 {
1210 	const unsigned int ref_clk_hz = cqspi->master_ref_clk_hz;
1211 	void __iomem *reg_base = cqspi->iobase;
1212 	u32 reg, div;
1213 
1214 	/* Recalculate the baudrate divisor based on QSPI specification. */
1215 	div = DIV_ROUND_UP(ref_clk_hz, 2 * cqspi->sclk) - 1;
1216 
1217 	/* Maximum baud divisor */
1218 	if (div > CQSPI_REG_CONFIG_BAUD_MASK) {
1219 		div = CQSPI_REG_CONFIG_BAUD_MASK;
1220 		dev_warn(&cqspi->pdev->dev,
1221 			"Unable to adjust clock <= %d hz. Reduced to %d hz\n",
1222 			cqspi->sclk, ref_clk_hz/((div+1)*2));
1223 	}
1224 
1225 	reg = readl(reg_base + CQSPI_REG_CONFIG);
1226 	reg &= ~(CQSPI_REG_CONFIG_BAUD_MASK << CQSPI_REG_CONFIG_BAUD_LSB);
1227 	reg |= (div & CQSPI_REG_CONFIG_BAUD_MASK) << CQSPI_REG_CONFIG_BAUD_LSB;
1228 	writel(reg, reg_base + CQSPI_REG_CONFIG);
1229 }
1230 
1231 static void cqspi_readdata_capture(struct cqspi_st *cqspi,
1232 				   const bool bypass,
1233 				   const unsigned int delay)
1234 {
1235 	void __iomem *reg_base = cqspi->iobase;
1236 	unsigned int reg;
1237 
1238 	reg = readl(reg_base + CQSPI_REG_READCAPTURE);
1239 
1240 	if (bypass)
1241 		reg |= (1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
1242 	else
1243 		reg &= ~(1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
1244 
1245 	reg &= ~(CQSPI_REG_READCAPTURE_DELAY_MASK
1246 		 << CQSPI_REG_READCAPTURE_DELAY_LSB);
1247 
1248 	reg |= (delay & CQSPI_REG_READCAPTURE_DELAY_MASK)
1249 		<< CQSPI_REG_READCAPTURE_DELAY_LSB;
1250 
1251 	writel(reg, reg_base + CQSPI_REG_READCAPTURE);
1252 }
1253 
1254 static void cqspi_configure(struct cqspi_flash_pdata *f_pdata,
1255 			    unsigned long sclk)
1256 {
1257 	struct cqspi_st *cqspi = f_pdata->cqspi;
1258 	int switch_cs = (cqspi->current_cs != f_pdata->cs);
1259 	int switch_ck = (cqspi->sclk != sclk);
1260 
1261 	if (switch_cs || switch_ck)
1262 		cqspi_controller_enable(cqspi, 0);
1263 
1264 	/* Switch chip select. */
1265 	if (switch_cs) {
1266 		cqspi->current_cs = f_pdata->cs;
1267 		cqspi_chipselect(f_pdata);
1268 	}
1269 
1270 	/* Setup baudrate divisor and delays */
1271 	if (switch_ck) {
1272 		cqspi->sclk = sclk;
1273 		cqspi_config_baudrate_div(cqspi);
1274 		cqspi_delay(f_pdata);
1275 		cqspi_readdata_capture(cqspi, !cqspi->rclk_en,
1276 				       f_pdata->read_delay);
1277 	}
1278 
1279 	if (switch_cs || switch_ck)
1280 		cqspi_controller_enable(cqspi, 1);
1281 }
1282 
1283 static ssize_t cqspi_write(struct cqspi_flash_pdata *f_pdata,
1284 			   const struct spi_mem_op *op)
1285 {
1286 	struct cqspi_st *cqspi = f_pdata->cqspi;
1287 	loff_t to = op->addr.val;
1288 	size_t len = op->data.nbytes;
1289 	const u_char *buf = op->data.buf.out;
1290 	int ret;
1291 
1292 	ret = cqspi_write_setup(f_pdata, op);
1293 	if (ret)
1294 		return ret;
1295 
1296 	/*
1297 	 * Some flashes like the Cypress Semper flash expect a dummy 4-byte
1298 	 * address (all 0s) with the read status register command in DTR mode.
1299 	 * But this controller does not support sending dummy address bytes to
1300 	 * the flash when it is polling the write completion register in DTR
1301 	 * mode. So, we can not use direct mode when in DTR mode for writing
1302 	 * data.
1303 	 */
1304 	if (!op->cmd.dtr && cqspi->use_direct_mode &&
1305 	    cqspi->use_direct_mode_wr && ((to + len) <= cqspi->ahb_size)) {
1306 		memcpy_toio(cqspi->ahb_base + to, buf, len);
1307 		return cqspi_wait_idle(cqspi);
1308 	}
1309 
1310 	return cqspi_indirect_write_execute(f_pdata, to, buf, len);
1311 }
1312 
1313 static void cqspi_rx_dma_callback(void *param)
1314 {
1315 	struct cqspi_st *cqspi = param;
1316 
1317 	complete(&cqspi->rx_dma_complete);
1318 }
1319 
1320 static int cqspi_direct_read_execute(struct cqspi_flash_pdata *f_pdata,
1321 				     u_char *buf, loff_t from, size_t len)
1322 {
1323 	struct cqspi_st *cqspi = f_pdata->cqspi;
1324 	struct device *dev = &cqspi->pdev->dev;
1325 	enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
1326 	dma_addr_t dma_src = (dma_addr_t)cqspi->mmap_phys_base + from;
1327 	int ret = 0;
1328 	struct dma_async_tx_descriptor *tx;
1329 	dma_cookie_t cookie;
1330 	dma_addr_t dma_dst;
1331 	struct device *ddev;
1332 
1333 	if (!cqspi->rx_chan || !virt_addr_valid(buf)) {
1334 		memcpy_fromio(buf, cqspi->ahb_base + from, len);
1335 		return 0;
1336 	}
1337 
1338 	ddev = cqspi->rx_chan->device->dev;
1339 	dma_dst = dma_map_single(ddev, buf, len, DMA_FROM_DEVICE);
1340 	if (dma_mapping_error(ddev, dma_dst)) {
1341 		dev_err(dev, "dma mapping failed\n");
1342 		return -ENOMEM;
1343 	}
1344 	tx = dmaengine_prep_dma_memcpy(cqspi->rx_chan, dma_dst, dma_src,
1345 				       len, flags);
1346 	if (!tx) {
1347 		dev_err(dev, "device_prep_dma_memcpy error\n");
1348 		ret = -EIO;
1349 		goto err_unmap;
1350 	}
1351 
1352 	tx->callback = cqspi_rx_dma_callback;
1353 	tx->callback_param = cqspi;
1354 	cookie = tx->tx_submit(tx);
1355 	reinit_completion(&cqspi->rx_dma_complete);
1356 
1357 	ret = dma_submit_error(cookie);
1358 	if (ret) {
1359 		dev_err(dev, "dma_submit_error %d\n", cookie);
1360 		ret = -EIO;
1361 		goto err_unmap;
1362 	}
1363 
1364 	dma_async_issue_pending(cqspi->rx_chan);
1365 	if (!wait_for_completion_timeout(&cqspi->rx_dma_complete,
1366 					 msecs_to_jiffies(max_t(size_t, len, 500)))) {
1367 		dmaengine_terminate_sync(cqspi->rx_chan);
1368 		dev_err(dev, "DMA wait_for_completion_timeout\n");
1369 		ret = -ETIMEDOUT;
1370 		goto err_unmap;
1371 	}
1372 
1373 err_unmap:
1374 	dma_unmap_single(ddev, dma_dst, len, DMA_FROM_DEVICE);
1375 
1376 	return ret;
1377 }
1378 
1379 static ssize_t cqspi_read(struct cqspi_flash_pdata *f_pdata,
1380 			  const struct spi_mem_op *op)
1381 {
1382 	struct cqspi_st *cqspi = f_pdata->cqspi;
1383 	const struct cqspi_driver_platdata *ddata = cqspi->ddata;
1384 	loff_t from = op->addr.val;
1385 	size_t len = op->data.nbytes;
1386 	u_char *buf = op->data.buf.in;
1387 	u64 dma_align = (u64)(uintptr_t)buf;
1388 	int ret;
1389 
1390 	ret = cqspi_read_setup(f_pdata, op);
1391 	if (ret)
1392 		return ret;
1393 
1394 	if (cqspi->use_direct_mode && ((from + len) <= cqspi->ahb_size))
1395 		return cqspi_direct_read_execute(f_pdata, buf, from, len);
1396 
1397 	if (cqspi->use_dma_read && ddata && ddata->indirect_read_dma &&
1398 	    virt_addr_valid(buf) && ((dma_align & CQSPI_DMA_UNALIGN) == 0))
1399 		return ddata->indirect_read_dma(f_pdata, buf, from, len);
1400 
1401 	return cqspi_indirect_read_execute(f_pdata, buf, from, len);
1402 }
1403 
1404 static int cqspi_mem_process(struct spi_mem *mem, const struct spi_mem_op *op)
1405 {
1406 	struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller);
1407 	struct cqspi_flash_pdata *f_pdata;
1408 
1409 	f_pdata = &cqspi->f_pdata[spi_get_chipselect(mem->spi, 0)];
1410 	cqspi_configure(f_pdata, mem->spi->max_speed_hz);
1411 
1412 	if (op->data.dir == SPI_MEM_DATA_IN && op->data.buf.in) {
1413 	/*
1414 	 * Performing reads in DAC mode forces to read minimum 4 bytes
1415 	 * which is unsupported on some flash devices during register
1416 	 * reads, prefer STIG mode for such small reads.
1417 	 */
1418 		if (!op->addr.nbytes ||
1419 		    op->data.nbytes <= CQSPI_STIG_DATA_LEN_MAX)
1420 			return cqspi_command_read(f_pdata, op);
1421 
1422 		return cqspi_read(f_pdata, op);
1423 	}
1424 
1425 	if (!op->addr.nbytes || !op->data.buf.out)
1426 		return cqspi_command_write(f_pdata, op);
1427 
1428 	return cqspi_write(f_pdata, op);
1429 }
1430 
1431 static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
1432 {
1433 	int ret;
1434 	struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller);
1435 	struct device *dev = &cqspi->pdev->dev;
1436 
1437 	ret = pm_runtime_resume_and_get(dev);
1438 	if (ret) {
1439 		dev_err(&mem->spi->dev, "resume failed with %d\n", ret);
1440 		return ret;
1441 	}
1442 
1443 	ret = cqspi_mem_process(mem, op);
1444 
1445 	pm_runtime_mark_last_busy(dev);
1446 	pm_runtime_put_autosuspend(dev);
1447 
1448 	if (ret)
1449 		dev_err(&mem->spi->dev, "operation failed with %d\n", ret);
1450 
1451 	return ret;
1452 }
1453 
1454 static bool cqspi_supports_mem_op(struct spi_mem *mem,
1455 				  const struct spi_mem_op *op)
1456 {
1457 	bool all_true, all_false;
1458 
1459 	/*
1460 	 * op->dummy.dtr is required for converting nbytes into ncycles.
1461 	 * Also, don't check the dtr field of the op phase having zero nbytes.
1462 	 */
1463 	all_true = op->cmd.dtr &&
1464 		   (!op->addr.nbytes || op->addr.dtr) &&
1465 		   (!op->dummy.nbytes || op->dummy.dtr) &&
1466 		   (!op->data.nbytes || op->data.dtr);
1467 
1468 	all_false = !op->cmd.dtr && !op->addr.dtr && !op->dummy.dtr &&
1469 		    !op->data.dtr;
1470 
1471 	if (all_true) {
1472 		/* Right now we only support 8-8-8 DTR mode. */
1473 		if (op->cmd.nbytes && op->cmd.buswidth != 8)
1474 			return false;
1475 		if (op->addr.nbytes && op->addr.buswidth != 8)
1476 			return false;
1477 		if (op->data.nbytes && op->data.buswidth != 8)
1478 			return false;
1479 	} else if (!all_false) {
1480 		/* Mixed DTR modes are not supported. */
1481 		return false;
1482 	}
1483 
1484 	return spi_mem_default_supports_op(mem, op);
1485 }
1486 
1487 static int cqspi_of_get_flash_pdata(struct platform_device *pdev,
1488 				    struct cqspi_flash_pdata *f_pdata,
1489 				    struct device_node *np)
1490 {
1491 	if (of_property_read_u32(np, "cdns,read-delay", &f_pdata->read_delay)) {
1492 		dev_err(&pdev->dev, "couldn't determine read-delay\n");
1493 		return -ENXIO;
1494 	}
1495 
1496 	if (of_property_read_u32(np, "cdns,tshsl-ns", &f_pdata->tshsl_ns)) {
1497 		dev_err(&pdev->dev, "couldn't determine tshsl-ns\n");
1498 		return -ENXIO;
1499 	}
1500 
1501 	if (of_property_read_u32(np, "cdns,tsd2d-ns", &f_pdata->tsd2d_ns)) {
1502 		dev_err(&pdev->dev, "couldn't determine tsd2d-ns\n");
1503 		return -ENXIO;
1504 	}
1505 
1506 	if (of_property_read_u32(np, "cdns,tchsh-ns", &f_pdata->tchsh_ns)) {
1507 		dev_err(&pdev->dev, "couldn't determine tchsh-ns\n");
1508 		return -ENXIO;
1509 	}
1510 
1511 	if (of_property_read_u32(np, "cdns,tslch-ns", &f_pdata->tslch_ns)) {
1512 		dev_err(&pdev->dev, "couldn't determine tslch-ns\n");
1513 		return -ENXIO;
1514 	}
1515 
1516 	if (of_property_read_u32(np, "spi-max-frequency", &f_pdata->clk_rate)) {
1517 		dev_err(&pdev->dev, "couldn't determine spi-max-frequency\n");
1518 		return -ENXIO;
1519 	}
1520 
1521 	return 0;
1522 }
1523 
1524 static int cqspi_of_get_pdata(struct cqspi_st *cqspi)
1525 {
1526 	struct device *dev = &cqspi->pdev->dev;
1527 	struct device_node *np = dev->of_node;
1528 	u32 id[2];
1529 
1530 	cqspi->is_decoded_cs = of_property_read_bool(np, "cdns,is-decoded-cs");
1531 
1532 	if (of_property_read_u32(np, "cdns,fifo-depth", &cqspi->fifo_depth)) {
1533 		/* Zero signals FIFO depth should be runtime detected. */
1534 		cqspi->fifo_depth = 0;
1535 	}
1536 
1537 	if (of_property_read_u32(np, "cdns,fifo-width", &cqspi->fifo_width)) {
1538 		dev_err(dev, "couldn't determine fifo-width\n");
1539 		return -ENXIO;
1540 	}
1541 
1542 	if (of_property_read_u32(np, "cdns,trigger-address",
1543 				 &cqspi->trigger_address)) {
1544 		dev_err(dev, "couldn't determine trigger-address\n");
1545 		return -ENXIO;
1546 	}
1547 
1548 	if (of_property_read_u32(np, "num-cs", &cqspi->num_chipselect))
1549 		cqspi->num_chipselect = CQSPI_MAX_CHIPSELECT;
1550 
1551 	cqspi->rclk_en = of_property_read_bool(np, "cdns,rclk-en");
1552 
1553 	if (!of_property_read_u32_array(np, "power-domains", id,
1554 					ARRAY_SIZE(id)))
1555 		cqspi->pd_dev_id = id[1];
1556 
1557 	return 0;
1558 }
1559 
1560 static void cqspi_controller_init(struct cqspi_st *cqspi)
1561 {
1562 	u32 reg;
1563 
1564 	/* Configure the remap address register, no remap */
1565 	writel(0, cqspi->iobase + CQSPI_REG_REMAP);
1566 
1567 	/* Disable all interrupts. */
1568 	writel(0, cqspi->iobase + CQSPI_REG_IRQMASK);
1569 
1570 	/* Configure the SRAM split to 1:1 . */
1571 	writel(cqspi->fifo_depth / 2, cqspi->iobase + CQSPI_REG_SRAMPARTITION);
1572 
1573 	/* Load indirect trigger address. */
1574 	writel(cqspi->trigger_address,
1575 	       cqspi->iobase + CQSPI_REG_INDIRECTTRIGGER);
1576 
1577 	/* Program read watermark -- 1/2 of the FIFO. */
1578 	writel(cqspi->fifo_depth * cqspi->fifo_width / 2,
1579 	       cqspi->iobase + CQSPI_REG_INDIRECTRDWATERMARK);
1580 	/* Program write watermark -- 1/8 of the FIFO. */
1581 	writel(cqspi->fifo_depth * cqspi->fifo_width / 8,
1582 	       cqspi->iobase + CQSPI_REG_INDIRECTWRWATERMARK);
1583 
1584 	/* Disable direct access controller */
1585 	if (!cqspi->use_direct_mode) {
1586 		reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
1587 		reg &= ~CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL;
1588 		writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
1589 	}
1590 
1591 	/* Enable DMA interface */
1592 	if (cqspi->use_dma_read) {
1593 		reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
1594 		reg |= CQSPI_REG_CONFIG_DMA_MASK;
1595 		writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
1596 	}
1597 }
1598 
1599 static void cqspi_controller_detect_fifo_depth(struct cqspi_st *cqspi)
1600 {
1601 	struct device *dev = &cqspi->pdev->dev;
1602 	u32 reg, fifo_depth;
1603 
1604 	/*
1605 	 * Bits N-1:0 are writable while bits 31:N are read as zero, with 2^N
1606 	 * the FIFO depth.
1607 	 */
1608 	writel(U32_MAX, cqspi->iobase + CQSPI_REG_SRAMPARTITION);
1609 	reg = readl(cqspi->iobase + CQSPI_REG_SRAMPARTITION);
1610 	fifo_depth = reg + 1;
1611 
1612 	/* FIFO depth of zero means no value from devicetree was provided. */
1613 	if (cqspi->fifo_depth == 0) {
1614 		cqspi->fifo_depth = fifo_depth;
1615 		dev_dbg(dev, "using FIFO depth of %u\n", fifo_depth);
1616 	} else if (fifo_depth != cqspi->fifo_depth) {
1617 		dev_warn(dev, "detected FIFO depth (%u) different from config (%u)\n",
1618 			 fifo_depth, cqspi->fifo_depth);
1619 	}
1620 }
1621 
1622 static int cqspi_request_mmap_dma(struct cqspi_st *cqspi)
1623 {
1624 	dma_cap_mask_t mask;
1625 
1626 	dma_cap_zero(mask);
1627 	dma_cap_set(DMA_MEMCPY, mask);
1628 
1629 	cqspi->rx_chan = dma_request_chan_by_mask(&mask);
1630 	if (IS_ERR(cqspi->rx_chan)) {
1631 		int ret = PTR_ERR(cqspi->rx_chan);
1632 
1633 		cqspi->rx_chan = NULL;
1634 		return dev_err_probe(&cqspi->pdev->dev, ret, "No Rx DMA available\n");
1635 	}
1636 	init_completion(&cqspi->rx_dma_complete);
1637 
1638 	return 0;
1639 }
1640 
1641 static const char *cqspi_get_name(struct spi_mem *mem)
1642 {
1643 	struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller);
1644 	struct device *dev = &cqspi->pdev->dev;
1645 
1646 	return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev),
1647 			      spi_get_chipselect(mem->spi, 0));
1648 }
1649 
1650 static const struct spi_controller_mem_ops cqspi_mem_ops = {
1651 	.exec_op = cqspi_exec_mem_op,
1652 	.get_name = cqspi_get_name,
1653 	.supports_op = cqspi_supports_mem_op,
1654 };
1655 
1656 static const struct spi_controller_mem_caps cqspi_mem_caps = {
1657 	.dtr = true,
1658 };
1659 
1660 static int cqspi_setup_flash(struct cqspi_st *cqspi)
1661 {
1662 	unsigned int max_cs = cqspi->num_chipselect - 1;
1663 	struct platform_device *pdev = cqspi->pdev;
1664 	struct device *dev = &pdev->dev;
1665 	struct cqspi_flash_pdata *f_pdata;
1666 	unsigned int cs;
1667 	int ret;
1668 
1669 	/* Get flash device data */
1670 	for_each_available_child_of_node_scoped(dev->of_node, np) {
1671 		ret = of_property_read_u32(np, "reg", &cs);
1672 		if (ret) {
1673 			dev_err(dev, "Couldn't determine chip select.\n");
1674 			return ret;
1675 		}
1676 
1677 		if (cs >= cqspi->num_chipselect) {
1678 			dev_err(dev, "Chip select %d out of range.\n", cs);
1679 			return -EINVAL;
1680 		} else if (cs < max_cs) {
1681 			max_cs = cs;
1682 		}
1683 
1684 		f_pdata = &cqspi->f_pdata[cs];
1685 		f_pdata->cqspi = cqspi;
1686 		f_pdata->cs = cs;
1687 
1688 		ret = cqspi_of_get_flash_pdata(pdev, f_pdata, np);
1689 		if (ret)
1690 			return ret;
1691 	}
1692 
1693 	cqspi->num_chipselect = max_cs + 1;
1694 	return 0;
1695 }
1696 
1697 static int cqspi_jh7110_clk_init(struct platform_device *pdev, struct cqspi_st *cqspi)
1698 {
1699 	static struct clk_bulk_data qspiclk[] = {
1700 		{ .id = "apb" },
1701 		{ .id = "ahb" },
1702 	};
1703 
1704 	int ret = 0;
1705 
1706 	ret = devm_clk_bulk_get(&pdev->dev, ARRAY_SIZE(qspiclk), qspiclk);
1707 	if (ret) {
1708 		dev_err(&pdev->dev, "%s: failed to get qspi clocks\n", __func__);
1709 		return ret;
1710 	}
1711 
1712 	cqspi->clks[CLK_QSPI_APB] = qspiclk[0].clk;
1713 	cqspi->clks[CLK_QSPI_AHB] = qspiclk[1].clk;
1714 
1715 	ret = clk_prepare_enable(cqspi->clks[CLK_QSPI_APB]);
1716 	if (ret) {
1717 		dev_err(&pdev->dev, "%s: failed to enable CLK_QSPI_APB\n", __func__);
1718 		return ret;
1719 	}
1720 
1721 	ret = clk_prepare_enable(cqspi->clks[CLK_QSPI_AHB]);
1722 	if (ret) {
1723 		dev_err(&pdev->dev, "%s: failed to enable CLK_QSPI_AHB\n", __func__);
1724 		goto disable_apb_clk;
1725 	}
1726 
1727 	cqspi->is_jh7110 = true;
1728 
1729 	return 0;
1730 
1731 disable_apb_clk:
1732 	clk_disable_unprepare(cqspi->clks[CLK_QSPI_APB]);
1733 
1734 	return ret;
1735 }
1736 
1737 static void cqspi_jh7110_disable_clk(struct platform_device *pdev, struct cqspi_st *cqspi)
1738 {
1739 	clk_disable_unprepare(cqspi->clks[CLK_QSPI_AHB]);
1740 	clk_disable_unprepare(cqspi->clks[CLK_QSPI_APB]);
1741 }
1742 static int cqspi_probe(struct platform_device *pdev)
1743 {
1744 	const struct cqspi_driver_platdata *ddata;
1745 	struct reset_control *rstc, *rstc_ocp, *rstc_ref;
1746 	struct device *dev = &pdev->dev;
1747 	struct spi_controller *host;
1748 	struct resource *res_ahb;
1749 	struct cqspi_st *cqspi;
1750 	int ret;
1751 	int irq;
1752 
1753 	host = devm_spi_alloc_host(&pdev->dev, sizeof(*cqspi));
1754 	if (!host)
1755 		return -ENOMEM;
1756 
1757 	host->mode_bits = SPI_RX_QUAD | SPI_RX_DUAL;
1758 	host->mem_ops = &cqspi_mem_ops;
1759 	host->mem_caps = &cqspi_mem_caps;
1760 	host->dev.of_node = pdev->dev.of_node;
1761 
1762 	cqspi = spi_controller_get_devdata(host);
1763 
1764 	cqspi->pdev = pdev;
1765 	cqspi->host = host;
1766 	cqspi->is_jh7110 = false;
1767 	cqspi->ddata = ddata = of_device_get_match_data(dev);
1768 	platform_set_drvdata(pdev, cqspi);
1769 
1770 	/* Obtain configuration from OF. */
1771 	ret = cqspi_of_get_pdata(cqspi);
1772 	if (ret) {
1773 		dev_err(dev, "Cannot get mandatory OF data.\n");
1774 		return -ENODEV;
1775 	}
1776 
1777 	/* Obtain QSPI clock. */
1778 	cqspi->clk = devm_clk_get(dev, NULL);
1779 	if (IS_ERR(cqspi->clk)) {
1780 		dev_err(dev, "Cannot claim QSPI clock.\n");
1781 		ret = PTR_ERR(cqspi->clk);
1782 		return ret;
1783 	}
1784 
1785 	/* Obtain and remap controller address. */
1786 	cqspi->iobase = devm_platform_ioremap_resource(pdev, 0);
1787 	if (IS_ERR(cqspi->iobase)) {
1788 		dev_err(dev, "Cannot remap controller address.\n");
1789 		ret = PTR_ERR(cqspi->iobase);
1790 		return ret;
1791 	}
1792 
1793 	/* Obtain and remap AHB address. */
1794 	cqspi->ahb_base = devm_platform_get_and_ioremap_resource(pdev, 1, &res_ahb);
1795 	if (IS_ERR(cqspi->ahb_base)) {
1796 		dev_err(dev, "Cannot remap AHB address.\n");
1797 		ret = PTR_ERR(cqspi->ahb_base);
1798 		return ret;
1799 	}
1800 	cqspi->mmap_phys_base = (dma_addr_t)res_ahb->start;
1801 	cqspi->ahb_size = resource_size(res_ahb);
1802 
1803 	init_completion(&cqspi->transfer_complete);
1804 
1805 	/* Obtain IRQ line. */
1806 	irq = platform_get_irq(pdev, 0);
1807 	if (irq < 0)
1808 		return -ENXIO;
1809 
1810 	ret = pm_runtime_set_active(dev);
1811 	if (ret)
1812 		return ret;
1813 
1814 
1815 	ret = clk_prepare_enable(cqspi->clk);
1816 	if (ret) {
1817 		dev_err(dev, "Cannot enable QSPI clock.\n");
1818 		goto probe_clk_failed;
1819 	}
1820 
1821 	/* Obtain QSPI reset control */
1822 	rstc = devm_reset_control_get_optional_exclusive(dev, "qspi");
1823 	if (IS_ERR(rstc)) {
1824 		ret = PTR_ERR(rstc);
1825 		dev_err(dev, "Cannot get QSPI reset.\n");
1826 		goto probe_reset_failed;
1827 	}
1828 
1829 	rstc_ocp = devm_reset_control_get_optional_exclusive(dev, "qspi-ocp");
1830 	if (IS_ERR(rstc_ocp)) {
1831 		ret = PTR_ERR(rstc_ocp);
1832 		dev_err(dev, "Cannot get QSPI OCP reset.\n");
1833 		goto probe_reset_failed;
1834 	}
1835 
1836 	if (of_device_is_compatible(pdev->dev.of_node, "starfive,jh7110-qspi")) {
1837 		rstc_ref = devm_reset_control_get_optional_exclusive(dev, "rstc_ref");
1838 		if (IS_ERR(rstc_ref)) {
1839 			ret = PTR_ERR(rstc_ref);
1840 			dev_err(dev, "Cannot get QSPI REF reset.\n");
1841 			goto probe_reset_failed;
1842 		}
1843 		reset_control_assert(rstc_ref);
1844 		reset_control_deassert(rstc_ref);
1845 	}
1846 
1847 	reset_control_assert(rstc);
1848 	reset_control_deassert(rstc);
1849 
1850 	reset_control_assert(rstc_ocp);
1851 	reset_control_deassert(rstc_ocp);
1852 
1853 	cqspi->master_ref_clk_hz = clk_get_rate(cqspi->clk);
1854 	host->max_speed_hz = cqspi->master_ref_clk_hz;
1855 
1856 	/* write completion is supported by default */
1857 	cqspi->wr_completion = true;
1858 
1859 	if (ddata) {
1860 		if (ddata->quirks & CQSPI_NEEDS_WR_DELAY)
1861 			cqspi->wr_delay = 50 * DIV_ROUND_UP(NSEC_PER_SEC,
1862 						cqspi->master_ref_clk_hz);
1863 		if (ddata->hwcaps_mask & CQSPI_SUPPORTS_OCTAL)
1864 			host->mode_bits |= SPI_RX_OCTAL | SPI_TX_OCTAL;
1865 		if (!(ddata->quirks & CQSPI_DISABLE_DAC_MODE)) {
1866 			cqspi->use_direct_mode = true;
1867 			cqspi->use_direct_mode_wr = true;
1868 		}
1869 		if (ddata->quirks & CQSPI_SUPPORT_EXTERNAL_DMA)
1870 			cqspi->use_dma_read = true;
1871 		if (ddata->quirks & CQSPI_NO_SUPPORT_WR_COMPLETION)
1872 			cqspi->wr_completion = false;
1873 		if (ddata->quirks & CQSPI_SLOW_SRAM)
1874 			cqspi->slow_sram = true;
1875 		if (ddata->quirks & CQSPI_NEEDS_APB_AHB_HAZARD_WAR)
1876 			cqspi->apb_ahb_hazard = true;
1877 
1878 		if (ddata->jh7110_clk_init) {
1879 			ret = cqspi_jh7110_clk_init(pdev, cqspi);
1880 			if (ret)
1881 				goto probe_reset_failed;
1882 		}
1883 
1884 		if (of_device_is_compatible(pdev->dev.of_node,
1885 					    "xlnx,versal-ospi-1.0")) {
1886 			ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1887 			if (ret)
1888 				goto probe_reset_failed;
1889 		}
1890 	}
1891 
1892 	ret = devm_request_irq(dev, irq, cqspi_irq_handler, 0,
1893 			       pdev->name, cqspi);
1894 	if (ret) {
1895 		dev_err(dev, "Cannot request IRQ.\n");
1896 		goto probe_reset_failed;
1897 	}
1898 
1899 	cqspi_wait_idle(cqspi);
1900 	cqspi_controller_enable(cqspi, 0);
1901 	cqspi_controller_detect_fifo_depth(cqspi);
1902 	cqspi_controller_init(cqspi);
1903 	cqspi_controller_enable(cqspi, 1);
1904 	cqspi->current_cs = -1;
1905 	cqspi->sclk = 0;
1906 
1907 	ret = cqspi_setup_flash(cqspi);
1908 	if (ret) {
1909 		dev_err(dev, "failed to setup flash parameters %d\n", ret);
1910 		goto probe_setup_failed;
1911 	}
1912 
1913 	host->num_chipselect = cqspi->num_chipselect;
1914 
1915 	if (cqspi->use_direct_mode) {
1916 		ret = cqspi_request_mmap_dma(cqspi);
1917 		if (ret == -EPROBE_DEFER)
1918 			goto probe_setup_failed;
1919 	}
1920 
1921 	ret = devm_pm_runtime_enable(dev);
1922 	if (ret) {
1923 		if (cqspi->rx_chan)
1924 			dma_release_channel(cqspi->rx_chan);
1925 		goto probe_setup_failed;
1926 	}
1927 
1928 	pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT);
1929 	pm_runtime_use_autosuspend(dev);
1930 	pm_runtime_get_noresume(dev);
1931 
1932 	ret = spi_register_controller(host);
1933 	if (ret) {
1934 		dev_err(&pdev->dev, "failed to register SPI ctlr %d\n", ret);
1935 		goto probe_setup_failed;
1936 	}
1937 
1938 	pm_runtime_mark_last_busy(dev);
1939 	pm_runtime_put_autosuspend(dev);
1940 
1941 	return 0;
1942 probe_setup_failed:
1943 	cqspi_controller_enable(cqspi, 0);
1944 probe_reset_failed:
1945 	if (cqspi->is_jh7110)
1946 		cqspi_jh7110_disable_clk(pdev, cqspi);
1947 	clk_disable_unprepare(cqspi->clk);
1948 probe_clk_failed:
1949 	return ret;
1950 }
1951 
1952 static void cqspi_remove(struct platform_device *pdev)
1953 {
1954 	struct cqspi_st *cqspi = platform_get_drvdata(pdev);
1955 
1956 	spi_unregister_controller(cqspi->host);
1957 	cqspi_controller_enable(cqspi, 0);
1958 
1959 	if (cqspi->rx_chan)
1960 		dma_release_channel(cqspi->rx_chan);
1961 
1962 	clk_disable_unprepare(cqspi->clk);
1963 
1964 	if (cqspi->is_jh7110)
1965 		cqspi_jh7110_disable_clk(pdev, cqspi);
1966 
1967 	pm_runtime_put_sync(&pdev->dev);
1968 	pm_runtime_disable(&pdev->dev);
1969 }
1970 
1971 static int cqspi_runtime_suspend(struct device *dev)
1972 {
1973 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
1974 
1975 	cqspi_controller_enable(cqspi, 0);
1976 	clk_disable_unprepare(cqspi->clk);
1977 	return 0;
1978 }
1979 
1980 static int cqspi_runtime_resume(struct device *dev)
1981 {
1982 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
1983 
1984 	clk_prepare_enable(cqspi->clk);
1985 	cqspi_wait_idle(cqspi);
1986 	cqspi_controller_enable(cqspi, 0);
1987 	cqspi_controller_init(cqspi);
1988 	cqspi_controller_enable(cqspi, 1);
1989 
1990 	cqspi->current_cs = -1;
1991 	cqspi->sclk = 0;
1992 	return 0;
1993 }
1994 
1995 static int cqspi_suspend(struct device *dev)
1996 {
1997 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
1998 	int ret;
1999 
2000 	ret = spi_controller_suspend(cqspi->host);
2001 	if (ret)
2002 		return ret;
2003 
2004 	return pm_runtime_force_suspend(dev);
2005 }
2006 
2007 static int cqspi_resume(struct device *dev)
2008 {
2009 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
2010 	int ret;
2011 
2012 	ret = pm_runtime_force_resume(dev);
2013 	if (ret) {
2014 		dev_err(dev, "pm_runtime_force_resume failed on resume\n");
2015 		return ret;
2016 	}
2017 
2018 	return spi_controller_resume(cqspi->host);
2019 }
2020 
2021 static const struct dev_pm_ops cqspi_dev_pm_ops = {
2022 	RUNTIME_PM_OPS(cqspi_runtime_suspend, cqspi_runtime_resume, NULL)
2023 	SYSTEM_SLEEP_PM_OPS(cqspi_suspend, cqspi_resume)
2024 };
2025 
2026 static const struct cqspi_driver_platdata cdns_qspi = {
2027 	.quirks = CQSPI_DISABLE_DAC_MODE,
2028 };
2029 
2030 static const struct cqspi_driver_platdata k2g_qspi = {
2031 	.quirks = CQSPI_NEEDS_WR_DELAY,
2032 };
2033 
2034 static const struct cqspi_driver_platdata am654_ospi = {
2035 	.hwcaps_mask = CQSPI_SUPPORTS_OCTAL,
2036 	.quirks = CQSPI_NEEDS_WR_DELAY,
2037 };
2038 
2039 static const struct cqspi_driver_platdata intel_lgm_qspi = {
2040 	.quirks = CQSPI_DISABLE_DAC_MODE,
2041 };
2042 
2043 static const struct cqspi_driver_platdata socfpga_qspi = {
2044 	.quirks = CQSPI_DISABLE_DAC_MODE
2045 			| CQSPI_NO_SUPPORT_WR_COMPLETION
2046 			| CQSPI_SLOW_SRAM,
2047 };
2048 
2049 static const struct cqspi_driver_platdata versal_ospi = {
2050 	.hwcaps_mask = CQSPI_SUPPORTS_OCTAL,
2051 	.quirks = CQSPI_DISABLE_DAC_MODE | CQSPI_SUPPORT_EXTERNAL_DMA,
2052 	.indirect_read_dma = cqspi_versal_indirect_read_dma,
2053 	.get_dma_status = cqspi_get_versal_dma_status,
2054 };
2055 
2056 static const struct cqspi_driver_platdata jh7110_qspi = {
2057 	.quirks = CQSPI_DISABLE_DAC_MODE,
2058 	.jh7110_clk_init = cqspi_jh7110_clk_init,
2059 };
2060 
2061 static const struct cqspi_driver_platdata pensando_cdns_qspi = {
2062 	.quirks = CQSPI_NEEDS_APB_AHB_HAZARD_WAR | CQSPI_DISABLE_DAC_MODE,
2063 };
2064 
2065 static const struct cqspi_driver_platdata mobileye_eyeq5_ospi = {
2066 	.hwcaps_mask = CQSPI_SUPPORTS_OCTAL,
2067 	.quirks = CQSPI_DISABLE_DAC_MODE | CQSPI_NO_SUPPORT_WR_COMPLETION |
2068 			CQSPI_RD_NO_IRQ,
2069 };
2070 
2071 static const struct of_device_id cqspi_dt_ids[] = {
2072 	{
2073 		.compatible = "cdns,qspi-nor",
2074 		.data = &cdns_qspi,
2075 	},
2076 	{
2077 		.compatible = "ti,k2g-qspi",
2078 		.data = &k2g_qspi,
2079 	},
2080 	{
2081 		.compatible = "ti,am654-ospi",
2082 		.data = &am654_ospi,
2083 	},
2084 	{
2085 		.compatible = "intel,lgm-qspi",
2086 		.data = &intel_lgm_qspi,
2087 	},
2088 	{
2089 		.compatible = "xlnx,versal-ospi-1.0",
2090 		.data = &versal_ospi,
2091 	},
2092 	{
2093 		.compatible = "intel,socfpga-qspi",
2094 		.data = &socfpga_qspi,
2095 	},
2096 	{
2097 		.compatible = "starfive,jh7110-qspi",
2098 		.data = &jh7110_qspi,
2099 	},
2100 	{
2101 		.compatible = "amd,pensando-elba-qspi",
2102 		.data = &pensando_cdns_qspi,
2103 	},
2104 	{
2105 		.compatible = "mobileye,eyeq5-ospi",
2106 		.data = &mobileye_eyeq5_ospi,
2107 	},
2108 	{ /* end of table */ }
2109 };
2110 
2111 MODULE_DEVICE_TABLE(of, cqspi_dt_ids);
2112 
2113 static struct platform_driver cqspi_platform_driver = {
2114 	.probe = cqspi_probe,
2115 	.remove_new = cqspi_remove,
2116 	.driver = {
2117 		.name = CQSPI_NAME,
2118 		.pm = pm_ptr(&cqspi_dev_pm_ops),
2119 		.of_match_table = cqspi_dt_ids,
2120 	},
2121 };
2122 
2123 module_platform_driver(cqspi_platform_driver);
2124 
2125 MODULE_DESCRIPTION("Cadence QSPI Controller Driver");
2126 MODULE_LICENSE("GPL v2");
2127 MODULE_ALIAS("platform:" CQSPI_NAME);
2128 MODULE_AUTHOR("Ley Foon Tan <lftan@altera.com>");
2129 MODULE_AUTHOR("Graham Moore <grmoore@opensource.altera.com>");
2130 MODULE_AUTHOR("Vadivel Murugan R <vadivel.muruganx.ramuthevar@intel.com>");
2131 MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
2132 MODULE_AUTHOR("Pratyush Yadav <p.yadav@ti.com>");
2133