xref: /linux/drivers/dma/fsl-edma-common.h (revision 4b132aacb0768ac1e652cf517097ea6f237214b9)
1 /* SPDX-License-Identifier: GPL-2.0+ */
2 /*
3  * Copyright 2013-2014 Freescale Semiconductor, Inc.
4  * Copyright 2018 Angelo Dureghello <angelo@sysam.it>
5  */
6 #ifndef _FSL_EDMA_COMMON_H_
7 #define _FSL_EDMA_COMMON_H_
8 
9 #include <linux/dma-direction.h>
10 #include <linux/platform_device.h>
11 #include "virt-dma.h"
12 
13 #define EDMA_CR_EDBG		BIT(1)
14 #define EDMA_CR_ERCA		BIT(2)
15 #define EDMA_CR_ERGA		BIT(3)
16 #define EDMA_CR_HOE		BIT(4)
17 #define EDMA_CR_HALT		BIT(5)
18 #define EDMA_CR_CLM		BIT(6)
19 #define EDMA_CR_EMLM		BIT(7)
20 #define EDMA_CR_ECX		BIT(16)
21 #define EDMA_CR_CX		BIT(17)
22 
23 #define EDMA_SEEI_SEEI(x)	((x) & GENMASK(4, 0))
24 #define EDMA_CEEI_CEEI(x)	((x) & GENMASK(4, 0))
25 #define EDMA_CINT_CINT(x)	((x) & GENMASK(4, 0))
26 #define EDMA_CERR_CERR(x)	((x) & GENMASK(4, 0))
27 
28 #define EDMA_TCD_ATTR_DSIZE(x)		(((x) & GENMASK(2, 0)))
29 #define EDMA_TCD_ATTR_DMOD(x)		(((x) & GENMASK(4, 0)) << 3)
30 #define EDMA_TCD_ATTR_SSIZE(x)		(((x) & GENMASK(2, 0)) << 8)
31 #define EDMA_TCD_ATTR_SMOD(x)		(((x) & GENMASK(4, 0)) << 11)
32 
33 #define EDMA_TCD_ITER_MASK		GENMASK(14, 0)
34 #define EDMA_TCD_CITER_CITER(x)		((x) & EDMA_TCD_ITER_MASK)
35 #define EDMA_TCD_BITER_BITER(x)		((x) & EDMA_TCD_ITER_MASK)
36 
37 #define EDMA_TCD_CSR_START		BIT(0)
38 #define EDMA_TCD_CSR_INT_MAJOR		BIT(1)
39 #define EDMA_TCD_CSR_INT_HALF		BIT(2)
40 #define EDMA_TCD_CSR_D_REQ		BIT(3)
41 #define EDMA_TCD_CSR_E_SG		BIT(4)
42 #define EDMA_TCD_CSR_E_LINK		BIT(5)
43 #define EDMA_TCD_CSR_ACTIVE		BIT(6)
44 #define EDMA_TCD_CSR_DONE		BIT(7)
45 
46 #define EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(x) ((x) & GENMASK(9, 0))
47 #define EDMA_V3_TCD_NBYTES_MLOFF(x)        (x << 10)
48 #define EDMA_V3_TCD_NBYTES_DMLOE           (1 << 30)
49 #define EDMA_V3_TCD_NBYTES_SMLOE           (1 << 31)
50 
51 #define EDMAMUX_CHCFG_DIS		0x0
52 #define EDMAMUX_CHCFG_ENBL		0x80
53 #define EDMAMUX_CHCFG_SOURCE(n)		((n) & 0x3F)
54 
55 #define DMAMUX_NR	2
56 
57 #define EDMA_TCD                0x1000
58 
59 #define FSL_EDMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
60 				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
61 				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
62 				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
63 
64 #define EDMA_V3_CH_SBR_RD          BIT(22)
65 #define EDMA_V3_CH_SBR_WR          BIT(21)
66 #define EDMA_V3_CH_CSR_ERQ         BIT(0)
67 #define EDMA_V3_CH_CSR_EARQ        BIT(1)
68 #define EDMA_V3_CH_CSR_EEI         BIT(2)
69 #define EDMA_V3_CH_CSR_DONE        BIT(30)
70 #define EDMA_V3_CH_CSR_ACTIVE      BIT(31)
71 
72 enum fsl_edma_pm_state {
73 	RUNNING = 0,
74 	SUSPENDED,
75 };
76 
77 struct fsl_edma_hw_tcd {
78 	__le32	saddr;
79 	__le16	soff;
80 	__le16	attr;
81 	__le32	nbytes;
82 	__le32	slast;
83 	__le32	daddr;
84 	__le16	doff;
85 	__le16	citer;
86 	__le32	dlast_sga;
87 	__le16	csr;
88 	__le16	biter;
89 };
90 
91 struct fsl_edma_hw_tcd64 {
92 	__le64  saddr;
93 	__le16  soff;
94 	__le16  attr;
95 	__le32  nbytes;
96 	__le64  slast;
97 	__le64  daddr;
98 	__le64  dlast_sga;
99 	__le16  doff;
100 	__le16  citer;
101 	__le16  csr;
102 	__le16  biter;
103 } __packed;
104 
105 struct fsl_edma3_ch_reg {
106 	__le32	ch_csr;
107 	__le32	ch_es;
108 	__le32	ch_int;
109 	__le32	ch_sbr;
110 	__le32	ch_pri;
111 	__le32	ch_mux;
112 	__le32  ch_mattr; /* edma4, reserved for edma3 */
113 	__le32  ch_reserved;
114 	union {
115 		struct fsl_edma_hw_tcd tcd;
116 		struct fsl_edma_hw_tcd64 tcd64;
117 	};
118 } __packed;
119 
120 /*
121  * These are iomem pointers, for both v32 and v64.
122  */
123 struct edma_regs {
124 	void __iomem *cr;
125 	void __iomem *es;
126 	void __iomem *erqh;
127 	void __iomem *erql;	/* aka erq on v32 */
128 	void __iomem *eeih;
129 	void __iomem *eeil;	/* aka eei on v32 */
130 	void __iomem *seei;
131 	void __iomem *ceei;
132 	void __iomem *serq;
133 	void __iomem *cerq;
134 	void __iomem *cint;
135 	void __iomem *cerr;
136 	void __iomem *ssrt;
137 	void __iomem *cdne;
138 	void __iomem *inth;
139 	void __iomem *intl;
140 	void __iomem *errh;
141 	void __iomem *errl;
142 };
143 
144 struct fsl_edma_sw_tcd {
145 	dma_addr_t			ptcd;
146 	void				*vtcd;
147 };
148 
149 struct fsl_edma_chan {
150 	struct virt_dma_chan		vchan;
151 	enum dma_status			status;
152 	enum fsl_edma_pm_state		pm_state;
153 	struct fsl_edma_engine		*edma;
154 	struct fsl_edma_desc		*edesc;
155 	struct dma_slave_config		cfg;
156 	u32				attr;
157 	bool                            is_sw;
158 	struct dma_pool			*tcd_pool;
159 	dma_addr_t			dma_dev_addr;
160 	u32				dma_dev_size;
161 	enum dma_data_direction		dma_dir;
162 	char				chan_name[32];
163 	void __iomem			*tcd;
164 	void __iomem			*mux_addr;
165 	u32				real_count;
166 	struct work_struct		issue_worker;
167 	struct platform_device		*pdev;
168 	struct device			*pd_dev;
169 	u32				srcid;
170 	struct clk			*clk;
171 	int                             priority;
172 	int				hw_chanid;
173 	int				txirq;
174 	irqreturn_t			(*irq_handler)(int irq, void *dev_id);
175 	bool				is_rxchan;
176 	bool				is_remote;
177 	bool				is_multi_fifo;
178 };
179 
180 struct fsl_edma_desc {
181 	struct virt_dma_desc		vdesc;
182 	struct fsl_edma_chan		*echan;
183 	bool				iscyclic;
184 	enum dma_transfer_direction	dirn;
185 	unsigned int			n_tcds;
186 	struct fsl_edma_sw_tcd		tcd[];
187 };
188 
189 #define FSL_EDMA_DRV_HAS_DMACLK		BIT(0)
190 #define FSL_EDMA_DRV_MUX_SWAP		BIT(1)
191 #define FSL_EDMA_DRV_CONFIG32		BIT(2)
192 #define FSL_EDMA_DRV_WRAP_IO		BIT(3)
193 #define FSL_EDMA_DRV_EDMA64		BIT(4)
194 #define FSL_EDMA_DRV_HAS_PD		BIT(5)
195 #define FSL_EDMA_DRV_HAS_CHCLK		BIT(6)
196 #define FSL_EDMA_DRV_HAS_CHMUX		BIT(7)
197 #define FSL_EDMA_DRV_MEM_REMOTE		BIT(8)
198 /* control and status register is in tcd address space, edma3 reg layout */
199 #define FSL_EDMA_DRV_SPLIT_REG		BIT(9)
200 #define FSL_EDMA_DRV_BUS_8BYTE		BIT(10)
201 #define FSL_EDMA_DRV_DEV_TO_DEV		BIT(11)
202 #define FSL_EDMA_DRV_ALIGN_64BYTE	BIT(12)
203 /* Need clean CHn_CSR DONE before enable TCD's ESG */
204 #define FSL_EDMA_DRV_CLEAR_DONE_E_SG	BIT(13)
205 /* Need clean CHn_CSR DONE before enable TCD's MAJORELINK */
206 #define FSL_EDMA_DRV_CLEAR_DONE_E_LINK	BIT(14)
207 #define FSL_EDMA_DRV_TCD64		BIT(15)
208 
209 #define FSL_EDMA_DRV_EDMA3	(FSL_EDMA_DRV_SPLIT_REG |	\
210 				 FSL_EDMA_DRV_BUS_8BYTE |	\
211 				 FSL_EDMA_DRV_DEV_TO_DEV |	\
212 				 FSL_EDMA_DRV_ALIGN_64BYTE |	\
213 				 FSL_EDMA_DRV_CLEAR_DONE_E_SG |	\
214 				 FSL_EDMA_DRV_CLEAR_DONE_E_LINK)
215 
216 #define FSL_EDMA_DRV_EDMA4	(FSL_EDMA_DRV_SPLIT_REG |	\
217 				 FSL_EDMA_DRV_BUS_8BYTE |	\
218 				 FSL_EDMA_DRV_DEV_TO_DEV |	\
219 				 FSL_EDMA_DRV_ALIGN_64BYTE |	\
220 				 FSL_EDMA_DRV_CLEAR_DONE_E_LINK)
221 
222 struct fsl_edma_drvdata {
223 	u32			dmamuxs; /* only used before v3 */
224 	u32			chreg_off;
225 	u32			chreg_space_sz;
226 	u32			flags;
227 	u32			mux_off;	/* channel mux register offset */
228 	u32			mux_skip;	/* how much skip for each channel */
229 	int			(*setup_irq)(struct platform_device *pdev,
230 					     struct fsl_edma_engine *fsl_edma);
231 };
232 
233 struct fsl_edma_engine {
234 	struct dma_device	dma_dev;
235 	void __iomem		*membase;
236 	void __iomem		*muxbase[DMAMUX_NR];
237 	struct clk		*muxclk[DMAMUX_NR];
238 	struct clk		*dmaclk;
239 	struct mutex		fsl_edma_mutex;
240 	const struct fsl_edma_drvdata *drvdata;
241 	u32			n_chans;
242 	int			txirq;
243 	int			errirq;
244 	bool			big_endian;
245 	struct edma_regs	regs;
246 	u64			chan_masked;
247 	struct fsl_edma_chan	chans[] __counted_by(n_chans);
248 };
249 
250 static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan)
251 {
252 	return fsl_chan->edma->drvdata->flags;
253 }
254 
255 #define edma_read_tcdreg_c(chan, _tcd,  __name)				\
256 _Generic(((_tcd)->__name),						\
257 	__iomem __le64 : edma_readq(chan->edma, &(_tcd)->__name),		\
258 	__iomem __le32 : edma_readl(chan->edma, &(_tcd)->__name),		\
259 	__iomem __le16 : edma_readw(chan->edma, &(_tcd)->__name)		\
260 	)
261 
262 #define edma_read_tcdreg(chan, __name)								\
263 ((fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) ?						\
264 	edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd64 __iomem *)chan->tcd), __name) :	\
265 	edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd __iomem *)chan->tcd), __name)		\
266 )
267 
268 #define edma_write_tcdreg_c(chan, _tcd, _val, __name)					\
269 _Generic((_tcd->__name),								\
270 	__iomem __le64 : edma_writeq(chan->edma, (u64 __force)(_val), &_tcd->__name),	\
271 	__iomem __le32 : edma_writel(chan->edma, (u32 __force)(_val), &_tcd->__name),	\
272 	__iomem __le16 : edma_writew(chan->edma, (u16 __force)(_val), &_tcd->__name),	\
273 	__iomem u8 : edma_writeb(chan->edma, _val, &_tcd->__name)			\
274 	)
275 
276 #define edma_write_tcdreg(chan, val, __name)							   \
277 do {												   \
278 	struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \
279 	struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd;	   \
280 												   \
281 	if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64)					   \
282 		edma_write_tcdreg_c(chan, tcd64_r, val, __name);				   \
283 	else											   \
284 		edma_write_tcdreg_c(chan, tcd_r, val, __name);					   \
285 } while (0)
286 
287 #define edma_cp_tcd_to_reg(chan, __tcd, __name)							   \
288 do {	\
289 	struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \
290 	struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd;	   \
291 	struct fsl_edma_hw_tcd64 *tcd64_m = (struct fsl_edma_hw_tcd64 *)__tcd;			   \
292 	struct fsl_edma_hw_tcd *tcd_m = (struct fsl_edma_hw_tcd *)__tcd;			   \
293 												   \
294 	if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64)					   \
295 		edma_write_tcdreg_c(chan, tcd64_r,  tcd64_m->__name, __name);			   \
296 	else											   \
297 		edma_write_tcdreg_c(chan, tcd_r, tcd_m->__name, __name);			   \
298 } while (0)
299 
300 #define edma_readl_chreg(chan, __name)				\
301 	edma_readl(chan->edma,					\
302 		   (void __iomem *)&(container_of(((__force void *)chan->tcd),\
303 						  struct fsl_edma3_ch_reg, tcd)->__name))
304 
305 #define edma_writel_chreg(chan, val,  __name)			\
306 	edma_writel(chan->edma, val,				\
307 		   (void __iomem *)&(container_of(((__force void *)chan->tcd),\
308 						  struct fsl_edma3_ch_reg, tcd)->__name))
309 
310 #define fsl_edma_get_tcd(_chan, _tcd, _field)			\
311 (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? (((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \
312 						 (((struct fsl_edma_hw_tcd *)_tcd)->_field))
313 
314 #define fsl_edma_le_to_cpu(x)						\
315 _Generic((x),								\
316 	__le64 : le64_to_cpu((x)),					\
317 	__le32 : le32_to_cpu((x)),					\
318 	__le16 : le16_to_cpu((x))					\
319 )
320 
321 #define fsl_edma_get_tcd_to_cpu(_chan, _tcd, _field)				\
322 (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ?				\
323 	fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd64 *)_tcd)->_field) :	\
324 	fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd *)_tcd)->_field))
325 
326 #define fsl_edma_set_tcd_to_le_c(_tcd, _val, _field)					\
327 _Generic(((_tcd)->_field),								\
328 	__le64 : (_tcd)->_field = cpu_to_le64(_val),					\
329 	__le32 : (_tcd)->_field = cpu_to_le32(_val),					\
330 	__le16 : (_tcd)->_field = cpu_to_le16(_val)					\
331 )
332 
333 #define fsl_edma_set_tcd_to_le(_chan, _tcd, _val, _field)	\
334 do {								\
335 	if (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64)	\
336 		fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd64 *)_tcd, _val, _field);	\
337 	else											\
338 		fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd *)_tcd, _val, _field);		\
339 } while (0)
340 
341 /* Need after struct defination */
342 #include "fsl-edma-trace.h"
343 
344 /*
345  * R/W functions for big- or little-endian registers:
346  * The eDMA controller's endian is independent of the CPU core's endian.
347  * For the big-endian IP module, the offset for 8-bit or 16-bit registers
348  * should also be swapped opposite to that in little-endian IP.
349  */
350 static inline u64 edma_readq(struct fsl_edma_engine *edma, void __iomem *addr)
351 {
352 	u64 l, h;
353 
354 	if (edma->big_endian) {
355 		l = ioread32be(addr);
356 		h = ioread32be(addr + 4);
357 	} else {
358 		l = ioread32(addr);
359 		h = ioread32(addr + 4);
360 	}
361 
362 	trace_edma_readl(edma, addr, l);
363 	trace_edma_readl(edma, addr + 4, h);
364 
365 	return (h << 32) | l;
366 }
367 
368 static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr)
369 {
370 	u32 val;
371 
372 	if (edma->big_endian)
373 		val = ioread32be(addr);
374 	else
375 		val = ioread32(addr);
376 
377 	trace_edma_readl(edma, addr, val);
378 
379 	return val;
380 }
381 
382 static inline u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr)
383 {
384 	u16 val;
385 
386 	if (edma->big_endian)
387 		val = ioread16be(addr);
388 	else
389 		val = ioread16(addr);
390 
391 	trace_edma_readw(edma, addr, val);
392 
393 	return val;
394 }
395 
396 static inline void edma_writeb(struct fsl_edma_engine *edma,
397 			       u8 val, void __iomem *addr)
398 {
399 	/* swap the reg offset for these in big-endian mode */
400 	if (edma->big_endian)
401 		iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3));
402 	else
403 		iowrite8(val, addr);
404 
405 	trace_edma_writeb(edma, addr, val);
406 }
407 
408 static inline void edma_writew(struct fsl_edma_engine *edma,
409 			       u16 val, void __iomem *addr)
410 {
411 	/* swap the reg offset for these in big-endian mode */
412 	if (edma->big_endian)
413 		iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2));
414 	else
415 		iowrite16(val, addr);
416 
417 	trace_edma_writew(edma, addr, val);
418 }
419 
420 static inline void edma_writel(struct fsl_edma_engine *edma,
421 			       u32 val, void __iomem *addr)
422 {
423 	if (edma->big_endian)
424 		iowrite32be(val, addr);
425 	else
426 		iowrite32(val, addr);
427 
428 	trace_edma_writel(edma, addr, val);
429 }
430 
431 static inline void edma_writeq(struct fsl_edma_engine *edma,
432 			       u64 val, void __iomem *addr)
433 {
434 	if (edma->big_endian) {
435 		iowrite32be(val & 0xFFFFFFFF, addr);
436 		iowrite32be(val >> 32, addr + 4);
437 	} else {
438 		iowrite32(val & 0xFFFFFFFF, addr);
439 		iowrite32(val >> 32, addr + 4);
440 	}
441 
442 	trace_edma_writel(edma, addr, val & 0xFFFFFFFF);
443 	trace_edma_writel(edma, addr + 4, val >> 32);
444 }
445 
446 static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan)
447 {
448 	return container_of(chan, struct fsl_edma_chan, vchan.chan);
449 }
450 
451 static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd)
452 {
453 	return container_of(vd, struct fsl_edma_desc, vdesc);
454 }
455 
456 static inline void fsl_edma_err_chan_handler(struct fsl_edma_chan *fsl_chan)
457 {
458 	fsl_chan->status = DMA_ERROR;
459 }
460 
461 void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan);
462 void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan);
463 void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan,
464 			unsigned int slot, bool enable);
465 void fsl_edma_free_desc(struct virt_dma_desc *vdesc);
466 int fsl_edma_terminate_all(struct dma_chan *chan);
467 int fsl_edma_pause(struct dma_chan *chan);
468 int fsl_edma_resume(struct dma_chan *chan);
469 int fsl_edma_slave_config(struct dma_chan *chan,
470 				 struct dma_slave_config *cfg);
471 enum dma_status fsl_edma_tx_status(struct dma_chan *chan,
472 		dma_cookie_t cookie, struct dma_tx_state *txstate);
473 struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic(
474 		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
475 		size_t period_len, enum dma_transfer_direction direction,
476 		unsigned long flags);
477 struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(
478 		struct dma_chan *chan, struct scatterlist *sgl,
479 		unsigned int sg_len, enum dma_transfer_direction direction,
480 		unsigned long flags, void *context);
481 struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(
482 		struct dma_chan *chan, dma_addr_t dma_dst, dma_addr_t dma_src,
483 		size_t len, unsigned long flags);
484 void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan);
485 void fsl_edma_issue_pending(struct dma_chan *chan);
486 int fsl_edma_alloc_chan_resources(struct dma_chan *chan);
487 void fsl_edma_free_chan_resources(struct dma_chan *chan);
488 void fsl_edma_cleanup_vchan(struct dma_device *dmadev);
489 void fsl_edma_setup_regs(struct fsl_edma_engine *edma);
490 
491 #endif /* _FSL_EDMA_COMMON_H_ */
492