xref: /linux/drivers/dma/ti/omap-dma.c (revision 922842a3bfbeff64dfebe7f01ce1f2ab01e4509d)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * OMAP DMAengine support
4  */
5 #include <linux/cpu_pm.h>
6 #include <linux/delay.h>
7 #include <linux/dmaengine.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/dmapool.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/interrupt.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/omap-dma.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/of.h>
20 #include <linux/of_dma.h>
21 
22 #include "../virt-dma.h"
23 
24 #define OMAP_SDMA_REQUESTS	127
25 #define OMAP_SDMA_CHANNELS	32
26 
27 struct omap_dma_config {
28 	int lch_end;
29 	unsigned int rw_priority:1;
30 	unsigned int needs_busy_check:1;
31 	unsigned int may_lose_context:1;
32 	unsigned int needs_lch_clear:1;
33 };
34 
35 struct omap_dma_context {
36 	u32 irqenable_l0;
37 	u32 irqenable_l1;
38 	u32 ocp_sysconfig;
39 	u32 gcr;
40 };
41 
42 struct omap_dmadev {
43 	struct dma_device ddev;
44 	spinlock_t lock;
45 	void __iomem *base;
46 	const struct omap_dma_reg *reg_map;
47 	struct omap_system_dma_plat_info *plat;
48 	const struct omap_dma_config *cfg;
49 	struct notifier_block nb;
50 	struct omap_dma_context context;
51 	int lch_count;
52 	DECLARE_BITMAP(lch_bitmap, OMAP_SDMA_CHANNELS);
53 	struct mutex lch_lock;		/* for assigning logical channels */
54 	bool legacy;
55 	bool ll123_supported;
56 	struct dma_pool *desc_pool;
57 	unsigned dma_requests;
58 	spinlock_t irq_lock;
59 	uint32_t irq_enable_mask;
60 	struct omap_chan **lch_map;
61 };
62 
63 struct omap_chan {
64 	struct virt_dma_chan vc;
65 	void __iomem *channel_base;
66 	const struct omap_dma_reg *reg_map;
67 	uint32_t ccr;
68 
69 	struct dma_slave_config	cfg;
70 	unsigned dma_sig;
71 	bool cyclic;
72 	bool paused;
73 	bool running;
74 
75 	int dma_ch;
76 	struct omap_desc *desc;
77 	unsigned sgidx;
78 };
79 
80 #define DESC_NXT_SV_REFRESH	(0x1 << 24)
81 #define DESC_NXT_SV_REUSE	(0x2 << 24)
82 #define DESC_NXT_DV_REFRESH	(0x1 << 26)
83 #define DESC_NXT_DV_REUSE	(0x2 << 26)
84 #define DESC_NTYPE_TYPE2	(0x2 << 29)
85 
86 /* Type 2 descriptor with Source or Destination address update */
87 struct omap_type2_desc {
88 	uint32_t next_desc;
89 	uint32_t en;
90 	uint32_t addr; /* src or dst */
91 	uint16_t fn;
92 	uint16_t cicr;
93 	int16_t cdei;
94 	int16_t csei;
95 	int32_t cdfi;
96 	int32_t csfi;
97 } __packed;
98 
99 struct omap_sg {
100 	dma_addr_t addr;
101 	uint32_t en;		/* number of elements (24-bit) */
102 	uint32_t fn;		/* number of frames (16-bit) */
103 	int32_t fi;		/* for double indexing */
104 	int16_t ei;		/* for double indexing */
105 
106 	/* Linked list */
107 	struct omap_type2_desc *t2_desc;
108 	dma_addr_t t2_desc_paddr;
109 };
110 
111 struct omap_desc {
112 	struct virt_dma_desc vd;
113 	bool using_ll;
114 	enum dma_transfer_direction dir;
115 	dma_addr_t dev_addr;
116 	bool polled;
117 
118 	int32_t fi;		/* for OMAP_DMA_SYNC_PACKET / double indexing */
119 	int16_t ei;		/* for double indexing */
120 	uint8_t es;		/* CSDP_DATA_TYPE_xxx */
121 	uint32_t ccr;		/* CCR value */
122 	uint16_t clnk_ctrl;	/* CLNK_CTRL value */
123 	uint16_t cicr;		/* CICR value */
124 	uint32_t csdp;		/* CSDP value */
125 
126 	unsigned sglen;
127 	struct omap_sg sg[] __counted_by(sglen);
128 };
129 
130 enum {
131 	CAPS_0_SUPPORT_LL123	= BIT(20),	/* Linked List type1/2/3 */
132 	CAPS_0_SUPPORT_LL4	= BIT(21),	/* Linked List type4 */
133 
134 	CCR_FS			= BIT(5),
135 	CCR_READ_PRIORITY	= BIT(6),
136 	CCR_ENABLE		= BIT(7),
137 	CCR_AUTO_INIT		= BIT(8),	/* OMAP1 only */
138 	CCR_REPEAT		= BIT(9),	/* OMAP1 only */
139 	CCR_OMAP31_DISABLE	= BIT(10),	/* OMAP1 only */
140 	CCR_SUSPEND_SENSITIVE	= BIT(8),	/* OMAP2+ only */
141 	CCR_RD_ACTIVE		= BIT(9),	/* OMAP2+ only */
142 	CCR_WR_ACTIVE		= BIT(10),	/* OMAP2+ only */
143 	CCR_SRC_AMODE_CONSTANT	= 0 << 12,
144 	CCR_SRC_AMODE_POSTINC	= 1 << 12,
145 	CCR_SRC_AMODE_SGLIDX	= 2 << 12,
146 	CCR_SRC_AMODE_DBLIDX	= 3 << 12,
147 	CCR_DST_AMODE_CONSTANT	= 0 << 14,
148 	CCR_DST_AMODE_POSTINC	= 1 << 14,
149 	CCR_DST_AMODE_SGLIDX	= 2 << 14,
150 	CCR_DST_AMODE_DBLIDX	= 3 << 14,
151 	CCR_CONSTANT_FILL	= BIT(16),
152 	CCR_TRANSPARENT_COPY	= BIT(17),
153 	CCR_BS			= BIT(18),
154 	CCR_SUPERVISOR		= BIT(22),
155 	CCR_PREFETCH		= BIT(23),
156 	CCR_TRIGGER_SRC		= BIT(24),
157 	CCR_BUFFERING_DISABLE	= BIT(25),
158 	CCR_WRITE_PRIORITY	= BIT(26),
159 	CCR_SYNC_ELEMENT	= 0,
160 	CCR_SYNC_FRAME		= CCR_FS,
161 	CCR_SYNC_BLOCK		= CCR_BS,
162 	CCR_SYNC_PACKET		= CCR_BS | CCR_FS,
163 
164 	CSDP_DATA_TYPE_8	= 0,
165 	CSDP_DATA_TYPE_16	= 1,
166 	CSDP_DATA_TYPE_32	= 2,
167 	CSDP_SRC_PORT_EMIFF	= 0 << 2, /* OMAP1 only */
168 	CSDP_SRC_PORT_EMIFS	= 1 << 2, /* OMAP1 only */
169 	CSDP_SRC_PORT_OCP_T1	= 2 << 2, /* OMAP1 only */
170 	CSDP_SRC_PORT_TIPB	= 3 << 2, /* OMAP1 only */
171 	CSDP_SRC_PORT_OCP_T2	= 4 << 2, /* OMAP1 only */
172 	CSDP_SRC_PORT_MPUI	= 5 << 2, /* OMAP1 only */
173 	CSDP_SRC_PACKED		= BIT(6),
174 	CSDP_SRC_BURST_1	= 0 << 7,
175 	CSDP_SRC_BURST_16	= 1 << 7,
176 	CSDP_SRC_BURST_32	= 2 << 7,
177 	CSDP_SRC_BURST_64	= 3 << 7,
178 	CSDP_DST_PORT_EMIFF	= 0 << 9, /* OMAP1 only */
179 	CSDP_DST_PORT_EMIFS	= 1 << 9, /* OMAP1 only */
180 	CSDP_DST_PORT_OCP_T1	= 2 << 9, /* OMAP1 only */
181 	CSDP_DST_PORT_TIPB	= 3 << 9, /* OMAP1 only */
182 	CSDP_DST_PORT_OCP_T2	= 4 << 9, /* OMAP1 only */
183 	CSDP_DST_PORT_MPUI	= 5 << 9, /* OMAP1 only */
184 	CSDP_DST_PACKED		= BIT(13),
185 	CSDP_DST_BURST_1	= 0 << 14,
186 	CSDP_DST_BURST_16	= 1 << 14,
187 	CSDP_DST_BURST_32	= 2 << 14,
188 	CSDP_DST_BURST_64	= 3 << 14,
189 	CSDP_WRITE_NON_POSTED	= 0 << 16,
190 	CSDP_WRITE_POSTED	= 1 << 16,
191 	CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
192 
193 	CICR_TOUT_IE		= BIT(0),	/* OMAP1 only */
194 	CICR_DROP_IE		= BIT(1),
195 	CICR_HALF_IE		= BIT(2),
196 	CICR_FRAME_IE		= BIT(3),
197 	CICR_LAST_IE		= BIT(4),
198 	CICR_BLOCK_IE		= BIT(5),
199 	CICR_PKT_IE		= BIT(7),	/* OMAP2+ only */
200 	CICR_TRANS_ERR_IE	= BIT(8),	/* OMAP2+ only */
201 	CICR_SUPERVISOR_ERR_IE	= BIT(10),	/* OMAP2+ only */
202 	CICR_MISALIGNED_ERR_IE	= BIT(11),	/* OMAP2+ only */
203 	CICR_DRAIN_IE		= BIT(12),	/* OMAP2+ only */
204 	CICR_SUPER_BLOCK_IE	= BIT(14),	/* OMAP2+ only */
205 
206 	CLNK_CTRL_ENABLE_LNK	= BIT(15),
207 
208 	CDP_DST_VALID_INC	= 0 << 0,
209 	CDP_DST_VALID_RELOAD	= 1 << 0,
210 	CDP_DST_VALID_REUSE	= 2 << 0,
211 	CDP_SRC_VALID_INC	= 0 << 2,
212 	CDP_SRC_VALID_RELOAD	= 1 << 2,
213 	CDP_SRC_VALID_REUSE	= 2 << 2,
214 	CDP_NTYPE_TYPE1		= 1 << 4,
215 	CDP_NTYPE_TYPE2		= 2 << 4,
216 	CDP_NTYPE_TYPE3		= 3 << 4,
217 	CDP_TMODE_NORMAL	= 0 << 8,
218 	CDP_TMODE_LLIST		= 1 << 8,
219 	CDP_FAST		= BIT(10),
220 };
221 
222 static const unsigned es_bytes[] = {
223 	[CSDP_DATA_TYPE_8] = 1,
224 	[CSDP_DATA_TYPE_16] = 2,
225 	[CSDP_DATA_TYPE_32] = 4,
226 };
227 
228 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param);
229 static struct of_dma_filter_info omap_dma_info = {
230 	.filter_fn = omap_dma_filter_fn,
231 };
232 
to_omap_dma_dev(struct dma_device * d)233 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
234 {
235 	return container_of(d, struct omap_dmadev, ddev);
236 }
237 
to_omap_dma_chan(struct dma_chan * c)238 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
239 {
240 	return container_of(c, struct omap_chan, vc.chan);
241 }
242 
to_omap_dma_desc(struct dma_async_tx_descriptor * t)243 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
244 {
245 	return container_of(t, struct omap_desc, vd.tx);
246 }
247 
omap_dma_desc_free(struct virt_dma_desc * vd)248 static void omap_dma_desc_free(struct virt_dma_desc *vd)
249 {
250 	struct omap_desc *d = to_omap_dma_desc(&vd->tx);
251 
252 	if (d->using_ll) {
253 		struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
254 		int i;
255 
256 		for (i = 0; i < d->sglen; i++) {
257 			if (d->sg[i].t2_desc)
258 				dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
259 					      d->sg[i].t2_desc_paddr);
260 		}
261 	}
262 
263 	kfree(d);
264 }
265 
omap_dma_fill_type2_desc(struct omap_desc * d,int idx,enum dma_transfer_direction dir,bool last)266 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
267 				     enum dma_transfer_direction dir, bool last)
268 {
269 	struct omap_sg *sg = &d->sg[idx];
270 	struct omap_type2_desc *t2_desc = sg->t2_desc;
271 
272 	if (idx)
273 		d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
274 	if (last)
275 		t2_desc->next_desc = 0xfffffffc;
276 
277 	t2_desc->en = sg->en;
278 	t2_desc->addr = sg->addr;
279 	t2_desc->fn = sg->fn & 0xffff;
280 	t2_desc->cicr = d->cicr;
281 	if (!last)
282 		t2_desc->cicr &= ~CICR_BLOCK_IE;
283 
284 	switch (dir) {
285 	case DMA_DEV_TO_MEM:
286 		t2_desc->cdei = sg->ei;
287 		t2_desc->csei = d->ei;
288 		t2_desc->cdfi = sg->fi;
289 		t2_desc->csfi = d->fi;
290 
291 		t2_desc->en |= DESC_NXT_DV_REFRESH;
292 		t2_desc->en |= DESC_NXT_SV_REUSE;
293 		break;
294 	case DMA_MEM_TO_DEV:
295 		t2_desc->cdei = d->ei;
296 		t2_desc->csei = sg->ei;
297 		t2_desc->cdfi = d->fi;
298 		t2_desc->csfi = sg->fi;
299 
300 		t2_desc->en |= DESC_NXT_SV_REFRESH;
301 		t2_desc->en |= DESC_NXT_DV_REUSE;
302 		break;
303 	default:
304 		return;
305 	}
306 
307 	t2_desc->en |= DESC_NTYPE_TYPE2;
308 }
309 
omap_dma_write(uint32_t val,unsigned type,void __iomem * addr)310 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
311 {
312 	switch (type) {
313 	case OMAP_DMA_REG_16BIT:
314 		writew_relaxed(val, addr);
315 		break;
316 	case OMAP_DMA_REG_2X16BIT:
317 		writew_relaxed(val, addr);
318 		writew_relaxed(val >> 16, addr + 2);
319 		break;
320 	case OMAP_DMA_REG_32BIT:
321 		writel_relaxed(val, addr);
322 		break;
323 	default:
324 		WARN_ON(1);
325 	}
326 }
327 
omap_dma_read(unsigned type,void __iomem * addr)328 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
329 {
330 	unsigned val;
331 
332 	switch (type) {
333 	case OMAP_DMA_REG_16BIT:
334 		val = readw_relaxed(addr);
335 		break;
336 	case OMAP_DMA_REG_2X16BIT:
337 		val = readw_relaxed(addr);
338 		val |= readw_relaxed(addr + 2) << 16;
339 		break;
340 	case OMAP_DMA_REG_32BIT:
341 		val = readl_relaxed(addr);
342 		break;
343 	default:
344 		WARN_ON(1);
345 		val = 0;
346 	}
347 
348 	return val;
349 }
350 
omap_dma_glbl_write(struct omap_dmadev * od,unsigned reg,unsigned val)351 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
352 {
353 	const struct omap_dma_reg *r = od->reg_map + reg;
354 
355 	WARN_ON(r->stride);
356 
357 	omap_dma_write(val, r->type, od->base + r->offset);
358 }
359 
omap_dma_glbl_read(struct omap_dmadev * od,unsigned reg)360 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
361 {
362 	const struct omap_dma_reg *r = od->reg_map + reg;
363 
364 	WARN_ON(r->stride);
365 
366 	return omap_dma_read(r->type, od->base + r->offset);
367 }
368 
omap_dma_chan_write(struct omap_chan * c,unsigned reg,unsigned val)369 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
370 {
371 	const struct omap_dma_reg *r = c->reg_map + reg;
372 
373 	omap_dma_write(val, r->type, c->channel_base + r->offset);
374 }
375 
omap_dma_chan_read(struct omap_chan * c,unsigned reg)376 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
377 {
378 	const struct omap_dma_reg *r = c->reg_map + reg;
379 
380 	return omap_dma_read(r->type, c->channel_base + r->offset);
381 }
382 
omap_dma_clear_csr(struct omap_chan * c)383 static void omap_dma_clear_csr(struct omap_chan *c)
384 {
385 	if (dma_omap1())
386 		omap_dma_chan_read(c, CSR);
387 	else
388 		omap_dma_chan_write(c, CSR, ~0);
389 }
390 
omap_dma_get_csr(struct omap_chan * c)391 static unsigned omap_dma_get_csr(struct omap_chan *c)
392 {
393 	unsigned val = omap_dma_chan_read(c, CSR);
394 
395 	if (!dma_omap1())
396 		omap_dma_chan_write(c, CSR, val);
397 
398 	return val;
399 }
400 
omap_dma_clear_lch(struct omap_dmadev * od,int lch)401 static void omap_dma_clear_lch(struct omap_dmadev *od, int lch)
402 {
403 	struct omap_chan *c;
404 	int i;
405 
406 	c = od->lch_map[lch];
407 	if (!c)
408 		return;
409 
410 	for (i = CSDP; i <= od->cfg->lch_end; i++)
411 		omap_dma_chan_write(c, i, 0);
412 }
413 
omap_dma_assign(struct omap_dmadev * od,struct omap_chan * c,unsigned lch)414 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
415 	unsigned lch)
416 {
417 	c->channel_base = od->base + od->plat->channel_stride * lch;
418 
419 	od->lch_map[lch] = c;
420 }
421 
omap_dma_start(struct omap_chan * c,struct omap_desc * d)422 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
423 {
424 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
425 	uint16_t cicr = d->cicr;
426 
427 	if (__dma_omap15xx(od->plat->dma_attr))
428 		omap_dma_chan_write(c, CPC, 0);
429 	else
430 		omap_dma_chan_write(c, CDAC, 0);
431 
432 	omap_dma_clear_csr(c);
433 
434 	if (d->using_ll) {
435 		uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
436 
437 		if (d->dir == DMA_DEV_TO_MEM)
438 			cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
439 		else
440 			cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
441 		omap_dma_chan_write(c, CDP, cdp);
442 
443 		omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
444 		omap_dma_chan_write(c, CCDN, 0);
445 		omap_dma_chan_write(c, CCFN, 0xffff);
446 		omap_dma_chan_write(c, CCEN, 0xffffff);
447 
448 		cicr &= ~CICR_BLOCK_IE;
449 	} else if (od->ll123_supported) {
450 		omap_dma_chan_write(c, CDP, 0);
451 	}
452 
453 	/* Enable interrupts */
454 	omap_dma_chan_write(c, CICR, cicr);
455 
456 	/* Enable channel */
457 	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
458 
459 	c->running = true;
460 }
461 
omap_dma_drain_chan(struct omap_chan * c)462 static void omap_dma_drain_chan(struct omap_chan *c)
463 {
464 	int i;
465 	u32 val;
466 
467 	/* Wait for sDMA FIFO to drain */
468 	for (i = 0; ; i++) {
469 		val = omap_dma_chan_read(c, CCR);
470 		if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
471 			break;
472 
473 		if (i > 100)
474 			break;
475 
476 		udelay(5);
477 	}
478 
479 	if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
480 		dev_err(c->vc.chan.device->dev,
481 			"DMA drain did not complete on lch %d\n",
482 			c->dma_ch);
483 }
484 
omap_dma_stop(struct omap_chan * c)485 static int omap_dma_stop(struct omap_chan *c)
486 {
487 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
488 	uint32_t val;
489 
490 	/* disable irq */
491 	omap_dma_chan_write(c, CICR, 0);
492 
493 	omap_dma_clear_csr(c);
494 
495 	val = omap_dma_chan_read(c, CCR);
496 	if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
497 		uint32_t sysconfig;
498 
499 		sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
500 		val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
501 		val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
502 		omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
503 
504 		val = omap_dma_chan_read(c, CCR);
505 		val &= ~CCR_ENABLE;
506 		omap_dma_chan_write(c, CCR, val);
507 
508 		if (!(c->ccr & CCR_BUFFERING_DISABLE))
509 			omap_dma_drain_chan(c);
510 
511 		omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
512 	} else {
513 		if (!(val & CCR_ENABLE))
514 			return -EINVAL;
515 
516 		val &= ~CCR_ENABLE;
517 		omap_dma_chan_write(c, CCR, val);
518 
519 		if (!(c->ccr & CCR_BUFFERING_DISABLE))
520 			omap_dma_drain_chan(c);
521 	}
522 
523 	mb();
524 
525 	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
526 		val = omap_dma_chan_read(c, CLNK_CTRL);
527 
528 		if (dma_omap1())
529 			val |= 1 << 14; /* set the STOP_LNK bit */
530 		else
531 			val &= ~CLNK_CTRL_ENABLE_LNK;
532 
533 		omap_dma_chan_write(c, CLNK_CTRL, val);
534 	}
535 	c->running = false;
536 	return 0;
537 }
538 
omap_dma_start_sg(struct omap_chan * c,struct omap_desc * d)539 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
540 {
541 	struct omap_sg *sg = d->sg + c->sgidx;
542 	unsigned cxsa, cxei, cxfi;
543 
544 	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
545 		cxsa = CDSA;
546 		cxei = CDEI;
547 		cxfi = CDFI;
548 	} else {
549 		cxsa = CSSA;
550 		cxei = CSEI;
551 		cxfi = CSFI;
552 	}
553 
554 	omap_dma_chan_write(c, cxsa, sg->addr);
555 	omap_dma_chan_write(c, cxei, sg->ei);
556 	omap_dma_chan_write(c, cxfi, sg->fi);
557 	omap_dma_chan_write(c, CEN, sg->en);
558 	omap_dma_chan_write(c, CFN, sg->fn);
559 
560 	omap_dma_start(c, d);
561 	c->sgidx++;
562 }
563 
omap_dma_start_desc(struct omap_chan * c)564 static void omap_dma_start_desc(struct omap_chan *c)
565 {
566 	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
567 	struct omap_desc *d;
568 	unsigned cxsa, cxei, cxfi;
569 
570 	if (!vd) {
571 		c->desc = NULL;
572 		return;
573 	}
574 
575 	list_del(&vd->node);
576 
577 	c->desc = d = to_omap_dma_desc(&vd->tx);
578 	c->sgidx = 0;
579 
580 	/*
581 	 * This provides the necessary barrier to ensure data held in
582 	 * DMA coherent memory is visible to the DMA engine prior to
583 	 * the transfer starting.
584 	 */
585 	mb();
586 
587 	omap_dma_chan_write(c, CCR, d->ccr);
588 	if (dma_omap1())
589 		omap_dma_chan_write(c, CCR2, d->ccr >> 16);
590 
591 	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
592 		cxsa = CSSA;
593 		cxei = CSEI;
594 		cxfi = CSFI;
595 	} else {
596 		cxsa = CDSA;
597 		cxei = CDEI;
598 		cxfi = CDFI;
599 	}
600 
601 	omap_dma_chan_write(c, cxsa, d->dev_addr);
602 	omap_dma_chan_write(c, cxei, d->ei);
603 	omap_dma_chan_write(c, cxfi, d->fi);
604 	omap_dma_chan_write(c, CSDP, d->csdp);
605 	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
606 
607 	omap_dma_start_sg(c, d);
608 }
609 
omap_dma_callback(int ch,u16 status,void * data)610 static void omap_dma_callback(int ch, u16 status, void *data)
611 {
612 	struct omap_chan *c = data;
613 	struct omap_desc *d;
614 	unsigned long flags;
615 
616 	spin_lock_irqsave(&c->vc.lock, flags);
617 	d = c->desc;
618 	if (d) {
619 		if (c->cyclic) {
620 			vchan_cyclic_callback(&d->vd);
621 		} else if (d->using_ll || c->sgidx == d->sglen) {
622 			omap_dma_start_desc(c);
623 			vchan_cookie_complete(&d->vd);
624 		} else {
625 			omap_dma_start_sg(c, d);
626 		}
627 	}
628 	spin_unlock_irqrestore(&c->vc.lock, flags);
629 }
630 
omap_dma_irq(int irq,void * devid)631 static irqreturn_t omap_dma_irq(int irq, void *devid)
632 {
633 	struct omap_dmadev *od = devid;
634 	unsigned status, channel;
635 
636 	spin_lock(&od->irq_lock);
637 
638 	status = omap_dma_glbl_read(od, IRQSTATUS_L1);
639 	status &= od->irq_enable_mask;
640 	if (status == 0) {
641 		spin_unlock(&od->irq_lock);
642 		return IRQ_NONE;
643 	}
644 
645 	while ((channel = ffs(status)) != 0) {
646 		unsigned mask, csr;
647 		struct omap_chan *c;
648 
649 		channel -= 1;
650 		mask = BIT(channel);
651 		status &= ~mask;
652 
653 		c = od->lch_map[channel];
654 		if (c == NULL) {
655 			/* This should never happen */
656 			dev_err(od->ddev.dev, "invalid channel %u\n", channel);
657 			continue;
658 		}
659 
660 		csr = omap_dma_get_csr(c);
661 		omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
662 
663 		omap_dma_callback(channel, csr, c);
664 	}
665 
666 	spin_unlock(&od->irq_lock);
667 
668 	return IRQ_HANDLED;
669 }
670 
omap_dma_get_lch(struct omap_dmadev * od,int * lch)671 static int omap_dma_get_lch(struct omap_dmadev *od, int *lch)
672 {
673 	int channel;
674 
675 	mutex_lock(&od->lch_lock);
676 	channel = find_first_zero_bit(od->lch_bitmap, od->lch_count);
677 	if (channel >= od->lch_count)
678 		goto out_busy;
679 	set_bit(channel, od->lch_bitmap);
680 	mutex_unlock(&od->lch_lock);
681 
682 	omap_dma_clear_lch(od, channel);
683 	*lch = channel;
684 
685 	return 0;
686 
687 out_busy:
688 	mutex_unlock(&od->lch_lock);
689 	*lch = -EINVAL;
690 
691 	return -EBUSY;
692 }
693 
omap_dma_put_lch(struct omap_dmadev * od,int lch)694 static void omap_dma_put_lch(struct omap_dmadev *od, int lch)
695 {
696 	omap_dma_clear_lch(od, lch);
697 	mutex_lock(&od->lch_lock);
698 	clear_bit(lch, od->lch_bitmap);
699 	mutex_unlock(&od->lch_lock);
700 }
701 
omap_dma_legacy(struct omap_dmadev * od)702 static inline bool omap_dma_legacy(struct omap_dmadev *od)
703 {
704 	return IS_ENABLED(CONFIG_ARCH_OMAP1) && od->legacy;
705 }
706 
omap_dma_alloc_chan_resources(struct dma_chan * chan)707 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
708 {
709 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
710 	struct omap_chan *c = to_omap_dma_chan(chan);
711 	struct device *dev = od->ddev.dev;
712 	int ret;
713 
714 	if (omap_dma_legacy(od)) {
715 		ret = omap_request_dma(c->dma_sig, "DMA engine",
716 				       omap_dma_callback, c, &c->dma_ch);
717 	} else {
718 		ret = omap_dma_get_lch(od, &c->dma_ch);
719 	}
720 
721 	dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
722 
723 	if (ret >= 0) {
724 		omap_dma_assign(od, c, c->dma_ch);
725 
726 		if (!omap_dma_legacy(od)) {
727 			unsigned val;
728 
729 			spin_lock_irq(&od->irq_lock);
730 			val = BIT(c->dma_ch);
731 			omap_dma_glbl_write(od, IRQSTATUS_L1, val);
732 			od->irq_enable_mask |= val;
733 			omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
734 
735 			val = omap_dma_glbl_read(od, IRQENABLE_L0);
736 			val &= ~BIT(c->dma_ch);
737 			omap_dma_glbl_write(od, IRQENABLE_L0, val);
738 			spin_unlock_irq(&od->irq_lock);
739 		}
740 	}
741 
742 	if (dma_omap1()) {
743 		if (__dma_omap16xx(od->plat->dma_attr)) {
744 			c->ccr = CCR_OMAP31_DISABLE;
745 			/* Duplicate what plat-omap/dma.c does */
746 			c->ccr |= c->dma_ch + 1;
747 		} else {
748 			c->ccr = c->dma_sig & 0x1f;
749 		}
750 	} else {
751 		c->ccr = c->dma_sig & 0x1f;
752 		c->ccr |= (c->dma_sig & ~0x1f) << 14;
753 	}
754 	if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
755 		c->ccr |= CCR_BUFFERING_DISABLE;
756 
757 	return ret;
758 }
759 
omap_dma_free_chan_resources(struct dma_chan * chan)760 static void omap_dma_free_chan_resources(struct dma_chan *chan)
761 {
762 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
763 	struct omap_chan *c = to_omap_dma_chan(chan);
764 
765 	if (!omap_dma_legacy(od)) {
766 		spin_lock_irq(&od->irq_lock);
767 		od->irq_enable_mask &= ~BIT(c->dma_ch);
768 		omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
769 		spin_unlock_irq(&od->irq_lock);
770 	}
771 
772 	c->channel_base = NULL;
773 	od->lch_map[c->dma_ch] = NULL;
774 	vchan_free_chan_resources(&c->vc);
775 
776 	if (omap_dma_legacy(od))
777 		omap_free_dma(c->dma_ch);
778 	else
779 		omap_dma_put_lch(od, c->dma_ch);
780 
781 	dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
782 		c->dma_sig);
783 	c->dma_sig = 0;
784 }
785 
omap_dma_sg_size(struct omap_sg * sg)786 static size_t omap_dma_sg_size(struct omap_sg *sg)
787 {
788 	return sg->en * sg->fn;
789 }
790 
omap_dma_desc_size(struct omap_desc * d)791 static size_t omap_dma_desc_size(struct omap_desc *d)
792 {
793 	unsigned i;
794 	size_t size;
795 
796 	for (size = i = 0; i < d->sglen; i++)
797 		size += omap_dma_sg_size(&d->sg[i]);
798 
799 	return size * es_bytes[d->es];
800 }
801 
omap_dma_desc_size_pos(struct omap_desc * d,dma_addr_t addr)802 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
803 {
804 	unsigned i;
805 	size_t size, es_size = es_bytes[d->es];
806 
807 	for (size = i = 0; i < d->sglen; i++) {
808 		size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
809 
810 		if (size)
811 			size += this_size;
812 		else if (addr >= d->sg[i].addr &&
813 			 addr < d->sg[i].addr + this_size)
814 			size += d->sg[i].addr + this_size - addr;
815 	}
816 	return size;
817 }
818 
819 /*
820  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
821  * read before the DMA controller finished disabling the channel.
822  */
omap_dma_chan_read_3_3(struct omap_chan * c,unsigned reg)823 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
824 {
825 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
826 	uint32_t val;
827 
828 	val = omap_dma_chan_read(c, reg);
829 	if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
830 		val = omap_dma_chan_read(c, reg);
831 
832 	return val;
833 }
834 
omap_dma_get_src_pos(struct omap_chan * c)835 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
836 {
837 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
838 	dma_addr_t addr, cdac;
839 
840 	if (__dma_omap15xx(od->plat->dma_attr)) {
841 		addr = omap_dma_chan_read(c, CPC);
842 	} else {
843 		addr = omap_dma_chan_read_3_3(c, CSAC);
844 		cdac = omap_dma_chan_read_3_3(c, CDAC);
845 
846 		/*
847 		 * CDAC == 0 indicates that the DMA transfer on the channel has
848 		 * not been started (no data has been transferred so far).
849 		 * Return the programmed source start address in this case.
850 		 */
851 		if (cdac == 0)
852 			addr = omap_dma_chan_read(c, CSSA);
853 	}
854 
855 	if (dma_omap1())
856 		addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
857 
858 	return addr;
859 }
860 
omap_dma_get_dst_pos(struct omap_chan * c)861 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
862 {
863 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
864 	dma_addr_t addr;
865 
866 	if (__dma_omap15xx(od->plat->dma_attr)) {
867 		addr = omap_dma_chan_read(c, CPC);
868 	} else {
869 		addr = omap_dma_chan_read_3_3(c, CDAC);
870 
871 		/*
872 		 * CDAC == 0 indicates that the DMA transfer on the channel
873 		 * has not been started (no data has been transferred so
874 		 * far).  Return the programmed destination start address in
875 		 * this case.
876 		 */
877 		if (addr == 0)
878 			addr = omap_dma_chan_read(c, CDSA);
879 	}
880 
881 	if (dma_omap1())
882 		addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
883 
884 	return addr;
885 }
886 
omap_dma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)887 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
888 	dma_cookie_t cookie, struct dma_tx_state *txstate)
889 {
890 	struct omap_chan *c = to_omap_dma_chan(chan);
891 	enum dma_status ret;
892 	unsigned long flags;
893 	struct omap_desc *d = NULL;
894 
895 	ret = dma_cookie_status(chan, cookie, txstate);
896 	if (ret == DMA_COMPLETE)
897 		return ret;
898 
899 	spin_lock_irqsave(&c->vc.lock, flags);
900 	if (c->desc && c->desc->vd.tx.cookie == cookie)
901 		d = c->desc;
902 
903 	if (!txstate)
904 		goto out;
905 
906 	if (d) {
907 		dma_addr_t pos;
908 
909 		if (d->dir == DMA_MEM_TO_DEV)
910 			pos = omap_dma_get_src_pos(c);
911 		else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
912 			pos = omap_dma_get_dst_pos(c);
913 		else
914 			pos = 0;
915 
916 		txstate->residue = omap_dma_desc_size_pos(d, pos);
917 	} else {
918 		struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie);
919 
920 		if (vd)
921 			txstate->residue = omap_dma_desc_size(
922 						to_omap_dma_desc(&vd->tx));
923 		else
924 			txstate->residue = 0;
925 	}
926 
927 out:
928 	if (ret == DMA_IN_PROGRESS && c->paused) {
929 		ret = DMA_PAUSED;
930 	} else if (d && d->polled && c->running) {
931 		uint32_t ccr = omap_dma_chan_read(c, CCR);
932 		/*
933 		 * The channel is no longer active, set the return value
934 		 * accordingly and mark it as completed
935 		 */
936 		if (!(ccr & CCR_ENABLE)) {
937 			ret = DMA_COMPLETE;
938 			omap_dma_start_desc(c);
939 			vchan_cookie_complete(&d->vd);
940 		}
941 	}
942 
943 	spin_unlock_irqrestore(&c->vc.lock, flags);
944 
945 	return ret;
946 }
947 
omap_dma_issue_pending(struct dma_chan * chan)948 static void omap_dma_issue_pending(struct dma_chan *chan)
949 {
950 	struct omap_chan *c = to_omap_dma_chan(chan);
951 	unsigned long flags;
952 
953 	spin_lock_irqsave(&c->vc.lock, flags);
954 	if (vchan_issue_pending(&c->vc) && !c->desc)
955 		omap_dma_start_desc(c);
956 	spin_unlock_irqrestore(&c->vc.lock, flags);
957 }
958 
omap_dma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned sglen,enum dma_transfer_direction dir,unsigned long tx_flags,void * context)959 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
960 	struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
961 	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
962 {
963 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
964 	struct omap_chan *c = to_omap_dma_chan(chan);
965 	enum dma_slave_buswidth dev_width;
966 	struct scatterlist *sgent;
967 	struct omap_desc *d;
968 	dma_addr_t dev_addr;
969 	unsigned i, es, en, frame_bytes;
970 	bool ll_failed = false;
971 	u32 burst;
972 	u32 port_window, port_window_bytes;
973 
974 	if (dir == DMA_DEV_TO_MEM) {
975 		dev_addr = c->cfg.src_addr;
976 		dev_width = c->cfg.src_addr_width;
977 		burst = c->cfg.src_maxburst;
978 		port_window = c->cfg.src_port_window_size;
979 	} else if (dir == DMA_MEM_TO_DEV) {
980 		dev_addr = c->cfg.dst_addr;
981 		dev_width = c->cfg.dst_addr_width;
982 		burst = c->cfg.dst_maxburst;
983 		port_window = c->cfg.dst_port_window_size;
984 	} else {
985 		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
986 		return NULL;
987 	}
988 
989 	/* Bus width translates to the element size (ES) */
990 	switch (dev_width) {
991 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
992 		es = CSDP_DATA_TYPE_8;
993 		break;
994 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
995 		es = CSDP_DATA_TYPE_16;
996 		break;
997 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
998 		es = CSDP_DATA_TYPE_32;
999 		break;
1000 	default: /* not reached */
1001 		return NULL;
1002 	}
1003 
1004 	/* Now allocate and setup the descriptor. */
1005 	d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC);
1006 	if (!d)
1007 		return NULL;
1008 	d->sglen = sglen;
1009 
1010 	d->dir = dir;
1011 	d->dev_addr = dev_addr;
1012 	d->es = es;
1013 
1014 	/* When the port_window is used, one frame must cover the window */
1015 	if (port_window) {
1016 		burst = port_window;
1017 		port_window_bytes = port_window * es_bytes[es];
1018 
1019 		d->ei = 1;
1020 		/*
1021 		 * One frame covers the port_window and by  configure
1022 		 * the source frame index to be -1 * (port_window - 1)
1023 		 * we instruct the sDMA that after a frame is processed
1024 		 * it should move back to the start of the window.
1025 		 */
1026 		d->fi = -(port_window_bytes - 1);
1027 	}
1028 
1029 	d->ccr = c->ccr | CCR_SYNC_FRAME;
1030 	if (dir == DMA_DEV_TO_MEM) {
1031 		d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
1032 
1033 		d->ccr |= CCR_DST_AMODE_POSTINC;
1034 		if (port_window) {
1035 			d->ccr |= CCR_SRC_AMODE_DBLIDX;
1036 
1037 			if (port_window_bytes >= 64)
1038 				d->csdp |= CSDP_SRC_BURST_64;
1039 			else if (port_window_bytes >= 32)
1040 				d->csdp |= CSDP_SRC_BURST_32;
1041 			else if (port_window_bytes >= 16)
1042 				d->csdp |= CSDP_SRC_BURST_16;
1043 
1044 		} else {
1045 			d->ccr |= CCR_SRC_AMODE_CONSTANT;
1046 		}
1047 	} else {
1048 		d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
1049 
1050 		d->ccr |= CCR_SRC_AMODE_POSTINC;
1051 		if (port_window) {
1052 			d->ccr |= CCR_DST_AMODE_DBLIDX;
1053 
1054 			if (port_window_bytes >= 64)
1055 				d->csdp |= CSDP_DST_BURST_64;
1056 			else if (port_window_bytes >= 32)
1057 				d->csdp |= CSDP_DST_BURST_32;
1058 			else if (port_window_bytes >= 16)
1059 				d->csdp |= CSDP_DST_BURST_16;
1060 		} else {
1061 			d->ccr |= CCR_DST_AMODE_CONSTANT;
1062 		}
1063 	}
1064 
1065 	d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
1066 	d->csdp |= es;
1067 
1068 	if (dma_omap1()) {
1069 		d->cicr |= CICR_TOUT_IE;
1070 
1071 		if (dir == DMA_DEV_TO_MEM)
1072 			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
1073 		else
1074 			d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
1075 	} else {
1076 		if (dir == DMA_DEV_TO_MEM)
1077 			d->ccr |= CCR_TRIGGER_SRC;
1078 
1079 		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1080 
1081 		if (port_window)
1082 			d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
1083 	}
1084 	if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
1085 		d->clnk_ctrl = c->dma_ch;
1086 
1087 	/*
1088 	 * Build our scatterlist entries: each contains the address,
1089 	 * the number of elements (EN) in each frame, and the number of
1090 	 * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1091 	 *
1092 	 * Burst size translates to number of elements with frame sync.
1093 	 * Note: DMA engine defines burst to be the number of dev-width
1094 	 * transfers.
1095 	 */
1096 	en = burst;
1097 	frame_bytes = es_bytes[es] * en;
1098 
1099 	if (sglen >= 2)
1100 		d->using_ll = od->ll123_supported;
1101 
1102 	for_each_sg(sgl, sgent, sglen, i) {
1103 		struct omap_sg *osg = &d->sg[i];
1104 
1105 		osg->addr = sg_dma_address(sgent);
1106 		osg->en = en;
1107 		osg->fn = sg_dma_len(sgent) / frame_bytes;
1108 
1109 		if (d->using_ll) {
1110 			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1111 						      &osg->t2_desc_paddr);
1112 			if (!osg->t2_desc) {
1113 				dev_err(chan->device->dev,
1114 					"t2_desc[%d] allocation failed\n", i);
1115 				ll_failed = true;
1116 				d->using_ll = false;
1117 				continue;
1118 			}
1119 
1120 			omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1121 		}
1122 	}
1123 
1124 	/* Release the dma_pool entries if one allocation failed */
1125 	if (ll_failed) {
1126 		for (i = 0; i < d->sglen; i++) {
1127 			struct omap_sg *osg = &d->sg[i];
1128 
1129 			if (osg->t2_desc) {
1130 				dma_pool_free(od->desc_pool, osg->t2_desc,
1131 					      osg->t2_desc_paddr);
1132 				osg->t2_desc = NULL;
1133 			}
1134 		}
1135 	}
1136 
1137 	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1138 }
1139 
omap_dma_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction dir,unsigned long flags)1140 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1141 	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1142 	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1143 {
1144 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1145 	struct omap_chan *c = to_omap_dma_chan(chan);
1146 	enum dma_slave_buswidth dev_width;
1147 	struct omap_desc *d;
1148 	dma_addr_t dev_addr;
1149 	unsigned es;
1150 	u32 burst;
1151 
1152 	if (dir == DMA_DEV_TO_MEM) {
1153 		dev_addr = c->cfg.src_addr;
1154 		dev_width = c->cfg.src_addr_width;
1155 		burst = c->cfg.src_maxburst;
1156 	} else if (dir == DMA_MEM_TO_DEV) {
1157 		dev_addr = c->cfg.dst_addr;
1158 		dev_width = c->cfg.dst_addr_width;
1159 		burst = c->cfg.dst_maxburst;
1160 	} else {
1161 		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1162 		return NULL;
1163 	}
1164 
1165 	/* Bus width translates to the element size (ES) */
1166 	switch (dev_width) {
1167 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1168 		es = CSDP_DATA_TYPE_8;
1169 		break;
1170 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1171 		es = CSDP_DATA_TYPE_16;
1172 		break;
1173 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1174 		es = CSDP_DATA_TYPE_32;
1175 		break;
1176 	default: /* not reached */
1177 		return NULL;
1178 	}
1179 
1180 	/* Now allocate and setup the descriptor. */
1181 	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1182 	if (!d)
1183 		return NULL;
1184 
1185 	d->dir = dir;
1186 	d->dev_addr = dev_addr;
1187 	d->fi = burst;
1188 	d->es = es;
1189 	d->sglen = 1;
1190 	d->sg[0].addr = buf_addr;
1191 	d->sg[0].en = period_len / es_bytes[es];
1192 	d->sg[0].fn = buf_len / period_len;
1193 
1194 	d->ccr = c->ccr;
1195 	if (dir == DMA_DEV_TO_MEM)
1196 		d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1197 	else
1198 		d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1199 
1200 	d->cicr = CICR_DROP_IE;
1201 	if (flags & DMA_PREP_INTERRUPT)
1202 		d->cicr |= CICR_FRAME_IE;
1203 
1204 	d->csdp = es;
1205 
1206 	if (dma_omap1()) {
1207 		d->cicr |= CICR_TOUT_IE;
1208 
1209 		if (dir == DMA_DEV_TO_MEM)
1210 			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1211 		else
1212 			d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1213 	} else {
1214 		if (burst)
1215 			d->ccr |= CCR_SYNC_PACKET;
1216 		else
1217 			d->ccr |= CCR_SYNC_ELEMENT;
1218 
1219 		if (dir == DMA_DEV_TO_MEM) {
1220 			d->ccr |= CCR_TRIGGER_SRC;
1221 			d->csdp |= CSDP_DST_PACKED;
1222 		} else {
1223 			d->csdp |= CSDP_SRC_PACKED;
1224 		}
1225 
1226 		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1227 
1228 		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1229 	}
1230 
1231 	if (__dma_omap15xx(od->plat->dma_attr))
1232 		d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1233 	else
1234 		d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1235 
1236 	c->cyclic = true;
1237 
1238 	return vchan_tx_prep(&c->vc, &d->vd, flags);
1239 }
1240 
omap_dma_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long tx_flags)1241 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1242 	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1243 	size_t len, unsigned long tx_flags)
1244 {
1245 	struct omap_chan *c = to_omap_dma_chan(chan);
1246 	struct omap_desc *d;
1247 	uint8_t data_type;
1248 
1249 	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1250 	if (!d)
1251 		return NULL;
1252 
1253 	data_type = __ffs((src | dest | len));
1254 	if (data_type > CSDP_DATA_TYPE_32)
1255 		data_type = CSDP_DATA_TYPE_32;
1256 
1257 	d->dir = DMA_MEM_TO_MEM;
1258 	d->dev_addr = src;
1259 	d->fi = 0;
1260 	d->es = data_type;
1261 	d->sglen = 1;
1262 	d->sg[0].en = len / BIT(data_type);
1263 	d->sg[0].fn = 1;
1264 	d->sg[0].addr = dest;
1265 	d->ccr = c->ccr;
1266 	d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1267 
1268 	if (tx_flags & DMA_PREP_INTERRUPT)
1269 		d->cicr |= CICR_FRAME_IE;
1270 	else
1271 		d->polled = true;
1272 
1273 	d->csdp = data_type;
1274 
1275 	if (dma_omap1()) {
1276 		d->cicr |= CICR_TOUT_IE;
1277 		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1278 	} else {
1279 		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1280 		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1281 		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1282 	}
1283 
1284 	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1285 }
1286 
omap_dma_prep_dma_interleaved(struct dma_chan * chan,struct dma_interleaved_template * xt,unsigned long flags)1287 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1288 	struct dma_chan *chan, struct dma_interleaved_template *xt,
1289 	unsigned long flags)
1290 {
1291 	struct omap_chan *c = to_omap_dma_chan(chan);
1292 	struct omap_desc *d;
1293 	struct omap_sg *sg;
1294 	uint8_t data_type;
1295 	size_t src_icg, dst_icg;
1296 
1297 	/* Slave mode is not supported */
1298 	if (is_slave_direction(xt->dir))
1299 		return NULL;
1300 
1301 	if (xt->frame_size != 1 || xt->numf == 0)
1302 		return NULL;
1303 
1304 	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1305 	if (!d)
1306 		return NULL;
1307 
1308 	data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1309 	if (data_type > CSDP_DATA_TYPE_32)
1310 		data_type = CSDP_DATA_TYPE_32;
1311 
1312 	d->sglen = 1;
1313 	sg = &d->sg[0];
1314 	d->dir = DMA_MEM_TO_MEM;
1315 	d->dev_addr = xt->src_start;
1316 	d->es = data_type;
1317 	sg->en = xt->sgl[0].size / BIT(data_type);
1318 	sg->fn = xt->numf;
1319 	sg->addr = xt->dst_start;
1320 	d->ccr = c->ccr;
1321 
1322 	src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1323 	dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1324 	if (src_icg) {
1325 		d->ccr |= CCR_SRC_AMODE_DBLIDX;
1326 		d->ei = 1;
1327 		d->fi = src_icg + 1;
1328 	} else if (xt->src_inc) {
1329 		d->ccr |= CCR_SRC_AMODE_POSTINC;
1330 		d->fi = 0;
1331 	} else {
1332 		dev_err(chan->device->dev,
1333 			"%s: SRC constant addressing is not supported\n",
1334 			__func__);
1335 		kfree(d);
1336 		return NULL;
1337 	}
1338 
1339 	if (dst_icg) {
1340 		d->ccr |= CCR_DST_AMODE_DBLIDX;
1341 		sg->ei = 1;
1342 		sg->fi = dst_icg + 1;
1343 	} else if (xt->dst_inc) {
1344 		d->ccr |= CCR_DST_AMODE_POSTINC;
1345 		sg->fi = 0;
1346 	} else {
1347 		dev_err(chan->device->dev,
1348 			"%s: DST constant addressing is not supported\n",
1349 			__func__);
1350 		kfree(d);
1351 		return NULL;
1352 	}
1353 
1354 	d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1355 
1356 	d->csdp = data_type;
1357 
1358 	if (dma_omap1()) {
1359 		d->cicr |= CICR_TOUT_IE;
1360 		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1361 	} else {
1362 		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1363 		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1364 		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1365 	}
1366 
1367 	return vchan_tx_prep(&c->vc, &d->vd, flags);
1368 }
1369 
omap_dma_slave_config(struct dma_chan * chan,struct dma_slave_config * cfg)1370 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1371 {
1372 	struct omap_chan *c = to_omap_dma_chan(chan);
1373 
1374 	if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1375 	    cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1376 		return -EINVAL;
1377 
1378 	if (cfg->src_maxburst > chan->device->max_burst ||
1379 	    cfg->dst_maxburst > chan->device->max_burst)
1380 		return -EINVAL;
1381 
1382 	memcpy(&c->cfg, cfg, sizeof(c->cfg));
1383 
1384 	return 0;
1385 }
1386 
omap_dma_terminate_all(struct dma_chan * chan)1387 static int omap_dma_terminate_all(struct dma_chan *chan)
1388 {
1389 	struct omap_chan *c = to_omap_dma_chan(chan);
1390 	unsigned long flags;
1391 	LIST_HEAD(head);
1392 
1393 	spin_lock_irqsave(&c->vc.lock, flags);
1394 
1395 	/*
1396 	 * Stop DMA activity: we assume the callback will not be called
1397 	 * after omap_dma_stop() returns (even if it does, it will see
1398 	 * c->desc is NULL and exit.)
1399 	 */
1400 	if (c->desc) {
1401 		vchan_terminate_vdesc(&c->desc->vd);
1402 		c->desc = NULL;
1403 		/* Avoid stopping the dma twice */
1404 		if (!c->paused)
1405 			omap_dma_stop(c);
1406 	}
1407 
1408 	c->cyclic = false;
1409 	c->paused = false;
1410 
1411 	vchan_get_all_descriptors(&c->vc, &head);
1412 	spin_unlock_irqrestore(&c->vc.lock, flags);
1413 	vchan_dma_desc_free_list(&c->vc, &head);
1414 
1415 	return 0;
1416 }
1417 
omap_dma_synchronize(struct dma_chan * chan)1418 static void omap_dma_synchronize(struct dma_chan *chan)
1419 {
1420 	struct omap_chan *c = to_omap_dma_chan(chan);
1421 
1422 	vchan_synchronize(&c->vc);
1423 }
1424 
omap_dma_pause(struct dma_chan * chan)1425 static int omap_dma_pause(struct dma_chan *chan)
1426 {
1427 	struct omap_chan *c = to_omap_dma_chan(chan);
1428 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1429 	unsigned long flags;
1430 	int ret = -EINVAL;
1431 	bool can_pause = false;
1432 
1433 	spin_lock_irqsave(&od->irq_lock, flags);
1434 
1435 	if (!c->desc)
1436 		goto out;
1437 
1438 	if (c->cyclic)
1439 		can_pause = true;
1440 
1441 	/*
1442 	 * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1443 	 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1444 	 * "When a channel is disabled during a transfer, the channel undergoes
1445 	 * an abort, unless it is hardware-source-synchronized …".
1446 	 * A source-synchronised channel is one where the fetching of data is
1447 	 * under control of the device. In other words, a device-to-memory
1448 	 * transfer. So, a destination-synchronised channel (which would be a
1449 	 * memory-to-device transfer) undergoes an abort if the CCR_ENABLE
1450 	 * bit is cleared.
1451 	 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1452 	 * aborts immediately after completion of current read/write
1453 	 * transactions and then the FIFO is cleaned up." The term "cleaned up"
1454 	 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1455 	 * are both clear _before_ disabling the channel, otherwise data loss
1456 	 * will occur.
1457 	 * The problem is that if the channel is active, then device activity
1458 	 * can result in DMA activity starting between reading those as both
1459 	 * clear and the write to DMA_CCR to clear the enable bit hitting the
1460 	 * hardware. If the DMA hardware can't drain the data in its FIFO to the
1461 	 * destination, then data loss "might" occur (say if we write to an UART
1462 	 * and the UART is not accepting any further data).
1463 	 */
1464 	else if (c->desc->dir == DMA_DEV_TO_MEM)
1465 		can_pause = true;
1466 
1467 	if (can_pause && !c->paused) {
1468 		ret = omap_dma_stop(c);
1469 		if (!ret)
1470 			c->paused = true;
1471 	}
1472 out:
1473 	spin_unlock_irqrestore(&od->irq_lock, flags);
1474 
1475 	return ret;
1476 }
1477 
omap_dma_resume(struct dma_chan * chan)1478 static int omap_dma_resume(struct dma_chan *chan)
1479 {
1480 	struct omap_chan *c = to_omap_dma_chan(chan);
1481 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1482 	unsigned long flags;
1483 	int ret = -EINVAL;
1484 
1485 	spin_lock_irqsave(&od->irq_lock, flags);
1486 
1487 	if (c->paused && c->desc) {
1488 		mb();
1489 
1490 		/* Restore channel link register */
1491 		omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1492 
1493 		omap_dma_start(c, c->desc);
1494 		c->paused = false;
1495 		ret = 0;
1496 	}
1497 	spin_unlock_irqrestore(&od->irq_lock, flags);
1498 
1499 	return ret;
1500 }
1501 
omap_dma_chan_init(struct omap_dmadev * od)1502 static int omap_dma_chan_init(struct omap_dmadev *od)
1503 {
1504 	struct omap_chan *c;
1505 
1506 	c = kzalloc(sizeof(*c), GFP_KERNEL);
1507 	if (!c)
1508 		return -ENOMEM;
1509 
1510 	c->reg_map = od->reg_map;
1511 	c->vc.desc_free = omap_dma_desc_free;
1512 	vchan_init(&c->vc, &od->ddev);
1513 
1514 	return 0;
1515 }
1516 
omap_dma_free(struct omap_dmadev * od)1517 static void omap_dma_free(struct omap_dmadev *od)
1518 {
1519 	while (!list_empty(&od->ddev.channels)) {
1520 		struct omap_chan *c = list_first_entry(&od->ddev.channels,
1521 			struct omap_chan, vc.chan.device_node);
1522 
1523 		list_del(&c->vc.chan.device_node);
1524 		tasklet_kill(&c->vc.task);
1525 		kfree(c);
1526 	}
1527 }
1528 
1529 /* Currently used by omap2 & 3 to block deeper SoC idle states */
omap_dma_busy(struct omap_dmadev * od)1530 static bool omap_dma_busy(struct omap_dmadev *od)
1531 {
1532 	struct omap_chan *c;
1533 	int lch = -1;
1534 
1535 	while (1) {
1536 		lch = find_next_bit(od->lch_bitmap, od->lch_count, lch + 1);
1537 		if (lch >= od->lch_count)
1538 			break;
1539 		c = od->lch_map[lch];
1540 		if (!c)
1541 			continue;
1542 		if (omap_dma_chan_read(c, CCR) & CCR_ENABLE)
1543 			return true;
1544 	}
1545 
1546 	return false;
1547 }
1548 
1549 /* Currently only used for omap2. For omap1, also a check for lcd_dma is needed */
omap_dma_busy_notifier(struct notifier_block * nb,unsigned long cmd,void * v)1550 static int omap_dma_busy_notifier(struct notifier_block *nb,
1551 				  unsigned long cmd, void *v)
1552 {
1553 	struct omap_dmadev *od;
1554 
1555 	od = container_of(nb, struct omap_dmadev, nb);
1556 
1557 	switch (cmd) {
1558 	case CPU_CLUSTER_PM_ENTER:
1559 		if (omap_dma_busy(od))
1560 			return NOTIFY_BAD;
1561 		break;
1562 	case CPU_CLUSTER_PM_ENTER_FAILED:
1563 	case CPU_CLUSTER_PM_EXIT:
1564 		break;
1565 	}
1566 
1567 	return NOTIFY_OK;
1568 }
1569 
1570 /*
1571  * We are using IRQENABLE_L1, and legacy DMA code was using IRQENABLE_L0.
1572  * As the DSP may be using IRQENABLE_L2 and L3, let's not touch those for
1573  * now. Context save seems to be only currently needed on omap3.
1574  */
omap_dma_context_save(struct omap_dmadev * od)1575 static void omap_dma_context_save(struct omap_dmadev *od)
1576 {
1577 	od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0);
1578 	od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1);
1579 	od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
1580 	od->context.gcr = omap_dma_glbl_read(od, GCR);
1581 }
1582 
omap_dma_context_restore(struct omap_dmadev * od)1583 static void omap_dma_context_restore(struct omap_dmadev *od)
1584 {
1585 	int i;
1586 
1587 	omap_dma_glbl_write(od, GCR, od->context.gcr);
1588 	omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig);
1589 	omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0);
1590 	omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1);
1591 
1592 	/* Clear IRQSTATUS_L0 as legacy DMA code is no longer doing it */
1593 	if (od->plat->errata & DMA_ROMCODE_BUG)
1594 		omap_dma_glbl_write(od, IRQSTATUS_L0, 0);
1595 
1596 	/* Clear dma channels */
1597 	for (i = 0; i < od->lch_count; i++)
1598 		omap_dma_clear_lch(od, i);
1599 }
1600 
1601 /* Currently only used for omap3 */
omap_dma_context_notifier(struct notifier_block * nb,unsigned long cmd,void * v)1602 static int omap_dma_context_notifier(struct notifier_block *nb,
1603 				     unsigned long cmd, void *v)
1604 {
1605 	struct omap_dmadev *od;
1606 
1607 	od = container_of(nb, struct omap_dmadev, nb);
1608 
1609 	switch (cmd) {
1610 	case CPU_CLUSTER_PM_ENTER:
1611 		if (omap_dma_busy(od))
1612 			return NOTIFY_BAD;
1613 		omap_dma_context_save(od);
1614 		break;
1615 	case CPU_CLUSTER_PM_ENTER_FAILED:	/* No need to restore context */
1616 		break;
1617 	case CPU_CLUSTER_PM_EXIT:
1618 		omap_dma_context_restore(od);
1619 		break;
1620 	}
1621 
1622 	return NOTIFY_OK;
1623 }
1624 
omap_dma_init_gcr(struct omap_dmadev * od,int arb_rate,int max_fifo_depth,int tparams)1625 static void omap_dma_init_gcr(struct omap_dmadev *od, int arb_rate,
1626 			      int max_fifo_depth, int tparams)
1627 {
1628 	u32 val;
1629 
1630 	/* Set only for omap2430 and later */
1631 	if (!od->cfg->rw_priority)
1632 		return;
1633 
1634 	if (max_fifo_depth == 0)
1635 		max_fifo_depth = 1;
1636 	if (arb_rate == 0)
1637 		arb_rate = 1;
1638 
1639 	val = 0xff & max_fifo_depth;
1640 	val |= (0x3 & tparams) << 12;
1641 	val |= (arb_rate & 0xff) << 16;
1642 
1643 	omap_dma_glbl_write(od, GCR, val);
1644 }
1645 
1646 #define OMAP_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1647 				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1648 				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1649 
1650 /*
1651  * No flags currently set for default configuration as omap1 is still
1652  * using platform data.
1653  */
1654 static const struct omap_dma_config default_cfg;
1655 
omap_dma_probe(struct platform_device * pdev)1656 static int omap_dma_probe(struct platform_device *pdev)
1657 {
1658 	const struct omap_dma_config *conf;
1659 	struct omap_dmadev *od;
1660 	int rc, i, irq;
1661 	u32 val;
1662 
1663 	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1664 	if (!od)
1665 		return -ENOMEM;
1666 
1667 	od->base = devm_platform_ioremap_resource(pdev, 0);
1668 	if (IS_ERR(od->base))
1669 		return PTR_ERR(od->base);
1670 
1671 	conf = of_device_get_match_data(&pdev->dev);
1672 	if (conf) {
1673 		od->cfg = conf;
1674 		od->plat = dev_get_platdata(&pdev->dev);
1675 		if (!od->plat) {
1676 			dev_err(&pdev->dev, "omap_system_dma_plat_info is missing");
1677 			return -ENODEV;
1678 		}
1679 	} else if (IS_ENABLED(CONFIG_ARCH_OMAP1)) {
1680 		od->cfg = &default_cfg;
1681 
1682 		od->plat = omap_get_plat_info();
1683 		if (!od->plat)
1684 			return -EPROBE_DEFER;
1685 	} else {
1686 		return -ENODEV;
1687 	}
1688 
1689 	od->reg_map = od->plat->reg_map;
1690 
1691 	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1692 	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1693 	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1694 	dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1695 	od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1696 	od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1697 	od->ddev.device_tx_status = omap_dma_tx_status;
1698 	od->ddev.device_issue_pending = omap_dma_issue_pending;
1699 	od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1700 	od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1701 	od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1702 	od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1703 	od->ddev.device_config = omap_dma_slave_config;
1704 	od->ddev.device_pause = omap_dma_pause;
1705 	od->ddev.device_resume = omap_dma_resume;
1706 	od->ddev.device_terminate_all = omap_dma_terminate_all;
1707 	od->ddev.device_synchronize = omap_dma_synchronize;
1708 	od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1709 	od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1710 	od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1711 	if (__dma_omap15xx(od->plat->dma_attr))
1712 		od->ddev.residue_granularity =
1713 				DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1714 	else
1715 		od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1716 	od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
1717 	od->ddev.dev = &pdev->dev;
1718 	INIT_LIST_HEAD(&od->ddev.channels);
1719 	mutex_init(&od->lch_lock);
1720 	spin_lock_init(&od->lock);
1721 	spin_lock_init(&od->irq_lock);
1722 
1723 	/* Number of DMA requests */
1724 	od->dma_requests = OMAP_SDMA_REQUESTS;
1725 	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1726 						      "dma-requests",
1727 						      &od->dma_requests)) {
1728 		dev_info(&pdev->dev,
1729 			 "Missing dma-requests property, using %u.\n",
1730 			 OMAP_SDMA_REQUESTS);
1731 	}
1732 
1733 	/* Number of available logical channels */
1734 	if (!pdev->dev.of_node) {
1735 		od->lch_count = od->plat->dma_attr->lch_count;
1736 		if (unlikely(!od->lch_count))
1737 			od->lch_count = OMAP_SDMA_CHANNELS;
1738 	} else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
1739 					&od->lch_count)) {
1740 		dev_info(&pdev->dev,
1741 			 "Missing dma-channels property, using %u.\n",
1742 			 OMAP_SDMA_CHANNELS);
1743 		od->lch_count = OMAP_SDMA_CHANNELS;
1744 	}
1745 
1746 	/* Mask of allowed logical channels */
1747 	if (pdev->dev.of_node && !of_property_read_u32(pdev->dev.of_node,
1748 						       "dma-channel-mask",
1749 						       &val)) {
1750 		/* Tag channels not in mask as reserved */
1751 		val = ~val;
1752 		bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count);
1753 	}
1754 	if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED)
1755 		bitmap_set(od->lch_bitmap, 0, 2);
1756 
1757 	od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count,
1758 				   sizeof(*od->lch_map),
1759 				   GFP_KERNEL);
1760 	if (!od->lch_map)
1761 		return -ENOMEM;
1762 
1763 	for (i = 0; i < od->dma_requests; i++) {
1764 		rc = omap_dma_chan_init(od);
1765 		if (rc) {
1766 			omap_dma_free(od);
1767 			return rc;
1768 		}
1769 	}
1770 
1771 	irq = platform_get_irq(pdev, 1);
1772 	if (irq <= 0) {
1773 		dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1774 		od->legacy = true;
1775 	} else {
1776 		/* Disable all interrupts */
1777 		od->irq_enable_mask = 0;
1778 		omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1779 
1780 		rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1781 				      IRQF_SHARED, "omap-dma-engine", od);
1782 		if (rc) {
1783 			omap_dma_free(od);
1784 			return rc;
1785 		}
1786 	}
1787 
1788 	if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1789 		od->ll123_supported = true;
1790 
1791 	od->ddev.filter.map = od->plat->slave_map;
1792 	od->ddev.filter.mapcnt = od->plat->slavecnt;
1793 	od->ddev.filter.fn = omap_dma_filter_fn;
1794 
1795 	if (od->ll123_supported) {
1796 		od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1797 						&pdev->dev,
1798 						sizeof(struct omap_type2_desc),
1799 						4, 0);
1800 		if (!od->desc_pool) {
1801 			dev_err(&pdev->dev,
1802 				"unable to allocate descriptor pool\n");
1803 			od->ll123_supported = false;
1804 		}
1805 	}
1806 
1807 	rc = dma_async_device_register(&od->ddev);
1808 	if (rc) {
1809 		pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1810 			rc);
1811 		omap_dma_free(od);
1812 		return rc;
1813 	}
1814 
1815 	platform_set_drvdata(pdev, od);
1816 
1817 	if (pdev->dev.of_node) {
1818 		omap_dma_info.dma_cap = od->ddev.cap_mask;
1819 
1820 		/* Device-tree DMA controller registration */
1821 		rc = of_dma_controller_register(pdev->dev.of_node,
1822 				of_dma_simple_xlate, &omap_dma_info);
1823 		if (rc) {
1824 			pr_warn("OMAP-DMA: failed to register DMA controller\n");
1825 			dma_async_device_unregister(&od->ddev);
1826 			omap_dma_free(od);
1827 		}
1828 	}
1829 
1830 	omap_dma_init_gcr(od, DMA_DEFAULT_ARB_RATE, DMA_DEFAULT_FIFO_DEPTH, 0);
1831 
1832 	if (od->cfg->needs_busy_check) {
1833 		od->nb.notifier_call = omap_dma_busy_notifier;
1834 		cpu_pm_register_notifier(&od->nb);
1835 	} else if (od->cfg->may_lose_context) {
1836 		od->nb.notifier_call = omap_dma_context_notifier;
1837 		cpu_pm_register_notifier(&od->nb);
1838 	}
1839 
1840 	dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1841 		 od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1842 
1843 	return rc;
1844 }
1845 
omap_dma_remove(struct platform_device * pdev)1846 static void omap_dma_remove(struct platform_device *pdev)
1847 {
1848 	struct omap_dmadev *od = platform_get_drvdata(pdev);
1849 	int irq;
1850 
1851 	if (od->cfg->may_lose_context)
1852 		cpu_pm_unregister_notifier(&od->nb);
1853 
1854 	if (pdev->dev.of_node)
1855 		of_dma_controller_free(pdev->dev.of_node);
1856 
1857 	irq = platform_get_irq(pdev, 1);
1858 	devm_free_irq(&pdev->dev, irq, od);
1859 
1860 	dma_async_device_unregister(&od->ddev);
1861 
1862 	if (!omap_dma_legacy(od)) {
1863 		/* Disable all interrupts */
1864 		omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1865 	}
1866 
1867 	if (od->ll123_supported)
1868 		dma_pool_destroy(od->desc_pool);
1869 
1870 	omap_dma_free(od);
1871 }
1872 
1873 static const struct omap_dma_config omap2420_data = {
1874 	.lch_end = CCFN,
1875 	.rw_priority = true,
1876 	.needs_lch_clear = true,
1877 	.needs_busy_check = true,
1878 };
1879 
1880 static const struct omap_dma_config omap2430_data = {
1881 	.lch_end = CCFN,
1882 	.rw_priority = true,
1883 	.needs_lch_clear = true,
1884 };
1885 
1886 static const struct omap_dma_config omap3430_data = {
1887 	.lch_end = CCFN,
1888 	.rw_priority = true,
1889 	.needs_lch_clear = true,
1890 	.may_lose_context = true,
1891 };
1892 
1893 static const struct omap_dma_config omap3630_data = {
1894 	.lch_end = CCDN,
1895 	.rw_priority = true,
1896 	.needs_lch_clear = true,
1897 	.may_lose_context = true,
1898 };
1899 
1900 static const struct omap_dma_config omap4_data = {
1901 	.lch_end = CCDN,
1902 	.rw_priority = true,
1903 	.needs_lch_clear = true,
1904 };
1905 
1906 static const struct of_device_id omap_dma_match[] = {
1907 	{ .compatible = "ti,omap2420-sdma", .data = &omap2420_data, },
1908 	{ .compatible = "ti,omap2430-sdma", .data = &omap2430_data, },
1909 	{ .compatible = "ti,omap3430-sdma", .data = &omap3430_data, },
1910 	{ .compatible = "ti,omap3630-sdma", .data = &omap3630_data, },
1911 	{ .compatible = "ti,omap4430-sdma", .data = &omap4_data, },
1912 	{},
1913 };
1914 MODULE_DEVICE_TABLE(of, omap_dma_match);
1915 
1916 static struct platform_driver omap_dma_driver = {
1917 	.probe	= omap_dma_probe,
1918 	.remove_new = omap_dma_remove,
1919 	.driver = {
1920 		.name = "omap-dma-engine",
1921 		.of_match_table = omap_dma_match,
1922 	},
1923 };
1924 
omap_dma_filter_fn(struct dma_chan * chan,void * param)1925 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1926 {
1927 	if (chan->device->dev->driver == &omap_dma_driver.driver) {
1928 		struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1929 		struct omap_chan *c = to_omap_dma_chan(chan);
1930 		unsigned req = *(unsigned *)param;
1931 
1932 		if (req <= od->dma_requests) {
1933 			c->dma_sig = req;
1934 			return true;
1935 		}
1936 	}
1937 	return false;
1938 }
1939 
omap_dma_init(void)1940 static int omap_dma_init(void)
1941 {
1942 	return platform_driver_register(&omap_dma_driver);
1943 }
1944 subsys_initcall(omap_dma_init);
1945 
omap_dma_exit(void)1946 static void __exit omap_dma_exit(void)
1947 {
1948 	platform_driver_unregister(&omap_dma_driver);
1949 }
1950 module_exit(omap_dma_exit);
1951 
1952 MODULE_AUTHOR("Russell King");
1953 MODULE_DESCRIPTION("Texas Instruments sDMA DMAengine support");
1954 MODULE_LICENSE("GPL");
1955