1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4 * Synopsys DesignWare eDMA core driver
5 *
6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
7 */
8
9 #include <linux/module.h>
10 #include <linux/device.h>
11 #include <linux/kernel.h>
12 #include <linux/dmaengine.h>
13 #include <linux/err.h>
14 #include <linux/interrupt.h>
15 #include <linux/irq.h>
16 #include <linux/dma/edma.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/string_choices.h>
19
20 #include "dw-edma-core.h"
21 #include "dw-edma-v0-core.h"
22 #include "dw-hdma-v0-core.h"
23 #include "../dmaengine.h"
24 #include "../virt-dma.h"
25
26 static inline
vd2dw_edma_desc(struct virt_dma_desc * vd)27 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
28 {
29 return container_of(vd, struct dw_edma_desc, vd);
30 }
31
32 static inline
dw_edma_get_pci_address(struct dw_edma_chan * chan,phys_addr_t cpu_addr)33 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
34 {
35 struct dw_edma_chip *chip = chan->dw->chip;
36
37 if (chip->ops->pci_address)
38 return chip->ops->pci_address(chip->dev, cpu_addr);
39
40 return cpu_addr;
41 }
42
dw_edma_alloc_burst(struct dw_edma_chunk * chunk)43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
44 {
45 struct dw_edma_burst *burst;
46
47 burst = kzalloc_obj(*burst, GFP_NOWAIT);
48 if (unlikely(!burst))
49 return NULL;
50
51 INIT_LIST_HEAD(&burst->list);
52 if (chunk->burst) {
53 /* Create and add new element into the linked list */
54 chunk->bursts_alloc++;
55 list_add_tail(&burst->list, &chunk->burst->list);
56 } else {
57 /* List head */
58 chunk->bursts_alloc = 0;
59 chunk->burst = burst;
60 }
61
62 return burst;
63 }
64
dw_edma_alloc_chunk(struct dw_edma_desc * desc)65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
66 {
67 struct dw_edma_chip *chip = desc->chan->dw->chip;
68 struct dw_edma_chan *chan = desc->chan;
69 struct dw_edma_chunk *chunk;
70
71 chunk = kzalloc_obj(*chunk, GFP_NOWAIT);
72 if (unlikely(!chunk))
73 return NULL;
74
75 INIT_LIST_HEAD(&chunk->list);
76 chunk->chan = chan;
77 /* Toggling change bit (CB) in each chunk, this is a mechanism to
78 * inform the eDMA HW block that this is a new linked list ready
79 * to be consumed.
80 * - Odd chunks originate CB equal to 0
81 * - Even chunks originate CB equal to 1
82 */
83 chunk->cb = !(desc->chunks_alloc % 2);
84 if (chan->dir == EDMA_DIR_WRITE) {
85 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
86 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
87 } else {
88 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
89 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
90 }
91
92 if (desc->chunk) {
93 /* Create and add new element into the linked list */
94 if (!dw_edma_alloc_burst(chunk)) {
95 kfree(chunk);
96 return NULL;
97 }
98 desc->chunks_alloc++;
99 list_add_tail(&chunk->list, &desc->chunk->list);
100 } else {
101 /* List head */
102 chunk->burst = NULL;
103 desc->chunks_alloc = 0;
104 desc->chunk = chunk;
105 }
106
107 return chunk;
108 }
109
dw_edma_alloc_desc(struct dw_edma_chan * chan)110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
111 {
112 struct dw_edma_desc *desc;
113
114 desc = kzalloc_obj(*desc, GFP_NOWAIT);
115 if (unlikely(!desc))
116 return NULL;
117
118 desc->chan = chan;
119 if (!dw_edma_alloc_chunk(desc)) {
120 kfree(desc);
121 return NULL;
122 }
123
124 return desc;
125 }
126
dw_edma_free_burst(struct dw_edma_chunk * chunk)127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
128 {
129 struct dw_edma_burst *child, *_next;
130
131 /* Remove all the list elements */
132 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
133 list_del(&child->list);
134 kfree(child);
135 chunk->bursts_alloc--;
136 }
137
138 /* Remove the list head */
139 kfree(child);
140 chunk->burst = NULL;
141 }
142
dw_edma_free_chunk(struct dw_edma_desc * desc)143 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
144 {
145 struct dw_edma_chunk *child, *_next;
146
147 if (!desc->chunk)
148 return;
149
150 /* Remove all the list elements */
151 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
152 dw_edma_free_burst(child);
153 list_del(&child->list);
154 kfree(child);
155 desc->chunks_alloc--;
156 }
157
158 /* Remove the list head */
159 kfree(child);
160 desc->chunk = NULL;
161 }
162
dw_edma_free_desc(struct dw_edma_desc * desc)163 static void dw_edma_free_desc(struct dw_edma_desc *desc)
164 {
165 dw_edma_free_chunk(desc);
166 kfree(desc);
167 }
168
vchan_free_desc(struct virt_dma_desc * vdesc)169 static void vchan_free_desc(struct virt_dma_desc *vdesc)
170 {
171 dw_edma_free_desc(vd2dw_edma_desc(vdesc));
172 }
173
dw_edma_start_transfer(struct dw_edma_chan * chan)174 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
175 {
176 struct dw_edma *dw = chan->dw;
177 struct dw_edma_chunk *child;
178 struct dw_edma_desc *desc;
179 struct virt_dma_desc *vd;
180
181 vd = vchan_next_desc(&chan->vc);
182 if (!vd)
183 return 0;
184
185 desc = vd2dw_edma_desc(vd);
186 if (!desc)
187 return 0;
188
189 child = list_first_entry_or_null(&desc->chunk->list,
190 struct dw_edma_chunk, list);
191 if (!child)
192 return 0;
193
194 dw_edma_core_start(dw, child, !desc->xfer_sz);
195 desc->xfer_sz += child->ll_region.sz;
196 dw_edma_free_burst(child);
197 list_del(&child->list);
198 kfree(child);
199 desc->chunks_alloc--;
200
201 return 1;
202 }
203
dw_edma_device_caps(struct dma_chan * dchan,struct dma_slave_caps * caps)204 static void dw_edma_device_caps(struct dma_chan *dchan,
205 struct dma_slave_caps *caps)
206 {
207 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
208
209 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
210 if (chan->dir == EDMA_DIR_READ)
211 caps->directions = BIT(DMA_DEV_TO_MEM);
212 else
213 caps->directions = BIT(DMA_MEM_TO_DEV);
214 } else {
215 if (chan->dir == EDMA_DIR_WRITE)
216 caps->directions = BIT(DMA_DEV_TO_MEM);
217 else
218 caps->directions = BIT(DMA_MEM_TO_DEV);
219 }
220 }
221
dw_edma_device_config(struct dma_chan * dchan,struct dma_slave_config * config)222 static int dw_edma_device_config(struct dma_chan *dchan,
223 struct dma_slave_config *config)
224 {
225 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
226 bool cfg_non_ll;
227 int non_ll = 0;
228
229 chan->non_ll = false;
230 if (chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE) {
231 if (config->peripheral_config &&
232 config->peripheral_size != sizeof(int)) {
233 dev_err(dchan->device->dev,
234 "config param peripheral size mismatch\n");
235 return -EINVAL;
236 }
237
238 /*
239 * When there is no valid LLP base address available then the
240 * default DMA ops will use the non-LL mode.
241 *
242 * Cases where LL mode is enabled and client wants to use the
243 * non-LL mode then also client can do so via providing the
244 * peripheral_config param.
245 */
246 cfg_non_ll = chan->dw->chip->cfg_non_ll;
247 if (config->peripheral_config) {
248 non_ll = *(int *)config->peripheral_config;
249
250 if (cfg_non_ll && !non_ll) {
251 dev_err(dchan->device->dev, "invalid configuration\n");
252 return -EINVAL;
253 }
254 }
255
256 if (cfg_non_ll || non_ll)
257 chan->non_ll = true;
258 } else if (config->peripheral_config) {
259 dev_err(dchan->device->dev,
260 "peripheral config param applicable only for HDMA\n");
261 return -EINVAL;
262 }
263
264 memcpy(&chan->config, config, sizeof(*config));
265 chan->configured = true;
266
267 return 0;
268 }
269
dw_edma_device_pause(struct dma_chan * dchan)270 static int dw_edma_device_pause(struct dma_chan *dchan)
271 {
272 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
273 int err = 0;
274
275 if (!chan->configured)
276 err = -EPERM;
277 else if (chan->status != EDMA_ST_BUSY)
278 err = -EPERM;
279 else if (chan->request != EDMA_REQ_NONE)
280 err = -EPERM;
281 else
282 chan->request = EDMA_REQ_PAUSE;
283
284 return err;
285 }
286
dw_edma_device_resume(struct dma_chan * dchan)287 static int dw_edma_device_resume(struct dma_chan *dchan)
288 {
289 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
290 int err = 0;
291
292 if (!chan->configured) {
293 err = -EPERM;
294 } else if (chan->status != EDMA_ST_PAUSE) {
295 err = -EPERM;
296 } else if (chan->request != EDMA_REQ_NONE) {
297 err = -EPERM;
298 } else {
299 chan->status = EDMA_ST_BUSY;
300 dw_edma_start_transfer(chan);
301 }
302
303 return err;
304 }
305
dw_edma_device_terminate_all(struct dma_chan * dchan)306 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
307 {
308 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
309 int err = 0;
310
311 if (!chan->configured) {
312 /* Do nothing */
313 } else if (chan->status == EDMA_ST_PAUSE) {
314 chan->status = EDMA_ST_IDLE;
315 chan->configured = false;
316 } else if (chan->status == EDMA_ST_IDLE) {
317 chan->configured = false;
318 } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
319 /*
320 * The channel is in a false BUSY state, probably didn't
321 * receive or lost an interrupt
322 */
323 chan->status = EDMA_ST_IDLE;
324 chan->configured = false;
325 } else if (chan->request > EDMA_REQ_PAUSE) {
326 err = -EPERM;
327 } else {
328 chan->request = EDMA_REQ_STOP;
329 }
330
331 return err;
332 }
333
dw_edma_device_issue_pending(struct dma_chan * dchan)334 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
335 {
336 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
337 unsigned long flags;
338
339 if (!chan->configured)
340 return;
341
342 spin_lock_irqsave(&chan->vc.lock, flags);
343 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&
344 chan->status == EDMA_ST_IDLE) {
345 chan->status = EDMA_ST_BUSY;
346 dw_edma_start_transfer(chan);
347 }
348 spin_unlock_irqrestore(&chan->vc.lock, flags);
349 }
350
351 static enum dma_status
dw_edma_device_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * txstate)352 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
353 struct dma_tx_state *txstate)
354 {
355 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
356 struct dw_edma_desc *desc;
357 struct virt_dma_desc *vd;
358 unsigned long flags;
359 enum dma_status ret;
360 u32 residue = 0;
361
362 ret = dma_cookie_status(dchan, cookie, txstate);
363 if (ret == DMA_COMPLETE)
364 return ret;
365
366 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
367 ret = DMA_PAUSED;
368
369 if (!txstate)
370 goto ret_residue;
371
372 spin_lock_irqsave(&chan->vc.lock, flags);
373 vd = vchan_find_desc(&chan->vc, cookie);
374 if (vd) {
375 desc = vd2dw_edma_desc(vd);
376 if (desc)
377 residue = desc->alloc_sz - desc->xfer_sz;
378 }
379 spin_unlock_irqrestore(&chan->vc.lock, flags);
380
381 ret_residue:
382 dma_set_residue(txstate, residue);
383
384 return ret;
385 }
386
387 static struct dma_async_tx_descriptor *
dw_edma_device_transfer(struct dw_edma_transfer * xfer)388 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
389 {
390 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
391 enum dma_transfer_direction dir = xfer->direction;
392 struct scatterlist *sg = NULL;
393 struct dw_edma_chunk *chunk;
394 struct dw_edma_burst *burst;
395 struct dw_edma_desc *desc;
396 u64 src_addr, dst_addr;
397 size_t fsz = 0;
398 u32 bursts_max;
399 u32 cnt = 0;
400 int i;
401
402 if (!chan->configured)
403 return NULL;
404
405 /*
406 * Local Root Port/End-point Remote End-point
407 * +-----------------------+ PCIe bus +----------------------+
408 * | | +-+ | |
409 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM |
410 * | | | | | |
411 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV |
412 * | | +-+ | |
413 * +-----------------------+ +----------------------+
414 *
415 * 1. Normal logic:
416 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the
417 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used
418 * for the device read operations (DEV_TO_MEM) and the Tx channel
419 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV).
420 *
421 * 2. Inverted logic:
422 * If eDMA is embedded into a Remote PCIe EP and is controlled by the
423 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx
424 * channel (EDMA_DIR_WRITE) will be used for the device read operations
425 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write
426 * operations (MEM_TO_DEV).
427 *
428 * It is the client driver responsibility to choose a proper channel
429 * for the DMA transfers.
430 */
431 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
432 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) ||
433 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV))
434 return NULL;
435 } else {
436 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) ||
437 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV))
438 return NULL;
439 }
440
441 if (xfer->type == EDMA_XFER_CYCLIC) {
442 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
443 return NULL;
444 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
445 if (xfer->xfer.sg.len < 1)
446 return NULL;
447 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
448 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)
449 return NULL;
450 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
451 return NULL;
452 } else {
453 return NULL;
454 }
455
456 /*
457 * For non-LL mode, only a single burst can be handled
458 * in a single chunk unlike LL mode where multiple bursts
459 * can be configured in a single chunk.
460 */
461 bursts_max = chan->non_ll ? 1 : chan->ll_max;
462
463 desc = dw_edma_alloc_desc(chan);
464 if (unlikely(!desc))
465 goto err_alloc;
466
467 chunk = dw_edma_alloc_chunk(desc);
468 if (unlikely(!chunk))
469 goto err_alloc;
470
471 if (xfer->type == EDMA_XFER_INTERLEAVED) {
472 src_addr = xfer->xfer.il->src_start;
473 dst_addr = xfer->xfer.il->dst_start;
474 } else {
475 src_addr = chan->config.src_addr;
476 dst_addr = chan->config.dst_addr;
477 }
478
479 if (dir == DMA_DEV_TO_MEM)
480 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
481 else
482 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
483
484 if (xfer->type == EDMA_XFER_CYCLIC) {
485 cnt = xfer->xfer.cyclic.cnt;
486 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
487 cnt = xfer->xfer.sg.len;
488 sg = xfer->xfer.sg.sgl;
489 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
490 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
491 fsz = xfer->xfer.il->frame_size;
492 }
493
494 for (i = 0; i < cnt; i++) {
495 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
496 break;
497
498 if (chunk->bursts_alloc == bursts_max) {
499 chunk = dw_edma_alloc_chunk(desc);
500 if (unlikely(!chunk))
501 goto err_alloc;
502 }
503
504 burst = dw_edma_alloc_burst(chunk);
505 if (unlikely(!burst))
506 goto err_alloc;
507
508 if (xfer->type == EDMA_XFER_CYCLIC)
509 burst->sz = xfer->xfer.cyclic.len;
510 else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
511 burst->sz = sg_dma_len(sg);
512 else if (xfer->type == EDMA_XFER_INTERLEAVED)
513 burst->sz = xfer->xfer.il->sgl[i % fsz].size;
514
515 chunk->ll_region.sz += burst->sz;
516 desc->alloc_sz += burst->sz;
517
518 if (dir == DMA_DEV_TO_MEM) {
519 burst->sar = src_addr;
520 if (xfer->type == EDMA_XFER_CYCLIC) {
521 burst->dar = xfer->xfer.cyclic.paddr;
522 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
523 src_addr += sg_dma_len(sg);
524 burst->dar = sg_dma_address(sg);
525 /* Unlike the typical assumption by other
526 * drivers/IPs the peripheral memory isn't
527 * a FIFO memory, in this case, it's a
528 * linear memory and that why the source
529 * and destination addresses are increased
530 * by the same portion (data length)
531 */
532 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
533 burst->dar = dst_addr;
534 }
535 } else {
536 burst->dar = dst_addr;
537 if (xfer->type == EDMA_XFER_CYCLIC) {
538 burst->sar = xfer->xfer.cyclic.paddr;
539 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
540 dst_addr += sg_dma_len(sg);
541 burst->sar = sg_dma_address(sg);
542 /* Unlike the typical assumption by other
543 * drivers/IPs the peripheral memory isn't
544 * a FIFO memory, in this case, it's a
545 * linear memory and that why the source
546 * and destination addresses are increased
547 * by the same portion (data length)
548 */
549 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
550 burst->sar = src_addr;
551 }
552 }
553
554 if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
555 sg = sg_next(sg);
556 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
557 struct dma_interleaved_template *il = xfer->xfer.il;
558 struct data_chunk *dc = &il->sgl[i % fsz];
559
560 src_addr += burst->sz;
561 if (il->src_sgl)
562 src_addr += dmaengine_get_src_icg(il, dc);
563
564 dst_addr += burst->sz;
565 if (il->dst_sgl)
566 dst_addr += dmaengine_get_dst_icg(il, dc);
567 }
568 }
569
570 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
571
572 err_alloc:
573 if (desc)
574 dw_edma_free_desc(desc);
575
576 return NULL;
577 }
578
579 static struct dma_async_tx_descriptor *
dw_edma_device_prep_slave_sg(struct dma_chan * dchan,struct scatterlist * sgl,unsigned int len,enum dma_transfer_direction direction,unsigned long flags,void * context)580 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
581 unsigned int len,
582 enum dma_transfer_direction direction,
583 unsigned long flags, void *context)
584 {
585 struct dw_edma_transfer xfer;
586
587 xfer.dchan = dchan;
588 xfer.direction = direction;
589 xfer.xfer.sg.sgl = sgl;
590 xfer.xfer.sg.len = len;
591 xfer.flags = flags;
592 xfer.type = EDMA_XFER_SCATTER_GATHER;
593
594 return dw_edma_device_transfer(&xfer);
595 }
596
597 static struct dma_async_tx_descriptor *
dw_edma_device_prep_dma_cyclic(struct dma_chan * dchan,dma_addr_t paddr,size_t len,size_t count,enum dma_transfer_direction direction,unsigned long flags)598 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
599 size_t len, size_t count,
600 enum dma_transfer_direction direction,
601 unsigned long flags)
602 {
603 struct dw_edma_transfer xfer;
604
605 xfer.dchan = dchan;
606 xfer.direction = direction;
607 xfer.xfer.cyclic.paddr = paddr;
608 xfer.xfer.cyclic.len = len;
609 xfer.xfer.cyclic.cnt = count;
610 xfer.flags = flags;
611 xfer.type = EDMA_XFER_CYCLIC;
612
613 return dw_edma_device_transfer(&xfer);
614 }
615
616 static struct dma_async_tx_descriptor *
dw_edma_device_prep_interleaved_dma(struct dma_chan * dchan,struct dma_interleaved_template * ilt,unsigned long flags)617 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
618 struct dma_interleaved_template *ilt,
619 unsigned long flags)
620 {
621 struct dw_edma_transfer xfer;
622
623 xfer.dchan = dchan;
624 xfer.direction = ilt->dir;
625 xfer.xfer.il = ilt;
626 xfer.flags = flags;
627 xfer.type = EDMA_XFER_INTERLEAVED;
628
629 return dw_edma_device_transfer(&xfer);
630 }
631
dw_hdma_set_callback_result(struct virt_dma_desc * vd,enum dmaengine_tx_result result)632 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
633 enum dmaengine_tx_result result)
634 {
635 u32 residue = 0;
636 struct dw_edma_desc *desc;
637 struct dmaengine_result *res;
638
639 if (!vd->tx.callback_result)
640 return;
641
642 desc = vd2dw_edma_desc(vd);
643 if (desc)
644 residue = desc->alloc_sz - desc->xfer_sz;
645
646 res = &vd->tx_result;
647 res->result = result;
648 res->residue = residue;
649 }
650
dw_edma_done_interrupt(struct dw_edma_chan * chan)651 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
652 {
653 struct dw_edma_desc *desc;
654 struct virt_dma_desc *vd;
655 unsigned long flags;
656
657 spin_lock_irqsave(&chan->vc.lock, flags);
658 vd = vchan_next_desc(&chan->vc);
659 if (vd) {
660 switch (chan->request) {
661 case EDMA_REQ_NONE:
662 desc = vd2dw_edma_desc(vd);
663 if (!desc->chunks_alloc) {
664 dw_hdma_set_callback_result(vd,
665 DMA_TRANS_NOERROR);
666 list_del(&vd->node);
667 vchan_cookie_complete(vd);
668 }
669
670 /* Continue transferring if there are remaining chunks or issued requests.
671 */
672 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
673 break;
674
675 case EDMA_REQ_STOP:
676 list_del(&vd->node);
677 vchan_cookie_complete(vd);
678 chan->request = EDMA_REQ_NONE;
679 chan->status = EDMA_ST_IDLE;
680 break;
681
682 case EDMA_REQ_PAUSE:
683 chan->request = EDMA_REQ_NONE;
684 chan->status = EDMA_ST_PAUSE;
685 break;
686
687 default:
688 break;
689 }
690 }
691 spin_unlock_irqrestore(&chan->vc.lock, flags);
692 }
693
dw_edma_abort_interrupt(struct dw_edma_chan * chan)694 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
695 {
696 struct virt_dma_desc *vd;
697 unsigned long flags;
698
699 spin_lock_irqsave(&chan->vc.lock, flags);
700 vd = vchan_next_desc(&chan->vc);
701 if (vd) {
702 dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED);
703 list_del(&vd->node);
704 vchan_cookie_complete(vd);
705 }
706 spin_unlock_irqrestore(&chan->vc.lock, flags);
707 chan->request = EDMA_REQ_NONE;
708 chan->status = EDMA_ST_IDLE;
709 }
710
dw_edma_emul_irq_ack(struct irq_data * d)711 static void dw_edma_emul_irq_ack(struct irq_data *d)
712 {
713 struct dw_edma *dw = irq_data_get_irq_chip_data(d);
714
715 dw_edma_core_ack_emulated_irq(dw);
716 }
717
718 /*
719 * irq_chip implementation for interrupt-emulation doorbells.
720 *
721 * The emulated source has no mask/unmask mechanism. With handle_level_irq(),
722 * the flow is therefore:
723 * 1) .irq_ack() deasserts the source
724 * 2) registered handlers (if any) are dispatched
725 * Since deassertion is already done in .irq_ack(), handlers do not need to take
726 * care of it, hence IRQCHIP_ONESHOT_SAFE.
727 */
728 static struct irq_chip dw_edma_emul_irqchip = {
729 .name = "dw-edma-emul",
730 .irq_ack = dw_edma_emul_irq_ack,
731 .flags = IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE,
732 };
733
dw_edma_emul_irq_alloc(struct dw_edma * dw)734 static int dw_edma_emul_irq_alloc(struct dw_edma *dw)
735 {
736 struct dw_edma_chip *chip = dw->chip;
737 int virq;
738
739 chip->db_irq = 0;
740 chip->db_offset = ~0;
741
742 /*
743 * Only meaningful when the core provides the deassert sequence
744 * for interrupt emulation.
745 */
746 if (!dw->core->ack_emulated_irq)
747 return 0;
748
749 /*
750 * Allocate a single, requestable Linux virtual IRQ number.
751 * Use >= 1 so that 0 can remain a "not available" sentinel.
752 */
753 virq = irq_alloc_desc(NUMA_NO_NODE);
754 if (virq < 0)
755 return virq;
756
757 irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq);
758 irq_set_chip_data(virq, dw);
759 irq_set_noprobe(virq);
760
761 chip->db_irq = virq;
762 chip->db_offset = dw_edma_core_db_offset(dw);
763
764 return 0;
765 }
766
dw_edma_emul_irq_free(struct dw_edma * dw)767 static void dw_edma_emul_irq_free(struct dw_edma *dw)
768 {
769 struct dw_edma_chip *chip = dw->chip;
770
771 if (!chip)
772 return;
773 if (chip->db_irq <= 0)
774 return;
775
776 irq_free_descs(chip->db_irq, 1);
777 chip->db_irq = 0;
778 chip->db_offset = ~0;
779 }
780
dw_edma_interrupt_emulated(void * data)781 static inline irqreturn_t dw_edma_interrupt_emulated(void *data)
782 {
783 struct dw_edma_irq *dw_irq = data;
784 struct dw_edma *dw = dw_irq->dw;
785 int db_irq = dw->chip->db_irq;
786
787 if (db_irq > 0) {
788 /*
789 * Interrupt emulation may assert the IRQ line without updating the
790 * normal DONE/ABORT status bits. With a shared IRQ handler we
791 * cannot reliably detect such events by status registers alone, so
792 * always perform the core-specific deassert sequence.
793 */
794 generic_handle_irq(db_irq);
795 return IRQ_HANDLED;
796 }
797 return IRQ_NONE;
798 }
799
dw_edma_interrupt_write_inner(int irq,void * data)800 static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data)
801 {
802 struct dw_edma_irq *dw_irq = data;
803
804 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
805 dw_edma_done_interrupt,
806 dw_edma_abort_interrupt);
807 }
808
dw_edma_interrupt_read_inner(int irq,void * data)809 static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
810 {
811 struct dw_edma_irq *dw_irq = data;
812
813 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
814 dw_edma_done_interrupt,
815 dw_edma_abort_interrupt);
816 }
817
dw_edma_interrupt_write(int irq,void * data)818 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
819 {
820 irqreturn_t ret = IRQ_NONE;
821
822 ret |= dw_edma_interrupt_write_inner(irq, data);
823 ret |= dw_edma_interrupt_emulated(data);
824
825 return ret;
826 }
827
dw_edma_interrupt_read(int irq,void * data)828 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
829 {
830 irqreturn_t ret = IRQ_NONE;
831
832 ret |= dw_edma_interrupt_read_inner(irq, data);
833 ret |= dw_edma_interrupt_emulated(data);
834
835 return ret;
836 }
837
dw_edma_interrupt_common(int irq,void * data)838 static inline irqreturn_t dw_edma_interrupt_common(int irq, void *data)
839 {
840 irqreturn_t ret = IRQ_NONE;
841
842 ret |= dw_edma_interrupt_write_inner(irq, data);
843 ret |= dw_edma_interrupt_read_inner(irq, data);
844 ret |= dw_edma_interrupt_emulated(data);
845
846 return ret;
847 }
848
dw_edma_alloc_chan_resources(struct dma_chan * dchan)849 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
850 {
851 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
852
853 if (chan->status != EDMA_ST_IDLE)
854 return -EBUSY;
855
856 return 0;
857 }
858
dw_edma_free_chan_resources(struct dma_chan * dchan)859 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
860 {
861 unsigned long timeout = jiffies + msecs_to_jiffies(5000);
862 int ret;
863
864 while (time_before(jiffies, timeout)) {
865 ret = dw_edma_device_terminate_all(dchan);
866 if (!ret)
867 break;
868
869 if (time_after_eq(jiffies, timeout))
870 return;
871
872 cpu_relax();
873 }
874 }
875
dw_edma_channel_setup(struct dw_edma * dw,u32 wr_alloc,u32 rd_alloc)876 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
877 {
878 struct dw_edma_chip *chip = dw->chip;
879 struct device *dev = chip->dev;
880 struct dw_edma_chan *chan;
881 struct dw_edma_irq *irq;
882 struct dma_device *dma;
883 u32 i, ch_cnt;
884 u32 pos;
885
886 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
887 dma = &dw->dma;
888
889 INIT_LIST_HEAD(&dma->channels);
890
891 for (i = 0; i < ch_cnt; i++) {
892 chan = &dw->chan[i];
893
894 chan->dw = dw;
895
896 if (i < dw->wr_ch_cnt) {
897 chan->id = i;
898 chan->dir = EDMA_DIR_WRITE;
899 } else {
900 chan->id = i - dw->wr_ch_cnt;
901 chan->dir = EDMA_DIR_READ;
902 }
903
904 chan->configured = false;
905 chan->request = EDMA_REQ_NONE;
906 chan->status = EDMA_ST_IDLE;
907
908 if (chan->dir == EDMA_DIR_WRITE)
909 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
910 else
911 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
912 chan->ll_max -= 1;
913
914 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
915 str_write_read(chan->dir == EDMA_DIR_WRITE),
916 chan->id, chan->ll_max);
917
918 if (dw->nr_irqs == 1)
919 pos = 0;
920 else if (chan->dir == EDMA_DIR_WRITE)
921 pos = chan->id % wr_alloc;
922 else
923 pos = wr_alloc + chan->id % rd_alloc;
924
925 irq = &dw->irq[pos];
926
927 if (chan->dir == EDMA_DIR_WRITE)
928 irq->wr_mask |= BIT(chan->id);
929 else
930 irq->rd_mask |= BIT(chan->id);
931
932 irq->dw = dw;
933 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
934
935 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
936 str_write_read(chan->dir == EDMA_DIR_WRITE),
937 chan->id,
938 chan->msi.address_hi, chan->msi.address_lo,
939 chan->msi.data);
940
941 chan->vc.desc_free = vchan_free_desc;
942 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?
943 &dw->chip->dt_region_wr[chan->id] :
944 &dw->chip->dt_region_rd[chan->id];
945
946 vchan_init(&chan->vc, dma);
947
948 dw_edma_core_ch_config(chan);
949 }
950
951 /* Set DMA channel capabilities */
952 dma_cap_zero(dma->cap_mask);
953 dma_cap_set(DMA_SLAVE, dma->cap_mask);
954 dma_cap_set(DMA_CYCLIC, dma->cap_mask);
955 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
956 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
957 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
958 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
959 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
960 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
961
962 /* Set DMA channel callbacks */
963 dma->dev = chip->dev;
964 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
965 dma->device_free_chan_resources = dw_edma_free_chan_resources;
966 dma->device_caps = dw_edma_device_caps;
967 dma->device_config = dw_edma_device_config;
968 dma->device_pause = dw_edma_device_pause;
969 dma->device_resume = dw_edma_device_resume;
970 dma->device_terminate_all = dw_edma_device_terminate_all;
971 dma->device_issue_pending = dw_edma_device_issue_pending;
972 dma->device_tx_status = dw_edma_device_tx_status;
973 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
974 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
975 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
976
977 dma_set_max_seg_size(dma->dev, U32_MAX);
978
979 /* Register DMA device */
980 return dma_async_device_register(dma);
981 }
982
dw_edma_dec_irq_alloc(int * nr_irqs,u32 * alloc,u16 cnt)983 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
984 {
985 if (*nr_irqs && *alloc < cnt) {
986 (*alloc)++;
987 (*nr_irqs)--;
988 }
989 }
990
dw_edma_add_irq_mask(u32 * mask,u32 alloc,u16 cnt)991 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
992 {
993 while (*mask * alloc < cnt)
994 (*mask)++;
995 }
996
dw_edma_irq_request(struct dw_edma * dw,u32 * wr_alloc,u32 * rd_alloc)997 static int dw_edma_irq_request(struct dw_edma *dw,
998 u32 *wr_alloc, u32 *rd_alloc)
999 {
1000 struct dw_edma_chip *chip = dw->chip;
1001 struct device *dev = dw->chip->dev;
1002 struct msi_desc *msi_desc;
1003 u32 wr_mask = 1;
1004 u32 rd_mask = 1;
1005 int i, err = 0;
1006 u32 ch_cnt;
1007 int irq;
1008
1009 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
1010
1011 if (chip->nr_irqs < 1 || !chip->ops->irq_vector)
1012 return -EINVAL;
1013
1014 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL);
1015 if (!dw->irq)
1016 return -ENOMEM;
1017
1018 if (chip->nr_irqs == 1) {
1019 /* Common IRQ shared among all channels */
1020 irq = chip->ops->irq_vector(dev, 0);
1021 err = request_irq(irq, dw_edma_interrupt_common,
1022 IRQF_SHARED, dw->name, &dw->irq[0]);
1023 if (err) {
1024 dw->nr_irqs = 0;
1025 return err;
1026 }
1027
1028 if (irq_get_msi_desc(irq))
1029 get_cached_msi_msg(irq, &dw->irq[0].msi);
1030
1031 dw->nr_irqs = 1;
1032 } else {
1033 /* Distribute IRQs equally among all channels */
1034 int tmp = chip->nr_irqs;
1035
1036 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
1037 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
1038 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
1039 }
1040
1041 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
1042 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
1043
1044 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
1045 irq = chip->ops->irq_vector(dev, i);
1046 err = request_irq(irq,
1047 i < *wr_alloc ?
1048 dw_edma_interrupt_write :
1049 dw_edma_interrupt_read,
1050 IRQF_SHARED, dw->name,
1051 &dw->irq[i]);
1052 if (err)
1053 goto err_irq_free;
1054 msi_desc = irq_get_msi_desc(irq);
1055 if (msi_desc) {
1056 get_cached_msi_msg(irq, &dw->irq[i].msi);
1057 if (!msi_desc->pci.msi_attrib.is_msix)
1058 dw->irq[i].msi.data = dw->irq[0].msi.data + i;
1059 }
1060 }
1061
1062 dw->nr_irqs = i;
1063 }
1064
1065 return 0;
1066
1067 err_irq_free:
1068 for (i--; i >= 0; i--) {
1069 irq = chip->ops->irq_vector(dev, i);
1070 free_irq(irq, &dw->irq[i]);
1071 }
1072
1073 return err;
1074 }
1075
dw_edma_probe(struct dw_edma_chip * chip)1076 int dw_edma_probe(struct dw_edma_chip *chip)
1077 {
1078 struct device *dev;
1079 struct dw_edma *dw;
1080 u32 wr_alloc = 0;
1081 u32 rd_alloc = 0;
1082 int i, err;
1083
1084 if (!chip)
1085 return -EINVAL;
1086
1087 dev = chip->dev;
1088 if (!dev || !chip->ops)
1089 return -EINVAL;
1090
1091 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
1092 if (!dw)
1093 return -ENOMEM;
1094
1095 dw->chip = chip;
1096
1097 if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
1098 dw_hdma_v0_core_register(dw);
1099 else
1100 dw_edma_v0_core_register(dw);
1101
1102 raw_spin_lock_init(&dw->lock);
1103
1104 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
1105 dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));
1106 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
1107
1108 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
1109 dw_edma_core_ch_count(dw, EDMA_DIR_READ));
1110 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
1111
1112 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
1113 return -EINVAL;
1114
1115 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
1116 dw->wr_ch_cnt, dw->rd_ch_cnt);
1117
1118 /* Allocate channels */
1119 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
1120 sizeof(*dw->chan), GFP_KERNEL);
1121 if (!dw->chan)
1122 return -ENOMEM;
1123
1124 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
1125 dev_name(chip->dev));
1126
1127 /* Disable eDMA, only to establish the ideal initial conditions */
1128 dw_edma_core_off(dw);
1129
1130 /* Request IRQs */
1131 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);
1132 if (err)
1133 return err;
1134
1135 /* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */
1136 err = dw_edma_emul_irq_alloc(dw);
1137 if (err)
1138 dev_warn(dev, "Failed to allocate emulation IRQ: %d\n", err);
1139
1140 /* Setup write/read channels */
1141 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
1142 if (err)
1143 goto err_irq_free;
1144
1145 /* Turn debugfs on */
1146 dw_edma_core_debugfs_on(dw);
1147
1148 chip->dw = dw;
1149
1150 return 0;
1151
1152 err_irq_free:
1153 for (i = (dw->nr_irqs - 1); i >= 0; i--)
1154 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
1155 dw_edma_emul_irq_free(dw);
1156
1157 return err;
1158 }
1159 EXPORT_SYMBOL_GPL(dw_edma_probe);
1160
dw_edma_remove(struct dw_edma_chip * chip)1161 int dw_edma_remove(struct dw_edma_chip *chip)
1162 {
1163 struct dw_edma_chan *chan, *_chan;
1164 struct device *dev = chip->dev;
1165 struct dw_edma *dw = chip->dw;
1166 int i;
1167
1168 /* Skip removal if no private data found */
1169 if (!dw)
1170 return -ENODEV;
1171
1172 /* Disable eDMA */
1173 dw_edma_core_off(dw);
1174
1175 /* Free irqs */
1176 for (i = (dw->nr_irqs - 1); i >= 0; i--)
1177 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
1178 dw_edma_emul_irq_free(dw);
1179
1180 /* Deregister eDMA device */
1181 dma_async_device_unregister(&dw->dma);
1182 list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
1183 vc.chan.device_node) {
1184 tasklet_kill(&chan->vc.task);
1185 list_del(&chan->vc.chan.device_node);
1186 }
1187
1188 return 0;
1189 }
1190 EXPORT_SYMBOL_GPL(dw_edma_remove);
1191
1192 MODULE_LICENSE("GPL v2");
1193 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
1194 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
1195