xref: /linux/drivers/dma/sf-pdma/sf-pdma.c (revision 06d07429858317ded2db7986113a9e0129cd599b)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * SiFive FU540 Platform DMA driver
4  * Copyright (C) 2019 SiFive
5  *
6  * Based partially on:
7  * - drivers/dma/fsl-edma.c
8  * - drivers/dma/dw-edma/
9  * - drivers/dma/pxa-dma.c
10  *
11  * See the following sources for further documentation:
12  * - Chapter 12 "Platform DMA Engine (PDMA)" of
13  *   SiFive FU540-C000 v1.0
14  *   https://static.dev.sifive.com/FU540-C000-v1.0.pdf
15  */
16 #include <linux/module.h>
17 #include <linux/device.h>
18 #include <linux/kernel.h>
19 #include <linux/platform_device.h>
20 #include <linux/mod_devicetable.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/of.h>
23 #include <linux/of_dma.h>
24 #include <linux/slab.h>
25 
26 #include "sf-pdma.h"
27 
28 #define PDMA_QUIRK_NO_STRICT_ORDERING   BIT(0)
29 
30 #ifndef readq
readq(void __iomem * addr)31 static inline unsigned long long readq(void __iomem *addr)
32 {
33 	return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL);
34 }
35 #endif
36 
37 #ifndef writeq
writeq(unsigned long long v,void __iomem * addr)38 static inline void writeq(unsigned long long v, void __iomem *addr)
39 {
40 	writel(lower_32_bits(v), addr);
41 	writel(upper_32_bits(v), addr + 4);
42 }
43 #endif
44 
to_sf_pdma_chan(struct dma_chan * dchan)45 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan)
46 {
47 	return container_of(dchan, struct sf_pdma_chan, vchan.chan);
48 }
49 
to_sf_pdma_desc(struct virt_dma_desc * vd)50 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd)
51 {
52 	return container_of(vd, struct sf_pdma_desc, vdesc);
53 }
54 
sf_pdma_alloc_desc(struct sf_pdma_chan * chan)55 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan)
56 {
57 	struct sf_pdma_desc *desc;
58 
59 	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
60 	if (!desc)
61 		return NULL;
62 
63 	desc->chan = chan;
64 
65 	return desc;
66 }
67 
sf_pdma_fill_desc(struct sf_pdma_desc * desc,u64 dst,u64 src,u64 size)68 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc,
69 			      u64 dst, u64 src, u64 size)
70 {
71 	desc->xfer_type =  desc->chan->pdma->transfer_type;
72 	desc->xfer_size = size;
73 	desc->dst_addr = dst;
74 	desc->src_addr = src;
75 }
76 
sf_pdma_disclaim_chan(struct sf_pdma_chan * chan)77 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan)
78 {
79 	struct pdma_regs *regs = &chan->regs;
80 
81 	writel(PDMA_CLEAR_CTRL, regs->ctrl);
82 }
83 
84 static struct dma_async_tx_descriptor *
sf_pdma_prep_dma_memcpy(struct dma_chan * dchan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)85 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan,	dma_addr_t dest, dma_addr_t src,
86 			size_t len, unsigned long flags)
87 {
88 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
89 	struct sf_pdma_desc *desc;
90 	unsigned long iflags;
91 
92 	if (chan && (!len || !dest || !src)) {
93 		dev_err(chan->pdma->dma_dev.dev,
94 			"Please check dma len, dest, src!\n");
95 		return NULL;
96 	}
97 
98 	desc = sf_pdma_alloc_desc(chan);
99 	if (!desc)
100 		return NULL;
101 
102 	desc->dirn = DMA_MEM_TO_MEM;
103 	desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
104 
105 	spin_lock_irqsave(&chan->vchan.lock, iflags);
106 	sf_pdma_fill_desc(desc, dest, src, len);
107 	spin_unlock_irqrestore(&chan->vchan.lock, iflags);
108 
109 	return desc->async_tx;
110 }
111 
sf_pdma_slave_config(struct dma_chan * dchan,struct dma_slave_config * cfg)112 static int sf_pdma_slave_config(struct dma_chan *dchan,
113 				struct dma_slave_config *cfg)
114 {
115 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
116 
117 	memcpy(&chan->cfg, cfg, sizeof(*cfg));
118 
119 	return 0;
120 }
121 
sf_pdma_alloc_chan_resources(struct dma_chan * dchan)122 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan)
123 {
124 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
125 	struct pdma_regs *regs = &chan->regs;
126 
127 	dma_cookie_init(dchan);
128 	writel(PDMA_CLAIM_MASK, regs->ctrl);
129 
130 	return 0;
131 }
132 
sf_pdma_disable_request(struct sf_pdma_chan * chan)133 static void sf_pdma_disable_request(struct sf_pdma_chan *chan)
134 {
135 	struct pdma_regs *regs = &chan->regs;
136 
137 	writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl);
138 }
139 
sf_pdma_free_chan_resources(struct dma_chan * dchan)140 static void sf_pdma_free_chan_resources(struct dma_chan *dchan)
141 {
142 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
143 	unsigned long flags;
144 	LIST_HEAD(head);
145 
146 	spin_lock_irqsave(&chan->vchan.lock, flags);
147 	sf_pdma_disable_request(chan);
148 	kfree(chan->desc);
149 	chan->desc = NULL;
150 	vchan_get_all_descriptors(&chan->vchan, &head);
151 	sf_pdma_disclaim_chan(chan);
152 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
153 	vchan_dma_desc_free_list(&chan->vchan, &head);
154 }
155 
sf_pdma_desc_residue(struct sf_pdma_chan * chan,dma_cookie_t cookie)156 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan,
157 				   dma_cookie_t cookie)
158 {
159 	struct virt_dma_desc *vd = NULL;
160 	struct pdma_regs *regs = &chan->regs;
161 	unsigned long flags;
162 	u64 residue = 0;
163 	struct sf_pdma_desc *desc;
164 	struct dma_async_tx_descriptor *tx = NULL;
165 
166 	spin_lock_irqsave(&chan->vchan.lock, flags);
167 
168 	list_for_each_entry(vd, &chan->vchan.desc_submitted, node)
169 		if (vd->tx.cookie == cookie)
170 			tx = &vd->tx;
171 
172 	if (!tx)
173 		goto out;
174 
175 	if (cookie == tx->chan->completed_cookie)
176 		goto out;
177 
178 	if (cookie == tx->cookie) {
179 		residue = readq(regs->residue);
180 	} else {
181 		vd = vchan_find_desc(&chan->vchan, cookie);
182 		if (!vd)
183 			goto out;
184 
185 		desc = to_sf_pdma_desc(vd);
186 		residue = desc->xfer_size;
187 	}
188 
189 out:
190 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
191 	return residue;
192 }
193 
194 static enum dma_status
sf_pdma_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * txstate)195 sf_pdma_tx_status(struct dma_chan *dchan,
196 		  dma_cookie_t cookie,
197 		  struct dma_tx_state *txstate)
198 {
199 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
200 	enum dma_status status;
201 
202 	status = dma_cookie_status(dchan, cookie, txstate);
203 
204 	if (txstate && status != DMA_ERROR)
205 		dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie));
206 
207 	return status;
208 }
209 
sf_pdma_terminate_all(struct dma_chan * dchan)210 static int sf_pdma_terminate_all(struct dma_chan *dchan)
211 {
212 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
213 	unsigned long flags;
214 	LIST_HEAD(head);
215 
216 	spin_lock_irqsave(&chan->vchan.lock, flags);
217 	sf_pdma_disable_request(chan);
218 	kfree(chan->desc);
219 	chan->desc = NULL;
220 	chan->xfer_err = false;
221 	vchan_get_all_descriptors(&chan->vchan, &head);
222 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
223 	vchan_dma_desc_free_list(&chan->vchan, &head);
224 
225 	return 0;
226 }
227 
sf_pdma_enable_request(struct sf_pdma_chan * chan)228 static void sf_pdma_enable_request(struct sf_pdma_chan *chan)
229 {
230 	struct pdma_regs *regs = &chan->regs;
231 	u32 v;
232 
233 	v = PDMA_CLAIM_MASK |
234 		PDMA_ENABLE_DONE_INT_MASK |
235 		PDMA_ENABLE_ERR_INT_MASK |
236 		PDMA_RUN_MASK;
237 
238 	writel(v, regs->ctrl);
239 }
240 
sf_pdma_get_first_pending_desc(struct sf_pdma_chan * chan)241 static struct sf_pdma_desc *sf_pdma_get_first_pending_desc(struct sf_pdma_chan *chan)
242 {
243 	struct virt_dma_chan *vchan = &chan->vchan;
244 	struct virt_dma_desc *vdesc;
245 
246 	if (list_empty(&vchan->desc_issued))
247 		return NULL;
248 
249 	vdesc = list_first_entry(&vchan->desc_issued, struct virt_dma_desc, node);
250 
251 	return container_of(vdesc, struct sf_pdma_desc, vdesc);
252 }
253 
sf_pdma_xfer_desc(struct sf_pdma_chan * chan)254 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan)
255 {
256 	struct sf_pdma_desc *desc = chan->desc;
257 	struct pdma_regs *regs = &chan->regs;
258 
259 	if (!desc) {
260 		dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n");
261 		return;
262 	}
263 
264 	writel(desc->xfer_type, regs->xfer_type);
265 	writeq(desc->xfer_size, regs->xfer_size);
266 	writeq(desc->dst_addr, regs->dst_addr);
267 	writeq(desc->src_addr, regs->src_addr);
268 
269 	chan->desc = desc;
270 	chan->status = DMA_IN_PROGRESS;
271 	sf_pdma_enable_request(chan);
272 }
273 
sf_pdma_issue_pending(struct dma_chan * dchan)274 static void sf_pdma_issue_pending(struct dma_chan *dchan)
275 {
276 	struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
277 	unsigned long flags;
278 
279 	spin_lock_irqsave(&chan->vchan.lock, flags);
280 
281 	if (!chan->desc && vchan_issue_pending(&chan->vchan)) {
282 		/* vchan_issue_pending has made a check that desc in not NULL */
283 		chan->desc = sf_pdma_get_first_pending_desc(chan);
284 		sf_pdma_xfer_desc(chan);
285 	}
286 
287 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
288 }
289 
sf_pdma_free_desc(struct virt_dma_desc * vdesc)290 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc)
291 {
292 	struct sf_pdma_desc *desc;
293 
294 	desc = to_sf_pdma_desc(vdesc);
295 	kfree(desc);
296 }
297 
sf_pdma_donebh_tasklet(struct tasklet_struct * t)298 static void sf_pdma_donebh_tasklet(struct tasklet_struct *t)
299 {
300 	struct sf_pdma_chan *chan = from_tasklet(chan, t, done_tasklet);
301 	unsigned long flags;
302 
303 	spin_lock_irqsave(&chan->lock, flags);
304 	if (chan->xfer_err) {
305 		chan->retries = MAX_RETRY;
306 		chan->status = DMA_COMPLETE;
307 		chan->xfer_err = false;
308 	}
309 	spin_unlock_irqrestore(&chan->lock, flags);
310 
311 	spin_lock_irqsave(&chan->vchan.lock, flags);
312 	list_del(&chan->desc->vdesc.node);
313 	vchan_cookie_complete(&chan->desc->vdesc);
314 
315 	chan->desc = sf_pdma_get_first_pending_desc(chan);
316 	if (chan->desc)
317 		sf_pdma_xfer_desc(chan);
318 
319 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
320 }
321 
sf_pdma_errbh_tasklet(struct tasklet_struct * t)322 static void sf_pdma_errbh_tasklet(struct tasklet_struct *t)
323 {
324 	struct sf_pdma_chan *chan = from_tasklet(chan, t, err_tasklet);
325 	struct sf_pdma_desc *desc = chan->desc;
326 	unsigned long flags;
327 
328 	spin_lock_irqsave(&chan->lock, flags);
329 	if (chan->retries <= 0) {
330 		/* fail to recover */
331 		spin_unlock_irqrestore(&chan->lock, flags);
332 		dmaengine_desc_get_callback_invoke(desc->async_tx, NULL);
333 	} else {
334 		/* retry */
335 		chan->retries--;
336 		chan->xfer_err = true;
337 		chan->status = DMA_ERROR;
338 
339 		sf_pdma_enable_request(chan);
340 		spin_unlock_irqrestore(&chan->lock, flags);
341 	}
342 }
343 
sf_pdma_done_isr(int irq,void * dev_id)344 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
345 {
346 	struct sf_pdma_chan *chan = dev_id;
347 	struct pdma_regs *regs = &chan->regs;
348 	u64 residue;
349 
350 	spin_lock(&chan->vchan.lock);
351 	writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl);
352 	residue = readq(regs->residue);
353 
354 	if (!residue) {
355 		tasklet_hi_schedule(&chan->done_tasklet);
356 	} else {
357 		/* submit next trascatioin if possible */
358 		struct sf_pdma_desc *desc = chan->desc;
359 
360 		desc->src_addr += desc->xfer_size - residue;
361 		desc->dst_addr += desc->xfer_size - residue;
362 		desc->xfer_size = residue;
363 
364 		sf_pdma_xfer_desc(chan);
365 	}
366 
367 	spin_unlock(&chan->vchan.lock);
368 
369 	return IRQ_HANDLED;
370 }
371 
sf_pdma_err_isr(int irq,void * dev_id)372 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id)
373 {
374 	struct sf_pdma_chan *chan = dev_id;
375 	struct pdma_regs *regs = &chan->regs;
376 
377 	spin_lock(&chan->lock);
378 	writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl);
379 	spin_unlock(&chan->lock);
380 
381 	tasklet_schedule(&chan->err_tasklet);
382 
383 	return IRQ_HANDLED;
384 }
385 
386 /**
387  * sf_pdma_irq_init() - Init PDMA IRQ Handlers
388  * @pdev: pointer of platform_device
389  * @pdma: pointer of PDMA engine. Caller should check NULL
390  *
391  * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should
392  * make sure the pointer passed in are non-NULL. This function should be called
393  * only one time during the device probe.
394  *
395  * Context: Any context.
396  *
397  * Return:
398  * * 0		- OK to init all IRQ handlers
399  * * -EINVAL	- Fail to request IRQ
400  */
sf_pdma_irq_init(struct platform_device * pdev,struct sf_pdma * pdma)401 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma)
402 {
403 	int irq, r, i;
404 	struct sf_pdma_chan *chan;
405 
406 	for (i = 0; i < pdma->n_chans; i++) {
407 		chan = &pdma->chans[i];
408 
409 		irq = platform_get_irq(pdev, i * 2);
410 		if (irq < 0)
411 			return -EINVAL;
412 
413 		r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0,
414 				     dev_name(&pdev->dev), (void *)chan);
415 		if (r) {
416 			dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r);
417 			return -EINVAL;
418 		}
419 
420 		chan->txirq = irq;
421 
422 		irq = platform_get_irq(pdev, (i * 2) + 1);
423 		if (irq < 0)
424 			return -EINVAL;
425 
426 		r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0,
427 				     dev_name(&pdev->dev), (void *)chan);
428 		if (r) {
429 			dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r);
430 			return -EINVAL;
431 		}
432 
433 		chan->errirq = irq;
434 	}
435 
436 	return 0;
437 }
438 
439 /**
440  * sf_pdma_setup_chans() - Init settings of each channel
441  * @pdma: pointer of PDMA engine. Caller should check NULL
442  *
443  * Initialize all data structure and register base. Caller should make sure
444  * the pointer passed in are non-NULL. This function should be called only
445  * one time during the device probe.
446  *
447  * Context: Any context.
448  *
449  * Return: none
450  */
sf_pdma_setup_chans(struct sf_pdma * pdma)451 static void sf_pdma_setup_chans(struct sf_pdma *pdma)
452 {
453 	int i;
454 	struct sf_pdma_chan *chan;
455 
456 	INIT_LIST_HEAD(&pdma->dma_dev.channels);
457 
458 	for (i = 0; i < pdma->n_chans; i++) {
459 		chan = &pdma->chans[i];
460 
461 		chan->regs.ctrl =
462 			SF_PDMA_REG_BASE(i) + PDMA_CTRL;
463 		chan->regs.xfer_type =
464 			SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE;
465 		chan->regs.xfer_size =
466 			SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE;
467 		chan->regs.dst_addr =
468 			SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR;
469 		chan->regs.src_addr =
470 			SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR;
471 		chan->regs.act_type =
472 			SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE;
473 		chan->regs.residue =
474 			SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE;
475 		chan->regs.cur_dst_addr =
476 			SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR;
477 		chan->regs.cur_src_addr =
478 			SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR;
479 
480 		chan->pdma = pdma;
481 		chan->pm_state = RUNNING;
482 		chan->slave_id = i;
483 		chan->xfer_err = false;
484 		spin_lock_init(&chan->lock);
485 
486 		chan->vchan.desc_free = sf_pdma_free_desc;
487 		vchan_init(&chan->vchan, &pdma->dma_dev);
488 
489 		writel(PDMA_CLEAR_CTRL, chan->regs.ctrl);
490 
491 		tasklet_setup(&chan->done_tasklet, sf_pdma_donebh_tasklet);
492 		tasklet_setup(&chan->err_tasklet, sf_pdma_errbh_tasklet);
493 	}
494 }
495 
sf_pdma_probe(struct platform_device * pdev)496 static int sf_pdma_probe(struct platform_device *pdev)
497 {
498 	const struct sf_pdma_driver_platdata *ddata;
499 	struct sf_pdma *pdma;
500 	int ret, n_chans;
501 	const enum dma_slave_buswidth widths =
502 		DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
503 		DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES |
504 		DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES |
505 		DMA_SLAVE_BUSWIDTH_64_BYTES;
506 
507 	ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans);
508 	if (ret) {
509 		/* backwards-compatibility for no dma-channels property */
510 		dev_dbg(&pdev->dev, "set number of channels to default value: 4\n");
511 		n_chans = PDMA_MAX_NR_CH;
512 	} else if (n_chans > PDMA_MAX_NR_CH) {
513 		dev_err(&pdev->dev, "the number of channels exceeds the maximum\n");
514 		return -EINVAL;
515 	}
516 
517 	pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans),
518 			    GFP_KERNEL);
519 	if (!pdma)
520 		return -ENOMEM;
521 
522 	pdma->n_chans = n_chans;
523 
524 	pdma->transfer_type = PDMA_FULL_SPEED | PDMA_STRICT_ORDERING;
525 
526 	ddata  = device_get_match_data(&pdev->dev);
527 	if (ddata) {
528 		if (ddata->quirks & PDMA_QUIRK_NO_STRICT_ORDERING)
529 			pdma->transfer_type &= ~PDMA_STRICT_ORDERING;
530 	}
531 
532 	pdma->membase = devm_platform_ioremap_resource(pdev, 0);
533 	if (IS_ERR(pdma->membase))
534 		return PTR_ERR(pdma->membase);
535 
536 	ret = sf_pdma_irq_init(pdev, pdma);
537 	if (ret)
538 		return ret;
539 
540 	sf_pdma_setup_chans(pdma);
541 
542 	pdma->dma_dev.dev = &pdev->dev;
543 
544 	/* Setup capability */
545 	dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask);
546 	pdma->dma_dev.copy_align = 2;
547 	pdma->dma_dev.src_addr_widths = widths;
548 	pdma->dma_dev.dst_addr_widths = widths;
549 	pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM);
550 	pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
551 	pdma->dma_dev.descriptor_reuse = true;
552 
553 	/* Setup DMA APIs */
554 	pdma->dma_dev.device_alloc_chan_resources =
555 		sf_pdma_alloc_chan_resources;
556 	pdma->dma_dev.device_free_chan_resources =
557 		sf_pdma_free_chan_resources;
558 	pdma->dma_dev.device_tx_status = sf_pdma_tx_status;
559 	pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy;
560 	pdma->dma_dev.device_config = sf_pdma_slave_config;
561 	pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all;
562 	pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending;
563 
564 	platform_set_drvdata(pdev, pdma);
565 
566 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
567 	if (ret)
568 		dev_warn(&pdev->dev,
569 			 "Failed to set DMA mask. Fall back to default.\n");
570 
571 	ret = dma_async_device_register(&pdma->dma_dev);
572 	if (ret) {
573 		dev_err(&pdev->dev,
574 			"Can't register SiFive Platform DMA. (%d)\n", ret);
575 		return ret;
576 	}
577 
578 	ret = of_dma_controller_register(pdev->dev.of_node,
579 					 of_dma_xlate_by_chan_id, pdma);
580 	if (ret < 0) {
581 		dev_err(&pdev->dev,
582 			"Can't register SiFive Platform OF_DMA. (%d)\n", ret);
583 		goto err_unregister;
584 	}
585 
586 	return 0;
587 
588 err_unregister:
589 	dma_async_device_unregister(&pdma->dma_dev);
590 
591 	return ret;
592 }
593 
sf_pdma_remove(struct platform_device * pdev)594 static void sf_pdma_remove(struct platform_device *pdev)
595 {
596 	struct sf_pdma *pdma = platform_get_drvdata(pdev);
597 	struct sf_pdma_chan *ch;
598 	int i;
599 
600 	for (i = 0; i < pdma->n_chans; i++) {
601 		ch = &pdma->chans[i];
602 
603 		devm_free_irq(&pdev->dev, ch->txirq, ch);
604 		devm_free_irq(&pdev->dev, ch->errirq, ch);
605 		list_del(&ch->vchan.chan.device_node);
606 		tasklet_kill(&ch->vchan.task);
607 		tasklet_kill(&ch->done_tasklet);
608 		tasklet_kill(&ch->err_tasklet);
609 	}
610 
611 	if (pdev->dev.of_node)
612 		of_dma_controller_free(pdev->dev.of_node);
613 
614 	dma_async_device_unregister(&pdma->dma_dev);
615 }
616 
617 static const struct sf_pdma_driver_platdata mpfs_pdma = {
618 	.quirks = PDMA_QUIRK_NO_STRICT_ORDERING,
619 };
620 
621 static const struct of_device_id sf_pdma_dt_ids[] = {
622 	{
623 		.compatible = "sifive,fu540-c000-pdma",
624 	}, {
625 		.compatible = "sifive,pdma0",
626 	}, {
627 		.compatible = "microchip,mpfs-pdma",
628 		.data	    = &mpfs_pdma,
629 	},
630 	{},
631 };
632 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids);
633 
634 static struct platform_driver sf_pdma_driver = {
635 	.probe		= sf_pdma_probe,
636 	.remove_new	= sf_pdma_remove,
637 	.driver		= {
638 		.name	= "sf-pdma",
639 		.of_match_table = sf_pdma_dt_ids,
640 	},
641 };
642 
sf_pdma_init(void)643 static int __init sf_pdma_init(void)
644 {
645 	return platform_driver_register(&sf_pdma_driver);
646 }
647 
sf_pdma_exit(void)648 static void __exit sf_pdma_exit(void)
649 {
650 	platform_driver_unregister(&sf_pdma_driver);
651 }
652 
653 /* do early init */
654 subsys_initcall(sf_pdma_init);
655 module_exit(sf_pdma_exit);
656 
657 MODULE_LICENSE("GPL v2");
658 MODULE_DESCRIPTION("SiFive Platform DMA driver");
659 MODULE_AUTHOR("Green Wan <green.wan@sifive.com>");
660