1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Driver for the Analog Devices AXI-DMAC core
4 *
5 * Copyright 2013-2019 Analog Devices Inc.
6 * Author: Lars-Peter Clausen <lars@metafoo.de>
7 */
8
9 #include <linux/adi-axi-common.h>
10 #include <linux/bitfield.h>
11 #include <linux/cleanup.h>
12 #include <linux/clk.h>
13 #include <linux/device.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dmaengine.h>
16 #include <linux/err.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/kernel.h>
20 #include <linux/module.h>
21 #include <linux/of.h>
22 #include <linux/of_dma.h>
23 #include <linux/of_address.h>
24 #include <linux/platform_device.h>
25 #include <linux/regmap.h>
26 #include <linux/slab.h>
27
28 #include <dt-bindings/dma/axi-dmac.h>
29
30 #include "dmaengine.h"
31 #include "virt-dma.h"
32
33 /*
34 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
35 * various instantiation parameters which decided the exact feature set support
36 * by the core.
37 *
38 * Each channel of the core has a source interface and a destination interface.
39 * The number of channels and the type of the channel interfaces is selected at
40 * configuration time. A interface can either be a connected to a central memory
41 * interconnect, which allows access to system memory, or it can be connected to
42 * a dedicated bus which is directly connected to a data port on a peripheral.
43 * Given that those are configuration options of the core that are selected when
44 * it is instantiated this means that they can not be changed by software at
45 * runtime. By extension this means that each channel is uni-directional. It can
46 * either be device to memory or memory to device, but not both. Also since the
47 * device side is a dedicated data bus only connected to a single peripheral
48 * there is no address than can or needs to be configured for the device side.
49 */
50
51 #define AXI_DMAC_REG_INTERFACE_DESC 0x10
52 #define AXI_DMAC_DMA_SRC_TYPE_MSK GENMASK(13, 12)
53 #define AXI_DMAC_DMA_SRC_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x)
54 #define AXI_DMAC_DMA_SRC_WIDTH_MSK GENMASK(11, 8)
55 #define AXI_DMAC_DMA_SRC_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x)
56 #define AXI_DMAC_DMA_DST_TYPE_MSK GENMASK(5, 4)
57 #define AXI_DMAC_DMA_DST_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x)
58 #define AXI_DMAC_DMA_DST_WIDTH_MSK GENMASK(3, 0)
59 #define AXI_DMAC_DMA_DST_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x)
60 #define AXI_DMAC_REG_COHERENCY_DESC 0x14
61 #define AXI_DMAC_DST_COHERENT_MSK BIT(0)
62 #define AXI_DMAC_DST_COHERENT_GET(x) FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x)
63
64 #define AXI_DMAC_REG_IRQ_MASK 0x80
65 #define AXI_DMAC_REG_IRQ_PENDING 0x84
66 #define AXI_DMAC_REG_IRQ_SOURCE 0x88
67
68 #define AXI_DMAC_REG_CTRL 0x400
69 #define AXI_DMAC_REG_TRANSFER_ID 0x404
70 #define AXI_DMAC_REG_START_TRANSFER 0x408
71 #define AXI_DMAC_REG_FLAGS 0x40c
72 #define AXI_DMAC_REG_DEST_ADDRESS 0x410
73 #define AXI_DMAC_REG_DEST_ADDRESS_HIGH 0x490
74 #define AXI_DMAC_REG_SRC_ADDRESS 0x414
75 #define AXI_DMAC_REG_SRC_ADDRESS_HIGH 0x494
76 #define AXI_DMAC_REG_X_LENGTH 0x418
77 #define AXI_DMAC_REG_Y_LENGTH 0x41c
78 #define AXI_DMAC_REG_DEST_STRIDE 0x420
79 #define AXI_DMAC_REG_SRC_STRIDE 0x424
80 #define AXI_DMAC_REG_TRANSFER_DONE 0x428
81 #define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c
82 #define AXI_DMAC_REG_STATUS 0x430
83 #define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434
84 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438
85 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c
86 #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450
87 #define AXI_DMAC_REG_CURRENT_SG_ID 0x454
88 #define AXI_DMAC_REG_SG_ADDRESS 0x47c
89 #define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc
90
91 #define AXI_DMAC_CTRL_ENABLE BIT(0)
92 #define AXI_DMAC_CTRL_PAUSE BIT(1)
93 #define AXI_DMAC_CTRL_ENABLE_SG BIT(2)
94
95 #define AXI_DMAC_IRQ_SOT BIT(0)
96 #define AXI_DMAC_IRQ_EOT BIT(1)
97
98 #define AXI_DMAC_FLAG_CYCLIC BIT(0)
99 #define AXI_DMAC_FLAG_LAST BIT(1)
100 #define AXI_DMAC_FLAG_PARTIAL_REPORT BIT(2)
101
102 #define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31)
103
104 /* The maximum ID allocated by the hardware is 31 */
105 #define AXI_DMAC_SG_UNUSED 32U
106
107 /* Flags for axi_dmac_hw_desc.flags */
108 #define AXI_DMAC_HW_FLAG_LAST BIT(0)
109 #define AXI_DMAC_HW_FLAG_IRQ BIT(1)
110
111 struct axi_dmac_hw_desc {
112 u32 flags;
113 u32 id;
114 u64 dest_addr;
115 u64 src_addr;
116 u64 next_sg_addr;
117 u32 y_len;
118 u32 x_len;
119 u32 src_stride;
120 u32 dst_stride;
121 u64 __pad[2];
122 };
123
124 struct axi_dmac_sg {
125 unsigned int partial_len;
126 bool schedule_when_free;
127
128 struct axi_dmac_hw_desc *hw;
129 dma_addr_t hw_phys;
130 };
131
132 struct axi_dmac_desc {
133 struct virt_dma_desc vdesc;
134 struct axi_dmac_chan *chan;
135
136 bool cyclic;
137 bool have_partial_xfer;
138
139 unsigned int num_submitted;
140 unsigned int num_completed;
141 unsigned int num_sgs;
142 struct axi_dmac_sg sg[] __counted_by(num_sgs);
143 };
144
145 struct axi_dmac_chan {
146 struct virt_dma_chan vchan;
147
148 struct axi_dmac_desc *next_desc;
149 struct list_head active_descs;
150 enum dma_transfer_direction direction;
151
152 unsigned int src_width;
153 unsigned int dest_width;
154 unsigned int src_type;
155 unsigned int dest_type;
156
157 unsigned int max_length;
158 unsigned int address_align_mask;
159 unsigned int length_align_mask;
160
161 bool hw_partial_xfer;
162 bool hw_cyclic;
163 bool hw_2d;
164 bool hw_sg;
165 };
166
167 struct axi_dmac {
168 void __iomem *base;
169 int irq;
170
171 struct clk *clk;
172
173 struct dma_device dma_dev;
174 struct axi_dmac_chan chan;
175 };
176
chan_to_axi_dmac(struct axi_dmac_chan * chan)177 static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan)
178 {
179 return container_of(chan->vchan.chan.device, struct axi_dmac,
180 dma_dev);
181 }
182
to_axi_dmac_chan(struct dma_chan * c)183 static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c)
184 {
185 return container_of(c, struct axi_dmac_chan, vchan.chan);
186 }
187
to_axi_dmac_desc(struct virt_dma_desc * vdesc)188 static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc)
189 {
190 return container_of(vdesc, struct axi_dmac_desc, vdesc);
191 }
192
axi_dmac_write(struct axi_dmac * axi_dmac,unsigned int reg,unsigned int val)193 static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg,
194 unsigned int val)
195 {
196 writel(val, axi_dmac->base + reg);
197 }
198
axi_dmac_read(struct axi_dmac * axi_dmac,unsigned int reg)199 static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg)
200 {
201 return readl(axi_dmac->base + reg);
202 }
203
axi_dmac_src_is_mem(struct axi_dmac_chan * chan)204 static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan)
205 {
206 return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;
207 }
208
axi_dmac_dest_is_mem(struct axi_dmac_chan * chan)209 static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)
210 {
211 return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;
212 }
213
axi_dmac_check_len(struct axi_dmac_chan * chan,unsigned int len)214 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
215 {
216 if (len == 0)
217 return false;
218 if ((len & chan->length_align_mask) != 0) /* Not aligned */
219 return false;
220 return true;
221 }
222
axi_dmac_check_addr(struct axi_dmac_chan * chan,dma_addr_t addr)223 static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
224 {
225 if ((addr & chan->address_align_mask) != 0) /* Not aligned */
226 return false;
227 return true;
228 }
229
axi_dmac_start_transfer(struct axi_dmac_chan * chan)230 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
231 {
232 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
233 struct virt_dma_desc *vdesc;
234 struct axi_dmac_desc *desc;
235 struct axi_dmac_sg *sg;
236 unsigned int flags = 0;
237 unsigned int val;
238
239 val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
240 if (val) /* Queue is full, wait for the next SOT IRQ */
241 return;
242
243 desc = chan->next_desc;
244
245 if (!desc) {
246 vdesc = vchan_next_desc(&chan->vchan);
247 if (!vdesc)
248 return;
249 list_move_tail(&vdesc->node, &chan->active_descs);
250 desc = to_axi_dmac_desc(vdesc);
251 chan->next_desc = desc;
252 }
253 sg = &desc->sg[desc->num_submitted];
254
255 /* Already queued in cyclic mode. Wait for it to finish */
256 if (sg->hw->id != AXI_DMAC_SG_UNUSED) {
257 sg->schedule_when_free = true;
258 return;
259 }
260
261 if (chan->hw_sg) {
262 chan->next_desc = NULL;
263 } else if (++desc->num_submitted == desc->num_sgs ||
264 desc->have_partial_xfer) {
265 if (desc->cyclic)
266 desc->num_submitted = 0; /* Start again */
267 else
268 chan->next_desc = NULL;
269 flags |= AXI_DMAC_FLAG_LAST;
270 }
271
272 sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);
273
274 if (!chan->hw_sg) {
275 if (axi_dmac_dest_is_mem(chan)) {
276 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr);
277 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH,
278 sg->hw->dest_addr >> 32);
279 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride);
280 }
281
282 if (axi_dmac_src_is_mem(chan)) {
283 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr);
284 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH, sg->hw->src_addr >> 32);
285 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride);
286 }
287 }
288
289 /*
290 * If the hardware supports cyclic transfers and there is no callback to
291 * call, enable hw cyclic mode to avoid unnecessary interrupts.
292 */
293 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) {
294 if (chan->hw_sg)
295 desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ;
296 else if (desc->num_sgs == 1)
297 flags |= AXI_DMAC_FLAG_CYCLIC;
298 }
299
300 if (chan->hw_partial_xfer)
301 flags |= AXI_DMAC_FLAG_PARTIAL_REPORT;
302
303 if (chan->hw_sg) {
304 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys);
305 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH,
306 (u64)sg->hw_phys >> 32);
307 } else {
308 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len);
309 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len);
310 }
311 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
312 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
313 }
314
axi_dmac_active_desc(struct axi_dmac_chan * chan)315 static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
316 {
317 return list_first_entry_or_null(&chan->active_descs,
318 struct axi_dmac_desc, vdesc.node);
319 }
320
axi_dmac_total_sg_bytes(struct axi_dmac_chan * chan,struct axi_dmac_sg * sg)321 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan,
322 struct axi_dmac_sg *sg)
323 {
324 if (chan->hw_2d)
325 return (sg->hw->x_len + 1) * (sg->hw->y_len + 1);
326 else
327 return (sg->hw->x_len + 1);
328 }
329
axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan * chan)330 static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan)
331 {
332 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
333 struct axi_dmac_desc *desc;
334 struct axi_dmac_sg *sg;
335 u32 xfer_done, len, id, i;
336 bool found_sg;
337
338 do {
339 len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN);
340 id = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID);
341
342 found_sg = false;
343 list_for_each_entry(desc, &chan->active_descs, vdesc.node) {
344 for (i = 0; i < desc->num_sgs; i++) {
345 sg = &desc->sg[i];
346 if (sg->hw->id == AXI_DMAC_SG_UNUSED)
347 continue;
348 if (sg->hw->id == id) {
349 desc->have_partial_xfer = true;
350 sg->partial_len = len;
351 found_sg = true;
352 break;
353 }
354 }
355 if (found_sg)
356 break;
357 }
358
359 if (found_sg) {
360 dev_dbg(dmac->dma_dev.dev,
361 "Found partial segment id=%u, len=%u\n",
362 id, len);
363 } else {
364 dev_warn(dmac->dma_dev.dev,
365 "Not found partial segment id=%u, len=%u\n",
366 id, len);
367 }
368
369 /* Check if we have any more partial transfers */
370 xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
371 xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE);
372
373 } while (!xfer_done);
374 }
375
axi_dmac_compute_residue(struct axi_dmac_chan * chan,struct axi_dmac_desc * active)376 static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
377 struct axi_dmac_desc *active)
378 {
379 struct dmaengine_result *rslt = &active->vdesc.tx_result;
380 unsigned int start = active->num_completed - 1;
381 struct axi_dmac_sg *sg;
382 unsigned int i, total;
383
384 rslt->result = DMA_TRANS_NOERROR;
385 rslt->residue = 0;
386
387 if (chan->hw_sg)
388 return;
389
390 /*
391 * We get here if the last completed segment is partial, which
392 * means we can compute the residue from that segment onwards
393 */
394 for (i = start; i < active->num_sgs; i++) {
395 sg = &active->sg[i];
396 total = axi_dmac_total_sg_bytes(chan, sg);
397 rslt->residue += (total - sg->partial_len);
398 }
399 }
400
axi_dmac_transfer_done(struct axi_dmac_chan * chan,unsigned int completed_transfers)401 static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
402 unsigned int completed_transfers)
403 {
404 struct axi_dmac_desc *active;
405 struct axi_dmac_sg *sg;
406 bool start_next = false;
407
408 active = axi_dmac_active_desc(chan);
409 if (!active)
410 return false;
411
412 if (chan->hw_partial_xfer &&
413 (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE))
414 axi_dmac_dequeue_partial_xfers(chan);
415
416 if (chan->hw_sg) {
417 if (active->cyclic) {
418 vchan_cyclic_callback(&active->vdesc);
419 } else {
420 list_del(&active->vdesc.node);
421 vchan_cookie_complete(&active->vdesc);
422 active = axi_dmac_active_desc(chan);
423 start_next = !!active;
424 }
425 } else {
426 do {
427 sg = &active->sg[active->num_completed];
428 if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */
429 break;
430 if (!(BIT(sg->hw->id) & completed_transfers))
431 break;
432 active->num_completed++;
433 sg->hw->id = AXI_DMAC_SG_UNUSED;
434 if (sg->schedule_when_free) {
435 sg->schedule_when_free = false;
436 start_next = true;
437 }
438
439 if (sg->partial_len)
440 axi_dmac_compute_residue(chan, active);
441
442 if (active->cyclic)
443 vchan_cyclic_callback(&active->vdesc);
444
445 if (active->num_completed == active->num_sgs ||
446 sg->partial_len) {
447 if (active->cyclic) {
448 active->num_completed = 0; /* wrap around */
449 } else {
450 list_del(&active->vdesc.node);
451 vchan_cookie_complete(&active->vdesc);
452 active = axi_dmac_active_desc(chan);
453 }
454 }
455 } while (active);
456 }
457
458 return start_next;
459 }
460
axi_dmac_interrupt_handler(int irq,void * devid)461 static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid)
462 {
463 struct axi_dmac *dmac = devid;
464 unsigned int pending;
465 bool start_next = false;
466
467 pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING);
468 if (!pending)
469 return IRQ_NONE;
470
471 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending);
472
473 spin_lock(&dmac->chan.vchan.lock);
474 /* One or more transfers have finished */
475 if (pending & AXI_DMAC_IRQ_EOT) {
476 unsigned int completed;
477
478 completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
479 start_next = axi_dmac_transfer_done(&dmac->chan, completed);
480 }
481 /* Space has become available in the descriptor queue */
482 if ((pending & AXI_DMAC_IRQ_SOT) || start_next)
483 axi_dmac_start_transfer(&dmac->chan);
484 spin_unlock(&dmac->chan.vchan.lock);
485
486 return IRQ_HANDLED;
487 }
488
axi_dmac_terminate_all(struct dma_chan * c)489 static int axi_dmac_terminate_all(struct dma_chan *c)
490 {
491 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
492 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
493 unsigned long flags;
494 LIST_HEAD(head);
495
496 spin_lock_irqsave(&chan->vchan.lock, flags);
497 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0);
498 chan->next_desc = NULL;
499 vchan_get_all_descriptors(&chan->vchan, &head);
500 list_splice_tail_init(&chan->active_descs, &head);
501 spin_unlock_irqrestore(&chan->vchan.lock, flags);
502
503 vchan_dma_desc_free_list(&chan->vchan, &head);
504
505 return 0;
506 }
507
axi_dmac_synchronize(struct dma_chan * c)508 static void axi_dmac_synchronize(struct dma_chan *c)
509 {
510 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
511
512 vchan_synchronize(&chan->vchan);
513 }
514
axi_dmac_issue_pending(struct dma_chan * c)515 static void axi_dmac_issue_pending(struct dma_chan *c)
516 {
517 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
518 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
519 unsigned long flags;
520 u32 ctrl = AXI_DMAC_CTRL_ENABLE;
521
522 if (chan->hw_sg)
523 ctrl |= AXI_DMAC_CTRL_ENABLE_SG;
524
525 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl);
526
527 spin_lock_irqsave(&chan->vchan.lock, flags);
528 if (vchan_issue_pending(&chan->vchan))
529 axi_dmac_start_transfer(chan);
530 spin_unlock_irqrestore(&chan->vchan.lock, flags);
531 }
532
533 static struct axi_dmac_desc *
axi_dmac_alloc_desc(struct axi_dmac_chan * chan,unsigned int num_sgs)534 axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
535 {
536 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
537 struct device *dev = dmac->dma_dev.dev;
538 struct axi_dmac_hw_desc *hws;
539 struct axi_dmac_desc *desc;
540 dma_addr_t hw_phys;
541 unsigned int i;
542
543 desc = kzalloc_flex(*desc, sg, num_sgs, GFP_NOWAIT);
544 if (!desc)
545 return NULL;
546 desc->num_sgs = num_sgs;
547 desc->chan = chan;
548
549 hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)),
550 &hw_phys, GFP_ATOMIC);
551 if (!hws) {
552 kfree(desc);
553 return NULL;
554 }
555
556 for (i = 0; i < num_sgs; i++) {
557 desc->sg[i].hw = &hws[i];
558 desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws);
559
560 hws[i].id = AXI_DMAC_SG_UNUSED;
561 hws[i].flags = 0;
562
563 /* Link hardware descriptors */
564 hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws);
565 }
566
567 /* The last hardware descriptor will trigger an interrupt */
568 desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ;
569
570 return desc;
571 }
572
axi_dmac_free_desc(struct axi_dmac_desc * desc)573 static void axi_dmac_free_desc(struct axi_dmac_desc *desc)
574 {
575 struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan);
576 struct device *dev = dmac->dma_dev.dev;
577 struct axi_dmac_hw_desc *hw = desc->sg[0].hw;
578 dma_addr_t hw_phys = desc->sg[0].hw_phys;
579
580 dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)),
581 hw, hw_phys);
582 kfree(desc);
583 }
584
axi_dmac_fill_linear_sg(struct axi_dmac_chan * chan,enum dma_transfer_direction direction,dma_addr_t addr,unsigned int num_periods,unsigned int period_len,struct axi_dmac_sg * sg)585 static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
586 enum dma_transfer_direction direction, dma_addr_t addr,
587 unsigned int num_periods, unsigned int period_len,
588 struct axi_dmac_sg *sg)
589 {
590 unsigned int num_segments, i;
591 unsigned int segment_size;
592 unsigned int len;
593
594 /* Split into multiple equally sized segments if necessary */
595 num_segments = DIV_ROUND_UP(period_len, chan->max_length);
596 segment_size = DIV_ROUND_UP(period_len, num_segments);
597 /* Take care of alignment */
598 segment_size = ((segment_size - 1) | chan->length_align_mask) + 1;
599
600 for (i = 0; i < num_periods; i++) {
601 for (len = period_len; len > segment_size; sg++) {
602 if (direction == DMA_DEV_TO_MEM)
603 sg->hw->dest_addr = addr;
604 else
605 sg->hw->src_addr = addr;
606 sg->hw->x_len = segment_size - 1;
607 sg->hw->y_len = 0;
608 sg->hw->flags = 0;
609 addr += segment_size;
610 len -= segment_size;
611 }
612
613 if (direction == DMA_DEV_TO_MEM)
614 sg->hw->dest_addr = addr;
615 else
616 sg->hw->src_addr = addr;
617 sg->hw->x_len = len - 1;
618 sg->hw->y_len = 0;
619 sg++;
620 addr += len;
621 }
622
623 return sg;
624 }
625
626 static struct dma_async_tx_descriptor *
axi_dmac_prep_peripheral_dma_vec(struct dma_chan * c,const struct dma_vec * vecs,size_t nb,enum dma_transfer_direction direction,unsigned long flags)627 axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs,
628 size_t nb, enum dma_transfer_direction direction,
629 unsigned long flags)
630 {
631 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
632 struct axi_dmac_desc *desc;
633 unsigned int num_sgs = 0;
634 struct axi_dmac_sg *dsg;
635 size_t i;
636
637 if (direction != chan->direction)
638 return NULL;
639
640 for (i = 0; i < nb; i++)
641 num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length);
642
643 desc = axi_dmac_alloc_desc(chan, num_sgs);
644 if (!desc)
645 return NULL;
646
647 dsg = desc->sg;
648
649 for (i = 0; i < nb; i++) {
650 if (!axi_dmac_check_addr(chan, vecs[i].addr) ||
651 !axi_dmac_check_len(chan, vecs[i].len)) {
652 kfree(desc);
653 return NULL;
654 }
655
656 dsg = axi_dmac_fill_linear_sg(chan, direction, vecs[i].addr, 1,
657 vecs[i].len, dsg);
658 }
659
660 desc->cyclic = false;
661
662 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
663 }
664
axi_dmac_prep_slave_sg(struct dma_chan * c,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)665 static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
666 struct dma_chan *c, struct scatterlist *sgl,
667 unsigned int sg_len, enum dma_transfer_direction direction,
668 unsigned long flags, void *context)
669 {
670 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
671 struct axi_dmac_desc *desc;
672 struct axi_dmac_sg *dsg;
673 struct scatterlist *sg;
674 unsigned int num_sgs;
675 unsigned int i;
676
677 if (direction != chan->direction)
678 return NULL;
679
680 num_sgs = sg_nents_for_dma(sgl, sg_len, chan->max_length);
681 desc = axi_dmac_alloc_desc(chan, num_sgs);
682 if (!desc)
683 return NULL;
684
685 dsg = desc->sg;
686
687 for_each_sg(sgl, sg, sg_len, i) {
688 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||
689 !axi_dmac_check_len(chan, sg_dma_len(sg))) {
690 axi_dmac_free_desc(desc);
691 return NULL;
692 }
693
694 dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1,
695 sg_dma_len(sg), dsg);
696 }
697
698 desc->cyclic = false;
699
700 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
701 }
702
axi_dmac_prep_dma_cyclic(struct dma_chan * c,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction direction,unsigned long flags)703 static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
704 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
705 size_t period_len, enum dma_transfer_direction direction,
706 unsigned long flags)
707 {
708 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
709 struct axi_dmac_desc *desc;
710 unsigned int num_periods, num_segments, num_sgs;
711
712 if (direction != chan->direction)
713 return NULL;
714
715 if (!axi_dmac_check_len(chan, buf_len) ||
716 !axi_dmac_check_addr(chan, buf_addr))
717 return NULL;
718
719 if (period_len == 0 || buf_len % period_len)
720 return NULL;
721
722 num_periods = buf_len / period_len;
723 num_segments = DIV_ROUND_UP(period_len, chan->max_length);
724 num_sgs = num_periods * num_segments;
725
726 desc = axi_dmac_alloc_desc(chan, num_sgs);
727 if (!desc)
728 return NULL;
729
730 /* Chain the last descriptor to the first, and remove its "last" flag */
731 desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys;
732 desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST;
733
734 axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods,
735 period_len, desc->sg);
736
737 desc->cyclic = true;
738
739 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
740 }
741
axi_dmac_prep_interleaved(struct dma_chan * c,struct dma_interleaved_template * xt,unsigned long flags)742 static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
743 struct dma_chan *c, struct dma_interleaved_template *xt,
744 unsigned long flags)
745 {
746 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
747 struct axi_dmac_desc *desc;
748 size_t dst_icg, src_icg;
749
750 if (xt->frame_size != 1)
751 return NULL;
752
753 if (xt->dir != chan->direction)
754 return NULL;
755
756 if (axi_dmac_src_is_mem(chan)) {
757 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))
758 return NULL;
759 }
760
761 if (axi_dmac_dest_is_mem(chan)) {
762 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))
763 return NULL;
764 }
765
766 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
767 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
768
769 if (chan->hw_2d) {
770 if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||
771 xt->numf == 0)
772 return NULL;
773 if (xt->sgl[0].size + dst_icg > chan->max_length ||
774 xt->sgl[0].size + src_icg > chan->max_length)
775 return NULL;
776 } else {
777 if (dst_icg != 0 || src_icg != 0)
778 return NULL;
779 if (chan->max_length / xt->sgl[0].size < xt->numf)
780 return NULL;
781 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))
782 return NULL;
783 }
784
785 desc = axi_dmac_alloc_desc(chan, 1);
786 if (!desc)
787 return NULL;
788
789 if (axi_dmac_src_is_mem(chan)) {
790 desc->sg[0].hw->src_addr = xt->src_start;
791 desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg;
792 }
793
794 if (axi_dmac_dest_is_mem(chan)) {
795 desc->sg[0].hw->dest_addr = xt->dst_start;
796 desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg;
797 }
798
799 if (chan->hw_2d) {
800 desc->sg[0].hw->x_len = xt->sgl[0].size - 1;
801 desc->sg[0].hw->y_len = xt->numf - 1;
802 } else {
803 desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1;
804 desc->sg[0].hw->y_len = 0;
805 }
806
807 if (flags & DMA_CYCLIC)
808 desc->cyclic = true;
809
810 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
811 }
812
axi_dmac_free_chan_resources(struct dma_chan * c)813 static void axi_dmac_free_chan_resources(struct dma_chan *c)
814 {
815 vchan_free_chan_resources(to_virt_chan(c));
816 }
817
axi_dmac_desc_free(struct virt_dma_desc * vdesc)818 static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
819 {
820 axi_dmac_free_desc(to_axi_dmac_desc(vdesc));
821 }
822
axi_dmac_regmap_rdwr(struct device * dev,unsigned int reg)823 static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg)
824 {
825 switch (reg) {
826 case AXI_DMAC_REG_IRQ_MASK:
827 case AXI_DMAC_REG_IRQ_SOURCE:
828 case AXI_DMAC_REG_IRQ_PENDING:
829 case AXI_DMAC_REG_CTRL:
830 case AXI_DMAC_REG_TRANSFER_ID:
831 case AXI_DMAC_REG_START_TRANSFER:
832 case AXI_DMAC_REG_FLAGS:
833 case AXI_DMAC_REG_DEST_ADDRESS:
834 case AXI_DMAC_REG_SRC_ADDRESS:
835 case AXI_DMAC_REG_X_LENGTH:
836 case AXI_DMAC_REG_Y_LENGTH:
837 case AXI_DMAC_REG_DEST_STRIDE:
838 case AXI_DMAC_REG_SRC_STRIDE:
839 case AXI_DMAC_REG_TRANSFER_DONE:
840 case AXI_DMAC_REG_ACTIVE_TRANSFER_ID:
841 case AXI_DMAC_REG_STATUS:
842 case AXI_DMAC_REG_CURRENT_SRC_ADDR:
843 case AXI_DMAC_REG_CURRENT_DEST_ADDR:
844 case AXI_DMAC_REG_PARTIAL_XFER_LEN:
845 case AXI_DMAC_REG_PARTIAL_XFER_ID:
846 case AXI_DMAC_REG_CURRENT_SG_ID:
847 case AXI_DMAC_REG_SG_ADDRESS:
848 case AXI_DMAC_REG_SG_ADDRESS_HIGH:
849 return true;
850 default:
851 return false;
852 }
853 }
854
855 static const struct regmap_config axi_dmac_regmap_config = {
856 .reg_bits = 32,
857 .val_bits = 32,
858 .reg_stride = 4,
859 .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID,
860 .readable_reg = axi_dmac_regmap_rdwr,
861 .writeable_reg = axi_dmac_regmap_rdwr,
862 };
863
axi_dmac_adjust_chan_params(struct axi_dmac_chan * chan)864 static void axi_dmac_adjust_chan_params(struct axi_dmac_chan *chan)
865 {
866 chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1;
867
868 if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
869 chan->direction = DMA_MEM_TO_MEM;
870 else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
871 chan->direction = DMA_MEM_TO_DEV;
872 else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan))
873 chan->direction = DMA_DEV_TO_MEM;
874 else
875 chan->direction = DMA_DEV_TO_DEV;
876 }
877
878 /*
879 * The configuration stored in the devicetree matches the configuration
880 * parameters of the peripheral instance and allows the driver to know which
881 * features are implemented and how it should behave.
882 */
axi_dmac_parse_chan_dt(struct device_node * of_chan,struct axi_dmac_chan * chan)883 static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
884 struct axi_dmac_chan *chan)
885 {
886 u32 val;
887 int ret;
888
889 ret = of_property_read_u32(of_chan, "reg", &val);
890 if (ret)
891 return ret;
892
893 /* We only support 1 channel for now */
894 if (val != 0)
895 return -EINVAL;
896
897 ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val);
898 if (ret)
899 return ret;
900 if (val > AXI_DMAC_BUS_TYPE_FIFO)
901 return -EINVAL;
902 chan->src_type = val;
903
904 ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val);
905 if (ret)
906 return ret;
907 if (val > AXI_DMAC_BUS_TYPE_FIFO)
908 return -EINVAL;
909 chan->dest_type = val;
910
911 ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val);
912 if (ret)
913 return ret;
914 chan->src_width = val / 8;
915
916 ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val);
917 if (ret)
918 return ret;
919 chan->dest_width = val / 8;
920
921 axi_dmac_adjust_chan_params(chan);
922
923 return 0;
924 }
925
axi_dmac_parse_dt(struct device * dev,struct axi_dmac * dmac)926 static int axi_dmac_parse_dt(struct device *dev, struct axi_dmac *dmac)
927 {
928 int ret;
929
930 struct device_node *of_channels __free(device_node) = of_get_child_by_name(dev->of_node,
931 "adi,channels");
932 if (of_channels == NULL)
933 return -ENODEV;
934
935 for_each_child_of_node_scoped(of_channels, of_chan) {
936 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);
937 if (ret)
938 return -EINVAL;
939 }
940
941 return 0;
942 }
943
axi_dmac_read_chan_config(struct device * dev,struct axi_dmac * dmac)944 static int axi_dmac_read_chan_config(struct device *dev, struct axi_dmac *dmac)
945 {
946 struct axi_dmac_chan *chan = &dmac->chan;
947 unsigned int val, desc;
948
949 desc = axi_dmac_read(dmac, AXI_DMAC_REG_INTERFACE_DESC);
950 if (desc == 0) {
951 dev_err(dev, "DMA interface register reads zero\n");
952 return -EFAULT;
953 }
954
955 val = AXI_DMAC_DMA_SRC_TYPE_GET(desc);
956 if (val > AXI_DMAC_BUS_TYPE_FIFO) {
957 dev_err(dev, "Invalid source bus type read: %d\n", val);
958 return -EINVAL;
959 }
960 chan->src_type = val;
961
962 val = AXI_DMAC_DMA_DST_TYPE_GET(desc);
963 if (val > AXI_DMAC_BUS_TYPE_FIFO) {
964 dev_err(dev, "Invalid destination bus type read: %d\n", val);
965 return -EINVAL;
966 }
967 chan->dest_type = val;
968
969 val = AXI_DMAC_DMA_SRC_WIDTH_GET(desc);
970 if (val == 0) {
971 dev_err(dev, "Source bus width is zero\n");
972 return -EINVAL;
973 }
974 /* widths are stored in log2 */
975 chan->src_width = 1 << val;
976
977 val = AXI_DMAC_DMA_DST_WIDTH_GET(desc);
978 if (val == 0) {
979 dev_err(dev, "Destination bus width is zero\n");
980 return -EINVAL;
981 }
982 chan->dest_width = 1 << val;
983
984 axi_dmac_adjust_chan_params(chan);
985
986 return 0;
987 }
988
axi_dmac_detect_caps(struct axi_dmac * dmac,unsigned int version)989 static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version)
990 {
991 struct axi_dmac_chan *chan = &dmac->chan;
992 struct device *dev = dmac->dma_dev.dev;
993 u32 mask;
994 int ret;
995
996 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC);
997 if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
998 chan->hw_cyclic = true;
999
1000 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff);
1001 if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS))
1002 chan->hw_sg = true;
1003
1004 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1);
1005 if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1)
1006 chan->hw_2d = true;
1007
1008 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff);
1009 chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
1010 if (chan->max_length != UINT_MAX)
1011 chan->max_length++;
1012
1013 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff);
1014 if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 &&
1015 chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
1016 dev_err(dmac->dma_dev.dev,
1017 "Destination memory-mapped interface not supported.");
1018 return -ENODEV;
1019 }
1020
1021 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff);
1022 if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 &&
1023 chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
1024 dev_err(dmac->dma_dev.dev,
1025 "Source memory-mapped interface not supported.");
1026 return -ENODEV;
1027 }
1028
1029 if (axi_dmac_dest_is_mem(chan)) {
1030 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH, 0xffffffff);
1031 mask = axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH);
1032 } else {
1033 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH, 0xffffffff);
1034 mask = axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH);
1035 }
1036
1037 mask = 32 + fls(mask);
1038
1039 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(mask));
1040 if (ret) {
1041 dev_err(dev, "DMA mask set error %d\n", ret);
1042 return ret;
1043 }
1044
1045 if (version >= ADI_AXI_PCORE_VER(4, 2, 'a'))
1046 chan->hw_partial_xfer = true;
1047
1048 if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) {
1049 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00);
1050 chan->length_align_mask =
1051 axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
1052 } else {
1053 chan->length_align_mask = chan->address_align_mask;
1054 }
1055
1056 return 0;
1057 }
1058
axi_dmac_tasklet_kill(void * task)1059 static void axi_dmac_tasklet_kill(void *task)
1060 {
1061 tasklet_kill(task);
1062 }
1063
axi_dmac_free_dma_controller(void * of_node)1064 static void axi_dmac_free_dma_controller(void *of_node)
1065 {
1066 of_dma_controller_free(of_node);
1067 }
1068
axi_dmac_probe(struct platform_device * pdev)1069 static int axi_dmac_probe(struct platform_device *pdev)
1070 {
1071 struct dma_device *dma_dev;
1072 struct axi_dmac *dmac;
1073 struct regmap *regmap;
1074 unsigned int version;
1075 u32 irq_mask = 0;
1076 int ret;
1077
1078 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
1079 if (!dmac)
1080 return -ENOMEM;
1081
1082 dmac->irq = platform_get_irq(pdev, 0);
1083 if (dmac->irq < 0)
1084 return dmac->irq;
1085 if (dmac->irq == 0)
1086 return -EINVAL;
1087
1088 dmac->base = devm_platform_ioremap_resource(pdev, 0);
1089 if (IS_ERR(dmac->base))
1090 return PTR_ERR(dmac->base);
1091
1092 dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL);
1093 if (IS_ERR(dmac->clk))
1094 return PTR_ERR(dmac->clk);
1095
1096 version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION);
1097
1098 if (version >= ADI_AXI_PCORE_VER(4, 3, 'a'))
1099 ret = axi_dmac_read_chan_config(&pdev->dev, dmac);
1100 else
1101 ret = axi_dmac_parse_dt(&pdev->dev, dmac);
1102
1103 if (ret < 0)
1104 return ret;
1105
1106 INIT_LIST_HEAD(&dmac->chan.active_descs);
1107
1108 dma_set_max_seg_size(&pdev->dev, UINT_MAX);
1109
1110 dma_dev = &dmac->dma_dev;
1111 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1112 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1113 dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);
1114 dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
1115 dma_dev->device_tx_status = dma_cookie_status;
1116 dma_dev->device_issue_pending = axi_dmac_issue_pending;
1117 dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg;
1118 dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec;
1119 dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic;
1120 dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved;
1121 dma_dev->device_terminate_all = axi_dmac_terminate_all;
1122 dma_dev->device_synchronize = axi_dmac_synchronize;
1123 dma_dev->dev = &pdev->dev;
1124 dma_dev->src_addr_widths = BIT(dmac->chan.src_width);
1125 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);
1126 dma_dev->directions = BIT(dmac->chan.direction);
1127 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1128 dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */
1129 INIT_LIST_HEAD(&dma_dev->channels);
1130
1131 dmac->chan.vchan.desc_free = axi_dmac_desc_free;
1132 vchan_init(&dmac->chan.vchan, dma_dev);
1133
1134 ret = axi_dmac_detect_caps(dmac, version);
1135 if (ret)
1136 return ret;
1137
1138 dma_dev->copy_align = (dmac->chan.address_align_mask + 1);
1139
1140 if (dmac->chan.hw_sg)
1141 irq_mask |= AXI_DMAC_IRQ_SOT;
1142
1143 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask);
1144
1145 if (of_dma_is_coherent(pdev->dev.of_node)) {
1146 ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC);
1147
1148 if (version < ADI_AXI_PCORE_VER(4, 4, 'a') ||
1149 !AXI_DMAC_DST_COHERENT_GET(ret)) {
1150 dev_err(dmac->dma_dev.dev,
1151 "Coherent DMA not supported in hardware");
1152 return -EINVAL;
1153 }
1154 }
1155
1156 ret = dmaenginem_async_device_register(dma_dev);
1157 if (ret)
1158 return ret;
1159
1160 /*
1161 * Put the action in here so it get's done before unregistering the DMA
1162 * device.
1163 */
1164 ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill,
1165 &dmac->chan.vchan.task);
1166 if (ret)
1167 return ret;
1168
1169 ret = of_dma_controller_register(pdev->dev.of_node,
1170 of_dma_xlate_by_chan_id, dma_dev);
1171 if (ret)
1172 return ret;
1173
1174 ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller,
1175 pdev->dev.of_node);
1176 if (ret)
1177 return ret;
1178
1179 ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler,
1180 IRQF_SHARED, dev_name(&pdev->dev), dmac);
1181 if (ret)
1182 return ret;
1183
1184 regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base,
1185 &axi_dmac_regmap_config);
1186
1187 return PTR_ERR_OR_ZERO(regmap);
1188 }
1189
1190 static const struct of_device_id axi_dmac_of_match_table[] = {
1191 { .compatible = "adi,axi-dmac-1.00.a" },
1192 { },
1193 };
1194 MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table);
1195
1196 static struct platform_driver axi_dmac_driver = {
1197 .driver = {
1198 .name = "dma-axi-dmac",
1199 .of_match_table = axi_dmac_of_match_table,
1200 },
1201 .probe = axi_dmac_probe,
1202 };
1203 module_platform_driver(axi_dmac_driver);
1204
1205 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
1206 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");
1207 MODULE_LICENSE("GPL v2");
1208