1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2024-2025 Arm Limited
3 // Arm DMA-350 driver
4
5 #include <linux/bitfield.h>
6 #include <linux/dmaengine.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/io.h>
9 #include <linux/of.h>
10 #include <linux/module.h>
11 #include <linux/platform_device.h>
12
13 #include "dmaengine.h"
14 #include "virt-dma.h"
15
16 #define DMANSECCTRL 0x200
17
18 #define NSEC_CTRL 0x0c
19 #define INTREN_ANYCHINTR_EN BIT(0)
20
21 #define DMAINFO 0x0f00
22
23 #define DMA_BUILDCFG0 0xb0
24 #define DMA_CFG_DATA_WIDTH GENMASK(18, 16)
25 #define DMA_CFG_ADDR_WIDTH GENMASK(15, 10)
26 #define DMA_CFG_NUM_CHANNELS GENMASK(9, 4)
27
28 #define DMA_BUILDCFG1 0xb4
29 #define DMA_CFG_NUM_TRIGGER_IN GENMASK(8, 0)
30
31 #define IIDR 0xc8
32 #define IIDR_PRODUCTID GENMASK(31, 20)
33 #define IIDR_VARIANT GENMASK(19, 16)
34 #define IIDR_REVISION GENMASK(15, 12)
35 #define IIDR_IMPLEMENTER GENMASK(11, 0)
36
37 #define PRODUCTID_DMA350 0x3a0
38 #define IMPLEMENTER_ARM 0x43b
39
40 #define DMACH(n) (0x1000 + 0x0100 * (n))
41
42 #define CH_CMD 0x00
43 #define CH_CMD_RESUME BIT(5)
44 #define CH_CMD_PAUSE BIT(4)
45 #define CH_CMD_STOP BIT(3)
46 #define CH_CMD_DISABLE BIT(2)
47 #define CH_CMD_CLEAR BIT(1)
48 #define CH_CMD_ENABLE BIT(0)
49
50 #define CH_STATUS 0x04
51 #define CH_STAT_RESUMEWAIT BIT(21)
52 #define CH_STAT_PAUSED BIT(20)
53 #define CH_STAT_STOPPED BIT(19)
54 #define CH_STAT_DISABLED BIT(18)
55 #define CH_STAT_ERR BIT(17)
56 #define CH_STAT_DONE BIT(16)
57 #define CH_STAT_INTR_ERR BIT(1)
58 #define CH_STAT_INTR_DONE BIT(0)
59
60 #define CH_INTREN 0x08
61 #define CH_INTREN_ERR BIT(1)
62 #define CH_INTREN_DONE BIT(0)
63
64 #define CH_CTRL 0x0c
65 #define CH_CTRL_USEDESTRIGIN BIT(26)
66 #define CH_CTRL_USESRCTRIGIN BIT(26)
67 #define CH_CTRL_DONETYPE GENMASK(23, 21)
68 #define CH_CTRL_REGRELOADTYPE GENMASK(20, 18)
69 #define CH_CTRL_XTYPE GENMASK(11, 9)
70 #define CH_CTRL_TRANSIZE GENMASK(2, 0)
71
72 #define CH_SRCADDR 0x10
73 #define CH_SRCADDRHI 0x14
74 #define CH_DESADDR 0x18
75 #define CH_DESADDRHI 0x1c
76 #define CH_XSIZE 0x20
77 #define CH_XSIZEHI 0x24
78 #define CH_SRCTRANSCFG 0x28
79 #define CH_DESTRANSCFG 0x2c
80 #define CH_CFG_MAXBURSTLEN GENMASK(19, 16)
81 #define CH_CFG_PRIVATTR BIT(11)
82 #define CH_CFG_SHAREATTR GENMASK(9, 8)
83 #define CH_CFG_MEMATTR GENMASK(7, 0)
84
85 #define TRANSCFG_DEVICE \
86 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \
87 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \
88 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE)
89 #define TRANSCFG_NC \
90 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \
91 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \
92 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC)
93 #define TRANSCFG_WB \
94 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \
95 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) | \
96 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB)
97
98 #define CH_XADDRINC 0x30
99 #define CH_XY_DES GENMASK(31, 16)
100 #define CH_XY_SRC GENMASK(15, 0)
101
102 #define CH_FILLVAL 0x38
103 #define CH_SRCTRIGINCFG 0x4c
104 #define CH_DESTRIGINCFG 0x50
105 #define CH_LINKATTR 0x70
106 #define CH_LINK_SHAREATTR GENMASK(9, 8)
107 #define CH_LINK_MEMATTR GENMASK(7, 0)
108
109 #define CH_AUTOCFG 0x74
110 #define CH_LINKADDR 0x78
111 #define CH_LINKADDR_EN BIT(0)
112
113 #define CH_LINKADDRHI 0x7c
114 #define CH_ERRINFO 0x90
115 #define CH_ERRINFO_AXIRDPOISERR BIT(18)
116 #define CH_ERRINFO_AXIWRRESPERR BIT(17)
117 #define CH_ERRINFO_AXIRDRESPERR BIT(16)
118
119 #define CH_BUILDCFG0 0xf8
120 #define CH_CFG_INC_WIDTH GENMASK(29, 26)
121 #define CH_CFG_DATA_WIDTH GENMASK(24, 22)
122 #define CH_CFG_DATA_BUF_SIZE GENMASK(7, 0)
123
124 #define CH_BUILDCFG1 0xfc
125 #define CH_CFG_HAS_CMDLINK BIT(8)
126 #define CH_CFG_HAS_TRIGSEL BIT(7)
127 #define CH_CFG_HAS_TRIGIN BIT(5)
128 #define CH_CFG_HAS_WRAP BIT(1)
129
130
131 #define LINK_REGCLEAR BIT(0)
132 #define LINK_INTREN BIT(2)
133 #define LINK_CTRL BIT(3)
134 #define LINK_SRCADDR BIT(4)
135 #define LINK_SRCADDRHI BIT(5)
136 #define LINK_DESADDR BIT(6)
137 #define LINK_DESADDRHI BIT(7)
138 #define LINK_XSIZE BIT(8)
139 #define LINK_XSIZEHI BIT(9)
140 #define LINK_SRCTRANSCFG BIT(10)
141 #define LINK_DESTRANSCFG BIT(11)
142 #define LINK_XADDRINC BIT(12)
143 #define LINK_FILLVAL BIT(14)
144 #define LINK_SRCTRIGINCFG BIT(19)
145 #define LINK_DESTRIGINCFG BIT(20)
146 #define LINK_AUTOCFG BIT(29)
147 #define LINK_LINKADDR BIT(30)
148 #define LINK_LINKADDRHI BIT(31)
149
150
151 enum ch_ctrl_donetype {
152 CH_CTRL_DONETYPE_NONE = 0,
153 CH_CTRL_DONETYPE_CMD = 1,
154 CH_CTRL_DONETYPE_CYCLE = 3
155 };
156
157 enum ch_ctrl_xtype {
158 CH_CTRL_XTYPE_DISABLE = 0,
159 CH_CTRL_XTYPE_CONTINUE = 1,
160 CH_CTRL_XTYPE_WRAP = 2,
161 CH_CTRL_XTYPE_FILL = 3
162 };
163
164 enum ch_cfg_shareattr {
165 SHAREATTR_NSH = 0,
166 SHAREATTR_OSH = 2,
167 SHAREATTR_ISH = 3
168 };
169
170 enum ch_cfg_memattr {
171 MEMATTR_DEVICE = 0x00,
172 MEMATTR_NC = 0x44,
173 MEMATTR_WB = 0xff
174 };
175
176 struct d350_desc {
177 struct virt_dma_desc vd;
178 u32 command[16];
179 u16 xsize;
180 u16 xsizehi;
181 u8 tsz;
182 };
183
184 struct d350_chan {
185 struct virt_dma_chan vc;
186 struct d350_desc *desc;
187 void __iomem *base;
188 int irq;
189 enum dma_status status;
190 dma_cookie_t cookie;
191 u32 residue;
192 u8 tsz;
193 bool has_trig;
194 bool has_wrap;
195 bool coherent;
196 };
197
198 struct d350 {
199 struct dma_device dma;
200 int nchan;
201 int nreq;
202 struct d350_chan channels[] __counted_by(nchan);
203 };
204
to_d350_chan(struct dma_chan * chan)205 static inline struct d350_chan *to_d350_chan(struct dma_chan *chan)
206 {
207 return container_of(chan, struct d350_chan, vc.chan);
208 }
209
to_d350_desc(struct virt_dma_desc * vd)210 static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd)
211 {
212 return container_of(vd, struct d350_desc, vd);
213 }
214
d350_desc_free(struct virt_dma_desc * vd)215 static void d350_desc_free(struct virt_dma_desc *vd)
216 {
217 kfree(to_d350_desc(vd));
218 }
219
d350_prep_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)220 static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan,
221 dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags)
222 {
223 struct d350_chan *dch = to_d350_chan(chan);
224 struct d350_desc *desc;
225 u32 *cmd;
226
227 desc = kzalloc_obj(*desc, GFP_NOWAIT);
228 if (!desc)
229 return NULL;
230
231 desc->tsz = __ffs(len | dest | src | (1 << dch->tsz));
232 desc->xsize = lower_16_bits(len >> desc->tsz);
233 desc->xsizehi = upper_16_bits(len >> desc->tsz);
234
235 cmd = desc->command;
236 cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR |
237 LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG |
238 LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR;
239
240 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
241 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) |
242 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD);
243
244 cmd[2] = lower_32_bits(src);
245 cmd[3] = upper_32_bits(src);
246 cmd[4] = lower_32_bits(dest);
247 cmd[5] = upper_32_bits(dest);
248 cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize);
249 cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi);
250 cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
251 cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
252 cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1);
253 cmd[11] = 0;
254
255 return vchan_tx_prep(&dch->vc, &desc->vd, flags);
256 }
257
d350_prep_memset(struct dma_chan * chan,dma_addr_t dest,int value,size_t len,unsigned long flags)258 static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan,
259 dma_addr_t dest, int value, size_t len, unsigned long flags)
260 {
261 struct d350_chan *dch = to_d350_chan(chan);
262 struct d350_desc *desc;
263 u32 *cmd;
264
265 desc = kzalloc_obj(*desc, GFP_NOWAIT);
266 if (!desc)
267 return NULL;
268
269 desc->tsz = __ffs(len | dest | (1 << dch->tsz));
270 desc->xsize = lower_16_bits(len >> desc->tsz);
271 desc->xsizehi = upper_16_bits(len >> desc->tsz);
272
273 cmd = desc->command;
274 cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI |
275 LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG |
276 LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR;
277
278 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
279 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) |
280 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD);
281
282 cmd[2] = lower_32_bits(dest);
283 cmd[3] = upper_32_bits(dest);
284 cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize);
285 cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi);
286 cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
287 cmd[7] = FIELD_PREP(CH_XY_DES, 1);
288 cmd[8] = (u8)value * 0x01010101;
289 cmd[9] = 0;
290
291 return vchan_tx_prep(&dch->vc, &desc->vd, flags);
292 }
293
d350_pause(struct dma_chan * chan)294 static int d350_pause(struct dma_chan *chan)
295 {
296 struct d350_chan *dch = to_d350_chan(chan);
297 unsigned long flags;
298
299 spin_lock_irqsave(&dch->vc.lock, flags);
300 if (dch->status == DMA_IN_PROGRESS) {
301 writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD);
302 dch->status = DMA_PAUSED;
303 }
304 spin_unlock_irqrestore(&dch->vc.lock, flags);
305
306 return 0;
307 }
308
d350_resume(struct dma_chan * chan)309 static int d350_resume(struct dma_chan *chan)
310 {
311 struct d350_chan *dch = to_d350_chan(chan);
312 unsigned long flags;
313
314 spin_lock_irqsave(&dch->vc.lock, flags);
315 if (dch->status == DMA_PAUSED) {
316 writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD);
317 dch->status = DMA_IN_PROGRESS;
318 }
319 spin_unlock_irqrestore(&dch->vc.lock, flags);
320
321 return 0;
322 }
323
d350_get_residue(struct d350_chan * dch)324 static u32 d350_get_residue(struct d350_chan *dch)
325 {
326 u32 res, xsize, xsizehi, hi_new;
327 int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
328
329 hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
330 do {
331 xsizehi = hi_new;
332 xsize = readl_relaxed(dch->base + CH_XSIZE);
333 hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
334 } while (xsizehi != hi_new && --retries);
335
336 res = FIELD_GET(CH_XY_DES, xsize);
337 res |= FIELD_GET(CH_XY_DES, xsizehi) << 16;
338
339 return res << dch->desc->tsz;
340 }
341
d350_terminate_all(struct dma_chan * chan)342 static int d350_terminate_all(struct dma_chan *chan)
343 {
344 struct d350_chan *dch = to_d350_chan(chan);
345 unsigned long flags;
346 LIST_HEAD(list);
347
348 spin_lock_irqsave(&dch->vc.lock, flags);
349 writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD);
350 if (dch->desc) {
351 if (dch->status != DMA_ERROR)
352 vchan_terminate_vdesc(&dch->desc->vd);
353 dch->desc = NULL;
354 dch->status = DMA_COMPLETE;
355 }
356 vchan_get_all_descriptors(&dch->vc, &list);
357 list_splice_tail(&list, &dch->vc.desc_terminated);
358 spin_unlock_irqrestore(&dch->vc.lock, flags);
359
360 return 0;
361 }
362
d350_synchronize(struct dma_chan * chan)363 static void d350_synchronize(struct dma_chan *chan)
364 {
365 struct d350_chan *dch = to_d350_chan(chan);
366
367 vchan_synchronize(&dch->vc);
368 }
369
d350_desc_bytes(struct d350_desc * desc)370 static u32 d350_desc_bytes(struct d350_desc *desc)
371 {
372 return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz;
373 }
374
d350_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * state)375 static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
376 struct dma_tx_state *state)
377 {
378 struct d350_chan *dch = to_d350_chan(chan);
379 struct virt_dma_desc *vd;
380 enum dma_status status;
381 unsigned long flags;
382 u32 residue = 0;
383
384 status = dma_cookie_status(chan, cookie, state);
385
386 spin_lock_irqsave(&dch->vc.lock, flags);
387 if (cookie == dch->cookie) {
388 status = dch->status;
389 if (status == DMA_IN_PROGRESS || status == DMA_PAUSED)
390 dch->residue = d350_get_residue(dch);
391 residue = dch->residue;
392 } else if ((vd = vchan_find_desc(&dch->vc, cookie))) {
393 residue = d350_desc_bytes(to_d350_desc(vd));
394 } else if (status == DMA_IN_PROGRESS) {
395 /* Somebody else terminated it? */
396 status = DMA_ERROR;
397 }
398 spin_unlock_irqrestore(&dch->vc.lock, flags);
399
400 dma_set_residue(state, residue);
401 return status;
402 }
403
d350_start_next(struct d350_chan * dch)404 static void d350_start_next(struct d350_chan *dch)
405 {
406 u32 hdr, *reg;
407
408 dch->desc = to_d350_desc(vchan_next_desc(&dch->vc));
409 if (!dch->desc)
410 return;
411
412 list_del(&dch->desc->vd.node);
413 dch->status = DMA_IN_PROGRESS;
414 dch->cookie = dch->desc->vd.tx.cookie;
415 dch->residue = d350_desc_bytes(dch->desc);
416
417 hdr = dch->desc->command[0];
418 reg = &dch->desc->command[1];
419
420 if (hdr & LINK_INTREN)
421 writel_relaxed(*reg++, dch->base + CH_INTREN);
422 if (hdr & LINK_CTRL)
423 writel_relaxed(*reg++, dch->base + CH_CTRL);
424 if (hdr & LINK_SRCADDR)
425 writel_relaxed(*reg++, dch->base + CH_SRCADDR);
426 if (hdr & LINK_SRCADDRHI)
427 writel_relaxed(*reg++, dch->base + CH_SRCADDRHI);
428 if (hdr & LINK_DESADDR)
429 writel_relaxed(*reg++, dch->base + CH_DESADDR);
430 if (hdr & LINK_DESADDRHI)
431 writel_relaxed(*reg++, dch->base + CH_DESADDRHI);
432 if (hdr & LINK_XSIZE)
433 writel_relaxed(*reg++, dch->base + CH_XSIZE);
434 if (hdr & LINK_XSIZEHI)
435 writel_relaxed(*reg++, dch->base + CH_XSIZEHI);
436 if (hdr & LINK_SRCTRANSCFG)
437 writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG);
438 if (hdr & LINK_DESTRANSCFG)
439 writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG);
440 if (hdr & LINK_XADDRINC)
441 writel_relaxed(*reg++, dch->base + CH_XADDRINC);
442 if (hdr & LINK_FILLVAL)
443 writel_relaxed(*reg++, dch->base + CH_FILLVAL);
444 if (hdr & LINK_SRCTRIGINCFG)
445 writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG);
446 if (hdr & LINK_DESTRIGINCFG)
447 writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG);
448 if (hdr & LINK_AUTOCFG)
449 writel_relaxed(*reg++, dch->base + CH_AUTOCFG);
450 if (hdr & LINK_LINKADDR)
451 writel_relaxed(*reg++, dch->base + CH_LINKADDR);
452 if (hdr & LINK_LINKADDRHI)
453 writel_relaxed(*reg++, dch->base + CH_LINKADDRHI);
454
455 writel(CH_CMD_ENABLE, dch->base + CH_CMD);
456 }
457
d350_issue_pending(struct dma_chan * chan)458 static void d350_issue_pending(struct dma_chan *chan)
459 {
460 struct d350_chan *dch = to_d350_chan(chan);
461 unsigned long flags;
462
463 spin_lock_irqsave(&dch->vc.lock, flags);
464 if (vchan_issue_pending(&dch->vc) && !dch->desc)
465 d350_start_next(dch);
466 spin_unlock_irqrestore(&dch->vc.lock, flags);
467 }
468
d350_irq(int irq,void * data)469 static irqreturn_t d350_irq(int irq, void *data)
470 {
471 struct d350_chan *dch = data;
472 struct device *dev = dch->vc.chan.device->dev;
473 struct virt_dma_desc *vd = &dch->desc->vd;
474 u32 ch_status;
475
476 ch_status = readl(dch->base + CH_STATUS);
477 if (!ch_status)
478 return IRQ_NONE;
479
480 if (ch_status & CH_STAT_INTR_ERR) {
481 u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO);
482
483 if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR))
484 vd->tx_result.result = DMA_TRANS_READ_FAILED;
485 else if (errinfo & CH_ERRINFO_AXIWRRESPERR)
486 vd->tx_result.result = DMA_TRANS_WRITE_FAILED;
487 else
488 vd->tx_result.result = DMA_TRANS_ABORTED;
489
490 vd->tx_result.residue = d350_get_residue(dch);
491 } else if (!(ch_status & CH_STAT_INTR_DONE)) {
492 dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status);
493 }
494 writel_relaxed(ch_status, dch->base + CH_STATUS);
495
496 spin_lock(&dch->vc.lock);
497 vchan_cookie_complete(vd);
498 if (ch_status & CH_STAT_INTR_DONE) {
499 dch->status = DMA_COMPLETE;
500 dch->residue = 0;
501 d350_start_next(dch);
502 } else {
503 dch->status = DMA_ERROR;
504 dch->residue = vd->tx_result.residue;
505 }
506 spin_unlock(&dch->vc.lock);
507
508 return IRQ_HANDLED;
509 }
510
d350_alloc_chan_resources(struct dma_chan * chan)511 static int d350_alloc_chan_resources(struct dma_chan *chan)
512 {
513 struct d350_chan *dch = to_d350_chan(chan);
514 int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED,
515 dev_name(&dch->vc.chan.dev->device), dch);
516 if (!ret)
517 writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN);
518
519 return ret;
520 }
521
d350_free_chan_resources(struct dma_chan * chan)522 static void d350_free_chan_resources(struct dma_chan *chan)
523 {
524 struct d350_chan *dch = to_d350_chan(chan);
525
526 writel_relaxed(0, dch->base + CH_INTREN);
527 free_irq(dch->irq, dch);
528 vchan_free_chan_resources(&dch->vc);
529 }
530
d350_probe(struct platform_device * pdev)531 static int d350_probe(struct platform_device *pdev)
532 {
533 struct device *dev = &pdev->dev;
534 struct d350 *dmac;
535 void __iomem *base;
536 u32 reg;
537 int ret, nchan, dw, aw, r, p;
538 bool coherent, memset;
539
540 base = devm_platform_ioremap_resource(pdev, 0);
541 if (IS_ERR(base))
542 return PTR_ERR(base);
543
544 reg = readl_relaxed(base + DMAINFO + IIDR);
545 r = FIELD_GET(IIDR_VARIANT, reg);
546 p = FIELD_GET(IIDR_REVISION, reg);
547 if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM ||
548 FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350)
549 return dev_err_probe(dev, -ENODEV, "Not a DMA-350!");
550
551 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0);
552 nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1;
553 dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg);
554 aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1;
555
556 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw));
557 coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT;
558
559 dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL);
560 if (!dmac)
561 return -ENOMEM;
562
563 dmac->nchan = nchan;
564
565 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1);
566 dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg);
567
568 dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq);
569
570 dmac->dma.dev = dev;
571 for (int i = min(dw, 16); i > 0; i /= 2) {
572 dmac->dma.src_addr_widths |= BIT(i);
573 dmac->dma.dst_addr_widths |= BIT(i);
574 }
575 dmac->dma.directions = BIT(DMA_MEM_TO_MEM);
576 dmac->dma.descriptor_reuse = true;
577 dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
578 dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources;
579 dmac->dma.device_free_chan_resources = d350_free_chan_resources;
580 dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask);
581 dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy;
582 dmac->dma.device_pause = d350_pause;
583 dmac->dma.device_resume = d350_resume;
584 dmac->dma.device_terminate_all = d350_terminate_all;
585 dmac->dma.device_synchronize = d350_synchronize;
586 dmac->dma.device_tx_status = d350_tx_status;
587 dmac->dma.device_issue_pending = d350_issue_pending;
588 INIT_LIST_HEAD(&dmac->dma.channels);
589
590 reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL);
591 writel_relaxed(reg | INTREN_ANYCHINTR_EN,
592 base + DMANSECCTRL + NSEC_CTRL);
593
594 /* Would be nice to have per-channel caps for this... */
595 memset = true;
596 for (int i = 0; i < nchan; i++) {
597 struct d350_chan *dch = &dmac->channels[i];
598
599 dch->base = base + DMACH(i);
600 writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD);
601
602 reg = readl_relaxed(dch->base + CH_BUILDCFG1);
603 if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) {
604 dev_warn(dev, "No command link support on channel %d\n", i);
605 continue;
606 }
607 dch->irq = platform_get_irq(pdev, i);
608 if (dch->irq < 0)
609 return dev_err_probe(dev, dch->irq,
610 "Failed to get IRQ for channel %d\n", i);
611
612 dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg);
613 dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) &
614 FIELD_GET(CH_CFG_HAS_TRIGSEL, reg);
615
616 /* Fill is a special case of Wrap */
617 memset &= dch->has_wrap;
618
619 reg = readl_relaxed(dch->base + CH_BUILDCFG0);
620 dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg);
621
622 reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH);
623 reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC);
624 writel_relaxed(reg, dch->base + CH_LINKATTR);
625
626 dch->vc.desc_free = d350_desc_free;
627 vchan_init(&dch->vc, &dmac->dma);
628 }
629
630 if (memset) {
631 dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask);
632 dmac->dma.device_prep_dma_memset = d350_prep_memset;
633 }
634
635 platform_set_drvdata(pdev, dmac);
636
637 ret = dma_async_device_register(&dmac->dma);
638 if (ret)
639 return dev_err_probe(dev, ret, "Failed to register DMA device\n");
640
641 return 0;
642 }
643
d350_remove(struct platform_device * pdev)644 static void d350_remove(struct platform_device *pdev)
645 {
646 struct d350 *dmac = platform_get_drvdata(pdev);
647
648 dma_async_device_unregister(&dmac->dma);
649 }
650
651 static const struct of_device_id d350_of_match[] __maybe_unused = {
652 { .compatible = "arm,dma-350" },
653 {}
654 };
655 MODULE_DEVICE_TABLE(of, d350_of_match);
656
657 static struct platform_driver d350_driver = {
658 .driver = {
659 .name = "arm-dma350",
660 .of_match_table = of_match_ptr(d350_of_match),
661 },
662 .probe = d350_probe,
663 .remove = d350_remove,
664 };
665 module_platform_driver(d350_driver);
666
667 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
668 MODULE_DESCRIPTION("Arm DMA-350 driver");
669 MODULE_LICENSE("GPL v2");
670