1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2024-2025 Arm Limited 3 // Arm DMA-350 driver 4 5 #include <linux/bitfield.h> 6 #include <linux/dmaengine.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/io.h> 9 #include <linux/of.h> 10 #include <linux/module.h> 11 #include <linux/platform_device.h> 12 13 #include "dmaengine.h" 14 #include "virt-dma.h" 15 16 #define DMANSECCTRL 0x200 17 18 #define NSEC_CTRL 0x0c 19 #define INTREN_ANYCHINTR_EN BIT(0) 20 21 #define DMAINFO 0x0f00 22 23 #define DMA_BUILDCFG0 0xb0 24 #define DMA_CFG_DATA_WIDTH GENMASK(18, 16) 25 #define DMA_CFG_ADDR_WIDTH GENMASK(15, 10) 26 #define DMA_CFG_NUM_CHANNELS GENMASK(9, 4) 27 28 #define DMA_BUILDCFG1 0xb4 29 #define DMA_CFG_NUM_TRIGGER_IN GENMASK(8, 0) 30 31 #define IIDR 0xc8 32 #define IIDR_PRODUCTID GENMASK(31, 20) 33 #define IIDR_VARIANT GENMASK(19, 16) 34 #define IIDR_REVISION GENMASK(15, 12) 35 #define IIDR_IMPLEMENTER GENMASK(11, 0) 36 37 #define PRODUCTID_DMA350 0x3a0 38 #define IMPLEMENTER_ARM 0x43b 39 40 #define DMACH(n) (0x1000 + 0x0100 * (n)) 41 42 #define CH_CMD 0x00 43 #define CH_CMD_RESUME BIT(5) 44 #define CH_CMD_PAUSE BIT(4) 45 #define CH_CMD_STOP BIT(3) 46 #define CH_CMD_DISABLE BIT(2) 47 #define CH_CMD_CLEAR BIT(1) 48 #define CH_CMD_ENABLE BIT(0) 49 50 #define CH_STATUS 0x04 51 #define CH_STAT_RESUMEWAIT BIT(21) 52 #define CH_STAT_PAUSED BIT(20) 53 #define CH_STAT_STOPPED BIT(19) 54 #define CH_STAT_DISABLED BIT(18) 55 #define CH_STAT_ERR BIT(17) 56 #define CH_STAT_DONE BIT(16) 57 #define CH_STAT_INTR_ERR BIT(1) 58 #define CH_STAT_INTR_DONE BIT(0) 59 60 #define CH_INTREN 0x08 61 #define CH_INTREN_ERR BIT(1) 62 #define CH_INTREN_DONE BIT(0) 63 64 #define CH_CTRL 0x0c 65 #define CH_CTRL_USEDESTRIGIN BIT(26) 66 #define CH_CTRL_USESRCTRIGIN BIT(26) 67 #define CH_CTRL_DONETYPE GENMASK(23, 21) 68 #define CH_CTRL_REGRELOADTYPE GENMASK(20, 18) 69 #define CH_CTRL_XTYPE GENMASK(11, 9) 70 #define CH_CTRL_TRANSIZE GENMASK(2, 0) 71 72 #define CH_SRCADDR 0x10 73 #define CH_SRCADDRHI 0x14 74 #define CH_DESADDR 0x18 75 #define CH_DESADDRHI 0x1c 76 #define CH_XSIZE 0x20 77 #define CH_XSIZEHI 0x24 78 #define CH_SRCTRANSCFG 0x28 79 #define CH_DESTRANSCFG 0x2c 80 #define CH_CFG_MAXBURSTLEN GENMASK(19, 16) 81 #define CH_CFG_PRIVATTR BIT(11) 82 #define CH_CFG_SHAREATTR GENMASK(9, 8) 83 #define CH_CFG_MEMATTR GENMASK(7, 0) 84 85 #define TRANSCFG_DEVICE \ 86 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ 87 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ 88 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE) 89 #define TRANSCFG_NC \ 90 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ 91 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ 92 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC) 93 #define TRANSCFG_WB \ 94 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ 95 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) | \ 96 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB) 97 98 #define CH_XADDRINC 0x30 99 #define CH_XY_DES GENMASK(31, 16) 100 #define CH_XY_SRC GENMASK(15, 0) 101 102 #define CH_FILLVAL 0x38 103 #define CH_SRCTRIGINCFG 0x4c 104 #define CH_DESTRIGINCFG 0x50 105 #define CH_LINKATTR 0x70 106 #define CH_LINK_SHAREATTR GENMASK(9, 8) 107 #define CH_LINK_MEMATTR GENMASK(7, 0) 108 109 #define CH_AUTOCFG 0x74 110 #define CH_LINKADDR 0x78 111 #define CH_LINKADDR_EN BIT(0) 112 113 #define CH_LINKADDRHI 0x7c 114 #define CH_ERRINFO 0x90 115 #define CH_ERRINFO_AXIRDPOISERR BIT(18) 116 #define CH_ERRINFO_AXIWRRESPERR BIT(17) 117 #define CH_ERRINFO_AXIRDRESPERR BIT(16) 118 119 #define CH_BUILDCFG0 0xf8 120 #define CH_CFG_INC_WIDTH GENMASK(29, 26) 121 #define CH_CFG_DATA_WIDTH GENMASK(24, 22) 122 #define CH_CFG_DATA_BUF_SIZE GENMASK(7, 0) 123 124 #define CH_BUILDCFG1 0xfc 125 #define CH_CFG_HAS_CMDLINK BIT(8) 126 #define CH_CFG_HAS_TRIGSEL BIT(7) 127 #define CH_CFG_HAS_TRIGIN BIT(5) 128 #define CH_CFG_HAS_WRAP BIT(1) 129 130 131 #define LINK_REGCLEAR BIT(0) 132 #define LINK_INTREN BIT(2) 133 #define LINK_CTRL BIT(3) 134 #define LINK_SRCADDR BIT(4) 135 #define LINK_SRCADDRHI BIT(5) 136 #define LINK_DESADDR BIT(6) 137 #define LINK_DESADDRHI BIT(7) 138 #define LINK_XSIZE BIT(8) 139 #define LINK_XSIZEHI BIT(9) 140 #define LINK_SRCTRANSCFG BIT(10) 141 #define LINK_DESTRANSCFG BIT(11) 142 #define LINK_XADDRINC BIT(12) 143 #define LINK_FILLVAL BIT(14) 144 #define LINK_SRCTRIGINCFG BIT(19) 145 #define LINK_DESTRIGINCFG BIT(20) 146 #define LINK_AUTOCFG BIT(29) 147 #define LINK_LINKADDR BIT(30) 148 #define LINK_LINKADDRHI BIT(31) 149 150 151 enum ch_ctrl_donetype { 152 CH_CTRL_DONETYPE_NONE = 0, 153 CH_CTRL_DONETYPE_CMD = 1, 154 CH_CTRL_DONETYPE_CYCLE = 3 155 }; 156 157 enum ch_ctrl_xtype { 158 CH_CTRL_XTYPE_DISABLE = 0, 159 CH_CTRL_XTYPE_CONTINUE = 1, 160 CH_CTRL_XTYPE_WRAP = 2, 161 CH_CTRL_XTYPE_FILL = 3 162 }; 163 164 enum ch_cfg_shareattr { 165 SHAREATTR_NSH = 0, 166 SHAREATTR_OSH = 2, 167 SHAREATTR_ISH = 3 168 }; 169 170 enum ch_cfg_memattr { 171 MEMATTR_DEVICE = 0x00, 172 MEMATTR_NC = 0x44, 173 MEMATTR_WB = 0xff 174 }; 175 176 struct d350_desc { 177 struct virt_dma_desc vd; 178 u32 command[16]; 179 u16 xsize; 180 u16 xsizehi; 181 u8 tsz; 182 }; 183 184 struct d350_chan { 185 struct virt_dma_chan vc; 186 struct d350_desc *desc; 187 void __iomem *base; 188 int irq; 189 enum dma_status status; 190 dma_cookie_t cookie; 191 u32 residue; 192 u8 tsz; 193 bool has_trig; 194 bool has_wrap; 195 bool coherent; 196 }; 197 198 struct d350 { 199 struct dma_device dma; 200 int nchan; 201 int nreq; 202 struct d350_chan channels[] __counted_by(nchan); 203 }; 204 205 static inline struct d350_chan *to_d350_chan(struct dma_chan *chan) 206 { 207 return container_of(chan, struct d350_chan, vc.chan); 208 } 209 210 static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd) 211 { 212 return container_of(vd, struct d350_desc, vd); 213 } 214 215 static void d350_desc_free(struct virt_dma_desc *vd) 216 { 217 kfree(to_d350_desc(vd)); 218 } 219 220 static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan, 221 dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) 222 { 223 struct d350_chan *dch = to_d350_chan(chan); 224 struct d350_desc *desc; 225 u32 *cmd; 226 227 desc = kzalloc_obj(*desc, GFP_NOWAIT); 228 if (!desc) 229 return NULL; 230 231 desc->tsz = __ffs(len | dest | src | (1 << dch->tsz)); 232 desc->xsize = lower_16_bits(len >> desc->tsz); 233 desc->xsizehi = upper_16_bits(len >> desc->tsz); 234 235 cmd = desc->command; 236 cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR | 237 LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG | 238 LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR; 239 240 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | 241 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) | 242 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); 243 244 cmd[2] = lower_32_bits(src); 245 cmd[3] = upper_32_bits(src); 246 cmd[4] = lower_32_bits(dest); 247 cmd[5] = upper_32_bits(dest); 248 cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize); 249 cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi); 250 cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; 251 cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; 252 cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1); 253 cmd[11] = 0; 254 255 return vchan_tx_prep(&dch->vc, &desc->vd, flags); 256 } 257 258 static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan, 259 dma_addr_t dest, int value, size_t len, unsigned long flags) 260 { 261 struct d350_chan *dch = to_d350_chan(chan); 262 struct d350_desc *desc; 263 u32 *cmd; 264 265 desc = kzalloc_obj(*desc, GFP_NOWAIT); 266 if (!desc) 267 return NULL; 268 269 desc->tsz = __ffs(len | dest | (1 << dch->tsz)); 270 desc->xsize = lower_16_bits(len >> desc->tsz); 271 desc->xsizehi = upper_16_bits(len >> desc->tsz); 272 273 cmd = desc->command; 274 cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI | 275 LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG | 276 LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR; 277 278 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | 279 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) | 280 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); 281 282 cmd[2] = lower_32_bits(dest); 283 cmd[3] = upper_32_bits(dest); 284 cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize); 285 cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi); 286 cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; 287 cmd[7] = FIELD_PREP(CH_XY_DES, 1); 288 cmd[8] = (u8)value * 0x01010101; 289 cmd[9] = 0; 290 291 return vchan_tx_prep(&dch->vc, &desc->vd, flags); 292 } 293 294 static int d350_pause(struct dma_chan *chan) 295 { 296 struct d350_chan *dch = to_d350_chan(chan); 297 unsigned long flags; 298 299 spin_lock_irqsave(&dch->vc.lock, flags); 300 if (dch->status == DMA_IN_PROGRESS) { 301 writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD); 302 dch->status = DMA_PAUSED; 303 } 304 spin_unlock_irqrestore(&dch->vc.lock, flags); 305 306 return 0; 307 } 308 309 static int d350_resume(struct dma_chan *chan) 310 { 311 struct d350_chan *dch = to_d350_chan(chan); 312 unsigned long flags; 313 314 spin_lock_irqsave(&dch->vc.lock, flags); 315 if (dch->status == DMA_PAUSED) { 316 writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD); 317 dch->status = DMA_IN_PROGRESS; 318 } 319 spin_unlock_irqrestore(&dch->vc.lock, flags); 320 321 return 0; 322 } 323 324 static u32 d350_get_residue(struct d350_chan *dch) 325 { 326 u32 res, xsize, xsizehi, hi_new; 327 int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ 328 329 hi_new = readl_relaxed(dch->base + CH_XSIZEHI); 330 do { 331 xsizehi = hi_new; 332 xsize = readl_relaxed(dch->base + CH_XSIZE); 333 hi_new = readl_relaxed(dch->base + CH_XSIZEHI); 334 } while (xsizehi != hi_new && --retries); 335 336 res = FIELD_GET(CH_XY_DES, xsize); 337 res |= FIELD_GET(CH_XY_DES, xsizehi) << 16; 338 339 return res << dch->desc->tsz; 340 } 341 342 static int d350_terminate_all(struct dma_chan *chan) 343 { 344 struct d350_chan *dch = to_d350_chan(chan); 345 unsigned long flags; 346 LIST_HEAD(list); 347 348 spin_lock_irqsave(&dch->vc.lock, flags); 349 writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD); 350 if (dch->desc) { 351 if (dch->status != DMA_ERROR) 352 vchan_terminate_vdesc(&dch->desc->vd); 353 dch->desc = NULL; 354 dch->status = DMA_COMPLETE; 355 } 356 vchan_get_all_descriptors(&dch->vc, &list); 357 list_splice_tail(&list, &dch->vc.desc_terminated); 358 spin_unlock_irqrestore(&dch->vc.lock, flags); 359 360 return 0; 361 } 362 363 static void d350_synchronize(struct dma_chan *chan) 364 { 365 struct d350_chan *dch = to_d350_chan(chan); 366 367 vchan_synchronize(&dch->vc); 368 } 369 370 static u32 d350_desc_bytes(struct d350_desc *desc) 371 { 372 return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz; 373 } 374 375 static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 376 struct dma_tx_state *state) 377 { 378 struct d350_chan *dch = to_d350_chan(chan); 379 struct virt_dma_desc *vd; 380 enum dma_status status; 381 unsigned long flags; 382 u32 residue = 0; 383 384 status = dma_cookie_status(chan, cookie, state); 385 386 spin_lock_irqsave(&dch->vc.lock, flags); 387 if (cookie == dch->cookie) { 388 status = dch->status; 389 if (status == DMA_IN_PROGRESS || status == DMA_PAUSED) 390 dch->residue = d350_get_residue(dch); 391 residue = dch->residue; 392 } else if ((vd = vchan_find_desc(&dch->vc, cookie))) { 393 residue = d350_desc_bytes(to_d350_desc(vd)); 394 } else if (status == DMA_IN_PROGRESS) { 395 /* Somebody else terminated it? */ 396 status = DMA_ERROR; 397 } 398 spin_unlock_irqrestore(&dch->vc.lock, flags); 399 400 dma_set_residue(state, residue); 401 return status; 402 } 403 404 static void d350_start_next(struct d350_chan *dch) 405 { 406 u32 hdr, *reg; 407 408 dch->desc = to_d350_desc(vchan_next_desc(&dch->vc)); 409 if (!dch->desc) 410 return; 411 412 list_del(&dch->desc->vd.node); 413 dch->status = DMA_IN_PROGRESS; 414 dch->cookie = dch->desc->vd.tx.cookie; 415 dch->residue = d350_desc_bytes(dch->desc); 416 417 hdr = dch->desc->command[0]; 418 reg = &dch->desc->command[1]; 419 420 if (hdr & LINK_INTREN) 421 writel_relaxed(*reg++, dch->base + CH_INTREN); 422 if (hdr & LINK_CTRL) 423 writel_relaxed(*reg++, dch->base + CH_CTRL); 424 if (hdr & LINK_SRCADDR) 425 writel_relaxed(*reg++, dch->base + CH_SRCADDR); 426 if (hdr & LINK_SRCADDRHI) 427 writel_relaxed(*reg++, dch->base + CH_SRCADDRHI); 428 if (hdr & LINK_DESADDR) 429 writel_relaxed(*reg++, dch->base + CH_DESADDR); 430 if (hdr & LINK_DESADDRHI) 431 writel_relaxed(*reg++, dch->base + CH_DESADDRHI); 432 if (hdr & LINK_XSIZE) 433 writel_relaxed(*reg++, dch->base + CH_XSIZE); 434 if (hdr & LINK_XSIZEHI) 435 writel_relaxed(*reg++, dch->base + CH_XSIZEHI); 436 if (hdr & LINK_SRCTRANSCFG) 437 writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG); 438 if (hdr & LINK_DESTRANSCFG) 439 writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG); 440 if (hdr & LINK_XADDRINC) 441 writel_relaxed(*reg++, dch->base + CH_XADDRINC); 442 if (hdr & LINK_FILLVAL) 443 writel_relaxed(*reg++, dch->base + CH_FILLVAL); 444 if (hdr & LINK_SRCTRIGINCFG) 445 writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG); 446 if (hdr & LINK_DESTRIGINCFG) 447 writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG); 448 if (hdr & LINK_AUTOCFG) 449 writel_relaxed(*reg++, dch->base + CH_AUTOCFG); 450 if (hdr & LINK_LINKADDR) 451 writel_relaxed(*reg++, dch->base + CH_LINKADDR); 452 if (hdr & LINK_LINKADDRHI) 453 writel_relaxed(*reg++, dch->base + CH_LINKADDRHI); 454 455 writel(CH_CMD_ENABLE, dch->base + CH_CMD); 456 } 457 458 static void d350_issue_pending(struct dma_chan *chan) 459 { 460 struct d350_chan *dch = to_d350_chan(chan); 461 unsigned long flags; 462 463 spin_lock_irqsave(&dch->vc.lock, flags); 464 if (vchan_issue_pending(&dch->vc) && !dch->desc) 465 d350_start_next(dch); 466 spin_unlock_irqrestore(&dch->vc.lock, flags); 467 } 468 469 static irqreturn_t d350_irq(int irq, void *data) 470 { 471 struct d350_chan *dch = data; 472 struct device *dev = dch->vc.chan.device->dev; 473 struct virt_dma_desc *vd = &dch->desc->vd; 474 u32 ch_status; 475 476 ch_status = readl(dch->base + CH_STATUS); 477 if (!ch_status) 478 return IRQ_NONE; 479 480 if (ch_status & CH_STAT_INTR_ERR) { 481 u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO); 482 483 if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR)) 484 vd->tx_result.result = DMA_TRANS_READ_FAILED; 485 else if (errinfo & CH_ERRINFO_AXIWRRESPERR) 486 vd->tx_result.result = DMA_TRANS_WRITE_FAILED; 487 else 488 vd->tx_result.result = DMA_TRANS_ABORTED; 489 490 vd->tx_result.residue = d350_get_residue(dch); 491 } else if (!(ch_status & CH_STAT_INTR_DONE)) { 492 dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status); 493 } 494 writel_relaxed(ch_status, dch->base + CH_STATUS); 495 496 spin_lock(&dch->vc.lock); 497 vchan_cookie_complete(vd); 498 if (ch_status & CH_STAT_INTR_DONE) { 499 dch->status = DMA_COMPLETE; 500 dch->residue = 0; 501 d350_start_next(dch); 502 } else { 503 dch->status = DMA_ERROR; 504 dch->residue = vd->tx_result.residue; 505 } 506 spin_unlock(&dch->vc.lock); 507 508 return IRQ_HANDLED; 509 } 510 511 static int d350_alloc_chan_resources(struct dma_chan *chan) 512 { 513 struct d350_chan *dch = to_d350_chan(chan); 514 int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED, 515 dev_name(&dch->vc.chan.dev->device), dch); 516 if (!ret) 517 writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN); 518 519 return ret; 520 } 521 522 static void d350_free_chan_resources(struct dma_chan *chan) 523 { 524 struct d350_chan *dch = to_d350_chan(chan); 525 526 writel_relaxed(0, dch->base + CH_INTREN); 527 free_irq(dch->irq, dch); 528 vchan_free_chan_resources(&dch->vc); 529 } 530 531 static int d350_probe(struct platform_device *pdev) 532 { 533 struct device *dev = &pdev->dev; 534 struct d350 *dmac; 535 void __iomem *base; 536 u32 reg; 537 int ret, nchan, dw, aw, r, p; 538 bool coherent, memset; 539 540 base = devm_platform_ioremap_resource(pdev, 0); 541 if (IS_ERR(base)) 542 return PTR_ERR(base); 543 544 reg = readl_relaxed(base + DMAINFO + IIDR); 545 r = FIELD_GET(IIDR_VARIANT, reg); 546 p = FIELD_GET(IIDR_REVISION, reg); 547 if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM || 548 FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350) 549 return dev_err_probe(dev, -ENODEV, "Not a DMA-350!"); 550 551 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0); 552 nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1; 553 dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg); 554 aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1; 555 556 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw)); 557 coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT; 558 559 dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL); 560 if (!dmac) 561 return -ENOMEM; 562 563 dmac->nchan = nchan; 564 565 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1); 566 dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg); 567 568 dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq); 569 570 dmac->dma.dev = dev; 571 for (int i = min(dw, 16); i > 0; i /= 2) { 572 dmac->dma.src_addr_widths |= BIT(i); 573 dmac->dma.dst_addr_widths |= BIT(i); 574 } 575 dmac->dma.directions = BIT(DMA_MEM_TO_MEM); 576 dmac->dma.descriptor_reuse = true; 577 dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 578 dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources; 579 dmac->dma.device_free_chan_resources = d350_free_chan_resources; 580 dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask); 581 dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy; 582 dmac->dma.device_pause = d350_pause; 583 dmac->dma.device_resume = d350_resume; 584 dmac->dma.device_terminate_all = d350_terminate_all; 585 dmac->dma.device_synchronize = d350_synchronize; 586 dmac->dma.device_tx_status = d350_tx_status; 587 dmac->dma.device_issue_pending = d350_issue_pending; 588 INIT_LIST_HEAD(&dmac->dma.channels); 589 590 reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL); 591 writel_relaxed(reg | INTREN_ANYCHINTR_EN, 592 base + DMANSECCTRL + NSEC_CTRL); 593 594 /* Would be nice to have per-channel caps for this... */ 595 memset = true; 596 for (int i = 0; i < nchan; i++) { 597 struct d350_chan *dch = &dmac->channels[i]; 598 599 dch->base = base + DMACH(i); 600 writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD); 601 602 reg = readl_relaxed(dch->base + CH_BUILDCFG1); 603 if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) { 604 dev_warn(dev, "No command link support on channel %d\n", i); 605 continue; 606 } 607 dch->irq = platform_get_irq(pdev, i); 608 if (dch->irq < 0) 609 return dev_err_probe(dev, dch->irq, 610 "Failed to get IRQ for channel %d\n", i); 611 612 dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg); 613 dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) & 614 FIELD_GET(CH_CFG_HAS_TRIGSEL, reg); 615 616 /* Fill is a special case of Wrap */ 617 memset &= dch->has_wrap; 618 619 reg = readl_relaxed(dch->base + CH_BUILDCFG0); 620 dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg); 621 622 reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH); 623 reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC); 624 writel_relaxed(reg, dch->base + CH_LINKATTR); 625 626 dch->vc.desc_free = d350_desc_free; 627 vchan_init(&dch->vc, &dmac->dma); 628 } 629 630 if (memset) { 631 dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask); 632 dmac->dma.device_prep_dma_memset = d350_prep_memset; 633 } 634 635 platform_set_drvdata(pdev, dmac); 636 637 ret = dma_async_device_register(&dmac->dma); 638 if (ret) 639 return dev_err_probe(dev, ret, "Failed to register DMA device\n"); 640 641 return 0; 642 } 643 644 static void d350_remove(struct platform_device *pdev) 645 { 646 struct d350 *dmac = platform_get_drvdata(pdev); 647 648 dma_async_device_unregister(&dmac->dma); 649 } 650 651 static const struct of_device_id d350_of_match[] __maybe_unused = { 652 { .compatible = "arm,dma-350" }, 653 {} 654 }; 655 MODULE_DEVICE_TABLE(of, d350_of_match); 656 657 static struct platform_driver d350_driver = { 658 .driver = { 659 .name = "arm-dma350", 660 .of_match_table = of_match_ptr(d350_of_match), 661 }, 662 .probe = d350_probe, 663 .remove = d350_remove, 664 }; 665 module_platform_driver(d350_driver); 666 667 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); 668 MODULE_DESCRIPTION("Arm DMA-350 driver"); 669 MODULE_LICENSE("GPL v2"); 670