1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2024-2025 Arm Limited 3 // Arm DMA-350 driver 4 5 #include <linux/bitfield.h> 6 #include <linux/dmaengine.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/io.h> 9 #include <linux/of.h> 10 #include <linux/module.h> 11 #include <linux/platform_device.h> 12 13 #include "dmaengine.h" 14 #include "virt-dma.h" 15 16 #define DMAINFO 0x0f00 17 18 #define DMA_BUILDCFG0 0xb0 19 #define DMA_CFG_DATA_WIDTH GENMASK(18, 16) 20 #define DMA_CFG_ADDR_WIDTH GENMASK(15, 10) 21 #define DMA_CFG_NUM_CHANNELS GENMASK(9, 4) 22 23 #define DMA_BUILDCFG1 0xb4 24 #define DMA_CFG_NUM_TRIGGER_IN GENMASK(8, 0) 25 26 #define IIDR 0xc8 27 #define IIDR_PRODUCTID GENMASK(31, 20) 28 #define IIDR_VARIANT GENMASK(19, 16) 29 #define IIDR_REVISION GENMASK(15, 12) 30 #define IIDR_IMPLEMENTER GENMASK(11, 0) 31 32 #define PRODUCTID_DMA350 0x3a0 33 #define IMPLEMENTER_ARM 0x43b 34 35 #define DMACH(n) (0x1000 + 0x0100 * (n)) 36 37 #define CH_CMD 0x00 38 #define CH_CMD_RESUME BIT(5) 39 #define CH_CMD_PAUSE BIT(4) 40 #define CH_CMD_STOP BIT(3) 41 #define CH_CMD_DISABLE BIT(2) 42 #define CH_CMD_CLEAR BIT(1) 43 #define CH_CMD_ENABLE BIT(0) 44 45 #define CH_STATUS 0x04 46 #define CH_STAT_RESUMEWAIT BIT(21) 47 #define CH_STAT_PAUSED BIT(20) 48 #define CH_STAT_STOPPED BIT(19) 49 #define CH_STAT_DISABLED BIT(18) 50 #define CH_STAT_ERR BIT(17) 51 #define CH_STAT_DONE BIT(16) 52 #define CH_STAT_INTR_ERR BIT(1) 53 #define CH_STAT_INTR_DONE BIT(0) 54 55 #define CH_INTREN 0x08 56 #define CH_INTREN_ERR BIT(1) 57 #define CH_INTREN_DONE BIT(0) 58 59 #define CH_CTRL 0x0c 60 #define CH_CTRL_USEDESTRIGIN BIT(26) 61 #define CH_CTRL_USESRCTRIGIN BIT(26) 62 #define CH_CTRL_DONETYPE GENMASK(23, 21) 63 #define CH_CTRL_REGRELOADTYPE GENMASK(20, 18) 64 #define CH_CTRL_XTYPE GENMASK(11, 9) 65 #define CH_CTRL_TRANSIZE GENMASK(2, 0) 66 67 #define CH_SRCADDR 0x10 68 #define CH_SRCADDRHI 0x14 69 #define CH_DESADDR 0x18 70 #define CH_DESADDRHI 0x1c 71 #define CH_XSIZE 0x20 72 #define CH_XSIZEHI 0x24 73 #define CH_SRCTRANSCFG 0x28 74 #define CH_DESTRANSCFG 0x2c 75 #define CH_CFG_MAXBURSTLEN GENMASK(19, 16) 76 #define CH_CFG_PRIVATTR BIT(11) 77 #define CH_CFG_SHAREATTR GENMASK(9, 8) 78 #define CH_CFG_MEMATTR GENMASK(7, 0) 79 80 #define TRANSCFG_DEVICE \ 81 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ 82 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ 83 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE) 84 #define TRANSCFG_NC \ 85 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ 86 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ 87 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC) 88 #define TRANSCFG_WB \ 89 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ 90 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) | \ 91 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB) 92 93 #define CH_XADDRINC 0x30 94 #define CH_XY_DES GENMASK(31, 16) 95 #define CH_XY_SRC GENMASK(15, 0) 96 97 #define CH_FILLVAL 0x38 98 #define CH_SRCTRIGINCFG 0x4c 99 #define CH_DESTRIGINCFG 0x50 100 #define CH_LINKATTR 0x70 101 #define CH_LINK_SHAREATTR GENMASK(9, 8) 102 #define CH_LINK_MEMATTR GENMASK(7, 0) 103 104 #define CH_AUTOCFG 0x74 105 #define CH_LINKADDR 0x78 106 #define CH_LINKADDR_EN BIT(0) 107 108 #define CH_LINKADDRHI 0x7c 109 #define CH_ERRINFO 0x90 110 #define CH_ERRINFO_AXIRDPOISERR BIT(18) 111 #define CH_ERRINFO_AXIWRRESPERR BIT(17) 112 #define CH_ERRINFO_AXIRDRESPERR BIT(16) 113 114 #define CH_BUILDCFG0 0xf8 115 #define CH_CFG_INC_WIDTH GENMASK(29, 26) 116 #define CH_CFG_DATA_WIDTH GENMASK(24, 22) 117 #define CH_CFG_DATA_BUF_SIZE GENMASK(7, 0) 118 119 #define CH_BUILDCFG1 0xfc 120 #define CH_CFG_HAS_CMDLINK BIT(8) 121 #define CH_CFG_HAS_TRIGSEL BIT(7) 122 #define CH_CFG_HAS_TRIGIN BIT(5) 123 #define CH_CFG_HAS_WRAP BIT(1) 124 125 126 #define LINK_REGCLEAR BIT(0) 127 #define LINK_INTREN BIT(2) 128 #define LINK_CTRL BIT(3) 129 #define LINK_SRCADDR BIT(4) 130 #define LINK_SRCADDRHI BIT(5) 131 #define LINK_DESADDR BIT(6) 132 #define LINK_DESADDRHI BIT(7) 133 #define LINK_XSIZE BIT(8) 134 #define LINK_XSIZEHI BIT(9) 135 #define LINK_SRCTRANSCFG BIT(10) 136 #define LINK_DESTRANSCFG BIT(11) 137 #define LINK_XADDRINC BIT(12) 138 #define LINK_FILLVAL BIT(14) 139 #define LINK_SRCTRIGINCFG BIT(19) 140 #define LINK_DESTRIGINCFG BIT(20) 141 #define LINK_AUTOCFG BIT(29) 142 #define LINK_LINKADDR BIT(30) 143 #define LINK_LINKADDRHI BIT(31) 144 145 146 enum ch_ctrl_donetype { 147 CH_CTRL_DONETYPE_NONE = 0, 148 CH_CTRL_DONETYPE_CMD = 1, 149 CH_CTRL_DONETYPE_CYCLE = 3 150 }; 151 152 enum ch_ctrl_xtype { 153 CH_CTRL_XTYPE_DISABLE = 0, 154 CH_CTRL_XTYPE_CONTINUE = 1, 155 CH_CTRL_XTYPE_WRAP = 2, 156 CH_CTRL_XTYPE_FILL = 3 157 }; 158 159 enum ch_cfg_shareattr { 160 SHAREATTR_NSH = 0, 161 SHAREATTR_OSH = 2, 162 SHAREATTR_ISH = 3 163 }; 164 165 enum ch_cfg_memattr { 166 MEMATTR_DEVICE = 0x00, 167 MEMATTR_NC = 0x44, 168 MEMATTR_WB = 0xff 169 }; 170 171 struct d350_desc { 172 struct virt_dma_desc vd; 173 u32 command[16]; 174 u16 xsize; 175 u16 xsizehi; 176 u8 tsz; 177 }; 178 179 struct d350_chan { 180 struct virt_dma_chan vc; 181 struct d350_desc *desc; 182 void __iomem *base; 183 int irq; 184 enum dma_status status; 185 dma_cookie_t cookie; 186 u32 residue; 187 u8 tsz; 188 bool has_trig; 189 bool has_wrap; 190 bool coherent; 191 }; 192 193 struct d350 { 194 struct dma_device dma; 195 int nchan; 196 int nreq; 197 struct d350_chan channels[] __counted_by(nchan); 198 }; 199 200 static inline struct d350_chan *to_d350_chan(struct dma_chan *chan) 201 { 202 return container_of(chan, struct d350_chan, vc.chan); 203 } 204 205 static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd) 206 { 207 return container_of(vd, struct d350_desc, vd); 208 } 209 210 static void d350_desc_free(struct virt_dma_desc *vd) 211 { 212 kfree(to_d350_desc(vd)); 213 } 214 215 static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan, 216 dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) 217 { 218 struct d350_chan *dch = to_d350_chan(chan); 219 struct d350_desc *desc; 220 u32 *cmd; 221 222 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 223 if (!desc) 224 return NULL; 225 226 desc->tsz = __ffs(len | dest | src | (1 << dch->tsz)); 227 desc->xsize = lower_16_bits(len >> desc->tsz); 228 desc->xsizehi = upper_16_bits(len >> desc->tsz); 229 230 cmd = desc->command; 231 cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR | 232 LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG | 233 LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR; 234 235 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | 236 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) | 237 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); 238 239 cmd[2] = lower_32_bits(src); 240 cmd[3] = upper_32_bits(src); 241 cmd[4] = lower_32_bits(dest); 242 cmd[5] = upper_32_bits(dest); 243 cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize); 244 cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi); 245 cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; 246 cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; 247 cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1); 248 cmd[11] = 0; 249 250 return vchan_tx_prep(&dch->vc, &desc->vd, flags); 251 } 252 253 static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan, 254 dma_addr_t dest, int value, size_t len, unsigned long flags) 255 { 256 struct d350_chan *dch = to_d350_chan(chan); 257 struct d350_desc *desc; 258 u32 *cmd; 259 260 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 261 if (!desc) 262 return NULL; 263 264 desc->tsz = __ffs(len | dest | (1 << dch->tsz)); 265 desc->xsize = lower_16_bits(len >> desc->tsz); 266 desc->xsizehi = upper_16_bits(len >> desc->tsz); 267 268 cmd = desc->command; 269 cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI | 270 LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG | 271 LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR; 272 273 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | 274 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) | 275 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); 276 277 cmd[2] = lower_32_bits(dest); 278 cmd[3] = upper_32_bits(dest); 279 cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize); 280 cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi); 281 cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; 282 cmd[7] = FIELD_PREP(CH_XY_DES, 1); 283 cmd[8] = (u8)value * 0x01010101; 284 cmd[9] = 0; 285 286 return vchan_tx_prep(&dch->vc, &desc->vd, flags); 287 } 288 289 static int d350_pause(struct dma_chan *chan) 290 { 291 struct d350_chan *dch = to_d350_chan(chan); 292 unsigned long flags; 293 294 spin_lock_irqsave(&dch->vc.lock, flags); 295 if (dch->status == DMA_IN_PROGRESS) { 296 writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD); 297 dch->status = DMA_PAUSED; 298 } 299 spin_unlock_irqrestore(&dch->vc.lock, flags); 300 301 return 0; 302 } 303 304 static int d350_resume(struct dma_chan *chan) 305 { 306 struct d350_chan *dch = to_d350_chan(chan); 307 unsigned long flags; 308 309 spin_lock_irqsave(&dch->vc.lock, flags); 310 if (dch->status == DMA_PAUSED) { 311 writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD); 312 dch->status = DMA_IN_PROGRESS; 313 } 314 spin_unlock_irqrestore(&dch->vc.lock, flags); 315 316 return 0; 317 } 318 319 static u32 d350_get_residue(struct d350_chan *dch) 320 { 321 u32 res, xsize, xsizehi, hi_new; 322 int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ 323 324 hi_new = readl_relaxed(dch->base + CH_XSIZEHI); 325 do { 326 xsizehi = hi_new; 327 xsize = readl_relaxed(dch->base + CH_XSIZE); 328 hi_new = readl_relaxed(dch->base + CH_XSIZEHI); 329 } while (xsizehi != hi_new && --retries); 330 331 res = FIELD_GET(CH_XY_DES, xsize); 332 res |= FIELD_GET(CH_XY_DES, xsizehi) << 16; 333 334 return res << dch->desc->tsz; 335 } 336 337 static int d350_terminate_all(struct dma_chan *chan) 338 { 339 struct d350_chan *dch = to_d350_chan(chan); 340 unsigned long flags; 341 LIST_HEAD(list); 342 343 spin_lock_irqsave(&dch->vc.lock, flags); 344 writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD); 345 if (dch->desc) { 346 if (dch->status != DMA_ERROR) 347 vchan_terminate_vdesc(&dch->desc->vd); 348 dch->desc = NULL; 349 dch->status = DMA_COMPLETE; 350 } 351 vchan_get_all_descriptors(&dch->vc, &list); 352 list_splice_tail(&list, &dch->vc.desc_terminated); 353 spin_unlock_irqrestore(&dch->vc.lock, flags); 354 355 return 0; 356 } 357 358 static void d350_synchronize(struct dma_chan *chan) 359 { 360 struct d350_chan *dch = to_d350_chan(chan); 361 362 vchan_synchronize(&dch->vc); 363 } 364 365 static u32 d350_desc_bytes(struct d350_desc *desc) 366 { 367 return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz; 368 } 369 370 static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 371 struct dma_tx_state *state) 372 { 373 struct d350_chan *dch = to_d350_chan(chan); 374 struct virt_dma_desc *vd; 375 enum dma_status status; 376 unsigned long flags; 377 u32 residue = 0; 378 379 status = dma_cookie_status(chan, cookie, state); 380 381 spin_lock_irqsave(&dch->vc.lock, flags); 382 if (cookie == dch->cookie) { 383 status = dch->status; 384 if (status == DMA_IN_PROGRESS || status == DMA_PAUSED) 385 dch->residue = d350_get_residue(dch); 386 residue = dch->residue; 387 } else if ((vd = vchan_find_desc(&dch->vc, cookie))) { 388 residue = d350_desc_bytes(to_d350_desc(vd)); 389 } else if (status == DMA_IN_PROGRESS) { 390 /* Somebody else terminated it? */ 391 status = DMA_ERROR; 392 } 393 spin_unlock_irqrestore(&dch->vc.lock, flags); 394 395 dma_set_residue(state, residue); 396 return status; 397 } 398 399 static void d350_start_next(struct d350_chan *dch) 400 { 401 u32 hdr, *reg; 402 403 dch->desc = to_d350_desc(vchan_next_desc(&dch->vc)); 404 if (!dch->desc) 405 return; 406 407 list_del(&dch->desc->vd.node); 408 dch->status = DMA_IN_PROGRESS; 409 dch->cookie = dch->desc->vd.tx.cookie; 410 dch->residue = d350_desc_bytes(dch->desc); 411 412 hdr = dch->desc->command[0]; 413 reg = &dch->desc->command[1]; 414 415 if (hdr & LINK_INTREN) 416 writel_relaxed(*reg++, dch->base + CH_INTREN); 417 if (hdr & LINK_CTRL) 418 writel_relaxed(*reg++, dch->base + CH_CTRL); 419 if (hdr & LINK_SRCADDR) 420 writel_relaxed(*reg++, dch->base + CH_SRCADDR); 421 if (hdr & LINK_SRCADDRHI) 422 writel_relaxed(*reg++, dch->base + CH_SRCADDRHI); 423 if (hdr & LINK_DESADDR) 424 writel_relaxed(*reg++, dch->base + CH_DESADDR); 425 if (hdr & LINK_DESADDRHI) 426 writel_relaxed(*reg++, dch->base + CH_DESADDRHI); 427 if (hdr & LINK_XSIZE) 428 writel_relaxed(*reg++, dch->base + CH_XSIZE); 429 if (hdr & LINK_XSIZEHI) 430 writel_relaxed(*reg++, dch->base + CH_XSIZEHI); 431 if (hdr & LINK_SRCTRANSCFG) 432 writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG); 433 if (hdr & LINK_DESTRANSCFG) 434 writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG); 435 if (hdr & LINK_XADDRINC) 436 writel_relaxed(*reg++, dch->base + CH_XADDRINC); 437 if (hdr & LINK_FILLVAL) 438 writel_relaxed(*reg++, dch->base + CH_FILLVAL); 439 if (hdr & LINK_SRCTRIGINCFG) 440 writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG); 441 if (hdr & LINK_DESTRIGINCFG) 442 writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG); 443 if (hdr & LINK_AUTOCFG) 444 writel_relaxed(*reg++, dch->base + CH_AUTOCFG); 445 if (hdr & LINK_LINKADDR) 446 writel_relaxed(*reg++, dch->base + CH_LINKADDR); 447 if (hdr & LINK_LINKADDRHI) 448 writel_relaxed(*reg++, dch->base + CH_LINKADDRHI); 449 450 writel(CH_CMD_ENABLE, dch->base + CH_CMD); 451 } 452 453 static void d350_issue_pending(struct dma_chan *chan) 454 { 455 struct d350_chan *dch = to_d350_chan(chan); 456 unsigned long flags; 457 458 spin_lock_irqsave(&dch->vc.lock, flags); 459 if (vchan_issue_pending(&dch->vc) && !dch->desc) 460 d350_start_next(dch); 461 spin_unlock_irqrestore(&dch->vc.lock, flags); 462 } 463 464 static irqreturn_t d350_irq(int irq, void *data) 465 { 466 struct d350_chan *dch = data; 467 struct device *dev = dch->vc.chan.device->dev; 468 struct virt_dma_desc *vd = &dch->desc->vd; 469 u32 ch_status; 470 471 ch_status = readl(dch->base + CH_STATUS); 472 if (!ch_status) 473 return IRQ_NONE; 474 475 if (ch_status & CH_STAT_INTR_ERR) { 476 u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO); 477 478 if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR)) 479 vd->tx_result.result = DMA_TRANS_READ_FAILED; 480 else if (errinfo & CH_ERRINFO_AXIWRRESPERR) 481 vd->tx_result.result = DMA_TRANS_WRITE_FAILED; 482 else 483 vd->tx_result.result = DMA_TRANS_ABORTED; 484 485 vd->tx_result.residue = d350_get_residue(dch); 486 } else if (!(ch_status & CH_STAT_INTR_DONE)) { 487 dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status); 488 } 489 writel_relaxed(ch_status, dch->base + CH_STATUS); 490 491 spin_lock(&dch->vc.lock); 492 vchan_cookie_complete(vd); 493 if (ch_status & CH_STAT_INTR_DONE) { 494 dch->status = DMA_COMPLETE; 495 dch->residue = 0; 496 d350_start_next(dch); 497 } else { 498 dch->status = DMA_ERROR; 499 dch->residue = vd->tx_result.residue; 500 } 501 spin_unlock(&dch->vc.lock); 502 503 return IRQ_HANDLED; 504 } 505 506 static int d350_alloc_chan_resources(struct dma_chan *chan) 507 { 508 struct d350_chan *dch = to_d350_chan(chan); 509 int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED, 510 dev_name(&dch->vc.chan.dev->device), dch); 511 if (!ret) 512 writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN); 513 514 return ret; 515 } 516 517 static void d350_free_chan_resources(struct dma_chan *chan) 518 { 519 struct d350_chan *dch = to_d350_chan(chan); 520 521 writel_relaxed(0, dch->base + CH_INTREN); 522 free_irq(dch->irq, dch); 523 vchan_free_chan_resources(&dch->vc); 524 } 525 526 static int d350_probe(struct platform_device *pdev) 527 { 528 struct device *dev = &pdev->dev; 529 struct d350 *dmac; 530 void __iomem *base; 531 u32 reg; 532 int ret, nchan, dw, aw, r, p; 533 bool coherent, memset; 534 535 base = devm_platform_ioremap_resource(pdev, 0); 536 if (IS_ERR(base)) 537 return PTR_ERR(base); 538 539 reg = readl_relaxed(base + DMAINFO + IIDR); 540 r = FIELD_GET(IIDR_VARIANT, reg); 541 p = FIELD_GET(IIDR_REVISION, reg); 542 if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM || 543 FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350) 544 return dev_err_probe(dev, -ENODEV, "Not a DMA-350!"); 545 546 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0); 547 nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1; 548 dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg); 549 aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1; 550 551 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw)); 552 coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT; 553 554 dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL); 555 if (!dmac) 556 return -ENOMEM; 557 558 dmac->nchan = nchan; 559 560 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1); 561 dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg); 562 563 dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq); 564 565 dmac->dma.dev = dev; 566 for (int i = min(dw, 16); i > 0; i /= 2) { 567 dmac->dma.src_addr_widths |= BIT(i); 568 dmac->dma.dst_addr_widths |= BIT(i); 569 } 570 dmac->dma.directions = BIT(DMA_MEM_TO_MEM); 571 dmac->dma.descriptor_reuse = true; 572 dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 573 dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources; 574 dmac->dma.device_free_chan_resources = d350_free_chan_resources; 575 dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask); 576 dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy; 577 dmac->dma.device_pause = d350_pause; 578 dmac->dma.device_resume = d350_resume; 579 dmac->dma.device_terminate_all = d350_terminate_all; 580 dmac->dma.device_synchronize = d350_synchronize; 581 dmac->dma.device_tx_status = d350_tx_status; 582 dmac->dma.device_issue_pending = d350_issue_pending; 583 INIT_LIST_HEAD(&dmac->dma.channels); 584 585 /* Would be nice to have per-channel caps for this... */ 586 memset = true; 587 for (int i = 0; i < nchan; i++) { 588 struct d350_chan *dch = &dmac->channels[i]; 589 590 dch->base = base + DMACH(i); 591 writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD); 592 593 reg = readl_relaxed(dch->base + CH_BUILDCFG1); 594 if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) { 595 dev_warn(dev, "No command link support on channel %d\n", i); 596 continue; 597 } 598 dch->irq = platform_get_irq(pdev, i); 599 if (dch->irq < 0) 600 return dev_err_probe(dev, dch->irq, 601 "Failed to get IRQ for channel %d\n", i); 602 603 dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg); 604 dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) & 605 FIELD_GET(CH_CFG_HAS_TRIGSEL, reg); 606 607 /* Fill is a special case of Wrap */ 608 memset &= dch->has_wrap; 609 610 reg = readl_relaxed(dch->base + CH_BUILDCFG0); 611 dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg); 612 613 reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH); 614 reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC); 615 writel_relaxed(reg, dch->base + CH_LINKATTR); 616 617 dch->vc.desc_free = d350_desc_free; 618 vchan_init(&dch->vc, &dmac->dma); 619 } 620 621 if (memset) { 622 dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask); 623 dmac->dma.device_prep_dma_memset = d350_prep_memset; 624 } 625 626 platform_set_drvdata(pdev, dmac); 627 628 ret = dma_async_device_register(&dmac->dma); 629 if (ret) 630 return dev_err_probe(dev, ret, "Failed to register DMA device\n"); 631 632 return 0; 633 } 634 635 static void d350_remove(struct platform_device *pdev) 636 { 637 struct d350 *dmac = platform_get_drvdata(pdev); 638 639 dma_async_device_unregister(&dmac->dma); 640 } 641 642 static const struct of_device_id d350_of_match[] __maybe_unused = { 643 { .compatible = "arm,dma-350" }, 644 {} 645 }; 646 MODULE_DEVICE_TABLE(of, d350_of_match); 647 648 static struct platform_driver d350_driver = { 649 .driver = { 650 .name = "arm-dma350", 651 .of_match_table = of_match_ptr(d350_of_match), 652 }, 653 .probe = d350_probe, 654 .remove = d350_remove, 655 }; 656 module_platform_driver(d350_driver); 657 658 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); 659 MODULE_DESCRIPTION("Arm DMA-350 driver"); 660 MODULE_LICENSE("GPL v2"); 661