1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas RZ/G2L DMA Controller Driver 4 * 5 * Based on imx-dma.c 6 * 7 * Copyright (C) 2021 Renesas Electronics Corp. 8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> 9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/cleanup.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/interrupt.h> 17 #include <linux/iopoll.h> 18 #include <linux/irqchip/irq-renesas-rzv2h.h> 19 #include <linux/list.h> 20 #include <linux/module.h> 21 #include <linux/of.h> 22 #include <linux/of_dma.h> 23 #include <linux/of_platform.h> 24 #include <linux/platform_device.h> 25 #include <linux/pm_runtime.h> 26 #include <linux/reset.h> 27 #include <linux/slab.h> 28 #include <linux/spinlock.h> 29 30 #include "../dmaengine.h" 31 #include "../virt-dma.h" 32 33 enum rz_dmac_prep_type { 34 RZ_DMAC_DESC_MEMCPY, 35 RZ_DMAC_DESC_SLAVE_SG, 36 }; 37 38 struct rz_lmdesc { 39 u32 header; 40 u32 sa; 41 u32 da; 42 u32 tb; 43 u32 chcfg; 44 u32 chitvl; 45 u32 chext; 46 u32 nxla; 47 }; 48 49 struct rz_dmac_desc { 50 struct virt_dma_desc vd; 51 dma_addr_t src; 52 dma_addr_t dest; 53 size_t len; 54 struct list_head node; 55 enum dma_transfer_direction direction; 56 enum rz_dmac_prep_type type; 57 /* For slave sg */ 58 struct scatterlist *sg; 59 unsigned int sgcount; 60 }; 61 62 #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) 63 64 struct rz_dmac_chan { 65 struct virt_dma_chan vc; 66 void __iomem *ch_base; 67 void __iomem *ch_cmn_base; 68 unsigned int index; 69 struct rz_dmac_desc *desc; 70 int descs_allocated; 71 72 dma_addr_t src_per_address; 73 dma_addr_t dst_per_address; 74 75 u32 chcfg; 76 u32 chctrl; 77 int mid_rid; 78 79 struct list_head ld_free; 80 struct list_head ld_queue; 81 struct list_head ld_active; 82 83 struct { 84 struct rz_lmdesc *base; 85 struct rz_lmdesc *head; 86 struct rz_lmdesc *tail; 87 dma_addr_t base_dma; 88 } lmdesc; 89 }; 90 91 #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) 92 93 struct rz_dmac_icu { 94 struct platform_device *pdev; 95 u8 dmac_index; 96 }; 97 98 struct rz_dmac { 99 struct dma_device engine; 100 struct rz_dmac_icu icu; 101 struct device *dev; 102 struct reset_control *rstc; 103 void __iomem *base; 104 void __iomem *ext_base; 105 106 unsigned int n_channels; 107 struct rz_dmac_chan *channels; 108 109 bool has_icu; 110 111 DECLARE_BITMAP(modules, 1024); 112 }; 113 114 #define to_rz_dmac(d) container_of(d, struct rz_dmac, engine) 115 116 /* 117 * ----------------------------------------------------------------------------- 118 * Registers 119 */ 120 121 #define CHSTAT 0x0024 122 #define CHCTRL 0x0028 123 #define CHCFG 0x002c 124 #define NXLA 0x0038 125 126 #define DCTRL 0x0000 127 128 #define EACH_CHANNEL_OFFSET 0x0040 129 #define CHANNEL_0_7_OFFSET 0x0000 130 #define CHANNEL_0_7_COMMON_BASE 0x0300 131 #define CHANNEL_8_15_OFFSET 0x0400 132 #define CHANNEL_8_15_COMMON_BASE 0x0700 133 134 #define CHSTAT_ER BIT(4) 135 #define CHSTAT_EN BIT(0) 136 137 #define CHCTRL_CLRINTMSK BIT(17) 138 #define CHCTRL_CLRSUS BIT(9) 139 #define CHCTRL_CLRTC BIT(6) 140 #define CHCTRL_CLREND BIT(5) 141 #define CHCTRL_CLRRQ BIT(4) 142 #define CHCTRL_SWRST BIT(3) 143 #define CHCTRL_STG BIT(2) 144 #define CHCTRL_CLREN BIT(1) 145 #define CHCTRL_SETEN BIT(0) 146 #define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \ 147 CHCTRL_CLRTC | CHCTRL_CLREND | \ 148 CHCTRL_CLRRQ | CHCTRL_SWRST | \ 149 CHCTRL_CLREN) 150 151 #define CHCFG_DMS BIT(31) 152 #define CHCFG_DEM BIT(24) 153 #define CHCFG_DAD BIT(21) 154 #define CHCFG_SAD BIT(20) 155 #define CHCFG_REQD BIT(3) 156 #define CHCFG_SEL(bits) ((bits) & 0x07) 157 #define CHCFG_MEM_COPY (0x80400008) 158 #define CHCFG_FILL_DDS_MASK GENMASK(19, 16) 159 #define CHCFG_FILL_SDS_MASK GENMASK(15, 12) 160 #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) 161 #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) 162 #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) 163 #define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5) 164 165 #define MID_RID_MASK GENMASK(9, 0) 166 #define CHCFG_MASK GENMASK(15, 10) 167 #define CHCFG_DS_INVALID 0xFF 168 #define DCTRL_LVINT BIT(1) 169 #define DCTRL_PR BIT(0) 170 #define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR) 171 172 /* LINK MODE DESCRIPTOR */ 173 #define HEADER_LV BIT(0) 174 175 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 176 #define RZ_DMAC_MAX_CHANNELS 16 177 #define DMAC_NR_LMDESC 64 178 179 /* RZ/V2H ICU related */ 180 #define RZV2H_MAX_DMAC_INDEX 4 181 182 /* 183 * ----------------------------------------------------------------------------- 184 * Device access 185 */ 186 187 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val, 188 unsigned int offset) 189 { 190 writel(val, dmac->base + offset); 191 } 192 193 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val, 194 unsigned int offset) 195 { 196 writel(val, dmac->ext_base + offset); 197 } 198 199 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset) 200 { 201 return readl(dmac->ext_base + offset); 202 } 203 204 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, 205 unsigned int offset, int which) 206 { 207 if (which) 208 writel(val, channel->ch_base + offset); 209 else 210 writel(val, channel->ch_cmn_base + offset); 211 } 212 213 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, 214 unsigned int offset, int which) 215 { 216 if (which) 217 return readl(channel->ch_base + offset); 218 else 219 return readl(channel->ch_cmn_base + offset); 220 } 221 222 /* 223 * ----------------------------------------------------------------------------- 224 * Initialization 225 */ 226 227 static void rz_lmdesc_setup(struct rz_dmac_chan *channel, 228 struct rz_lmdesc *lmdesc) 229 { 230 u32 nxla; 231 232 channel->lmdesc.base = lmdesc; 233 channel->lmdesc.head = lmdesc; 234 channel->lmdesc.tail = lmdesc; 235 nxla = channel->lmdesc.base_dma; 236 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { 237 lmdesc->header = 0; 238 nxla += sizeof(*lmdesc); 239 lmdesc->nxla = nxla; 240 lmdesc++; 241 } 242 243 lmdesc->header = 0; 244 lmdesc->nxla = channel->lmdesc.base_dma; 245 } 246 247 /* 248 * ----------------------------------------------------------------------------- 249 * Descriptors preparation 250 */ 251 252 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) 253 { 254 struct rz_lmdesc *lmdesc = channel->lmdesc.head; 255 256 while (!(lmdesc->header & HEADER_LV)) { 257 lmdesc->header = 0; 258 lmdesc++; 259 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 260 lmdesc = channel->lmdesc.base; 261 } 262 channel->lmdesc.head = lmdesc; 263 } 264 265 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) 266 { 267 struct dma_chan *chan = &channel->vc.chan; 268 struct rz_dmac *dmac = to_rz_dmac(chan->device); 269 unsigned long flags; 270 u32 nxla; 271 u32 chctrl; 272 u32 chstat; 273 274 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 275 276 local_irq_save(flags); 277 278 rz_dmac_lmdesc_recycle(channel); 279 280 nxla = channel->lmdesc.base_dma + 281 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head - 282 channel->lmdesc.base)); 283 284 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 285 if (!(chstat & CHSTAT_EN)) { 286 chctrl = (channel->chctrl | CHCTRL_SETEN); 287 rz_dmac_ch_writel(channel, nxla, NXLA, 1); 288 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); 289 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); 290 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); 291 } 292 293 local_irq_restore(flags); 294 } 295 296 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) 297 { 298 struct dma_chan *chan = &channel->vc.chan; 299 struct rz_dmac *dmac = to_rz_dmac(chan->device); 300 301 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 302 303 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 304 } 305 306 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) 307 { 308 u32 dmars_offset = (nr / 2) * 4; 309 u32 shift = (nr % 2) * 16; 310 u32 dmars32; 311 312 dmars32 = rz_dmac_ext_readl(dmac, dmars_offset); 313 dmars32 &= ~(0xffff << shift); 314 dmars32 |= dmars << shift; 315 316 rz_dmac_ext_writel(dmac, dmars32, dmars_offset); 317 } 318 319 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) 320 { 321 struct dma_chan *chan = &channel->vc.chan; 322 struct rz_dmac *dmac = to_rz_dmac(chan->device); 323 struct rz_lmdesc *lmdesc = channel->lmdesc.tail; 324 struct rz_dmac_desc *d = channel->desc; 325 u32 chcfg = CHCFG_MEM_COPY; 326 327 /* prepare descriptor */ 328 lmdesc->sa = d->src; 329 lmdesc->da = d->dest; 330 lmdesc->tb = d->len; 331 lmdesc->chcfg = chcfg; 332 lmdesc->chitvl = 0; 333 lmdesc->chext = 0; 334 lmdesc->header = HEADER_LV; 335 336 if (dmac->has_icu) { 337 rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, 338 channel->index, 339 RZV2H_ICU_DMAC_REQ_NO_DEFAULT); 340 } else { 341 rz_dmac_set_dmars_register(dmac, channel->index, 0); 342 } 343 344 channel->chcfg = chcfg; 345 channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; 346 } 347 348 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) 349 { 350 struct dma_chan *chan = &channel->vc.chan; 351 struct rz_dmac *dmac = to_rz_dmac(chan->device); 352 struct rz_dmac_desc *d = channel->desc; 353 struct scatterlist *sg, *sgl = d->sg; 354 struct rz_lmdesc *lmdesc; 355 unsigned int i, sg_len = d->sgcount; 356 357 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; 358 359 if (d->direction == DMA_DEV_TO_MEM) { 360 channel->chcfg |= CHCFG_SAD; 361 channel->chcfg &= ~CHCFG_REQD; 362 } else { 363 channel->chcfg |= CHCFG_DAD | CHCFG_REQD; 364 } 365 366 lmdesc = channel->lmdesc.tail; 367 368 for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { 369 if (d->direction == DMA_DEV_TO_MEM) { 370 lmdesc->sa = channel->src_per_address; 371 lmdesc->da = sg_dma_address(sg); 372 } else { 373 lmdesc->sa = sg_dma_address(sg); 374 lmdesc->da = channel->dst_per_address; 375 } 376 377 lmdesc->tb = sg_dma_len(sg); 378 lmdesc->chitvl = 0; 379 lmdesc->chext = 0; 380 if (i == (sg_len - 1)) { 381 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); 382 lmdesc->header = HEADER_LV; 383 } else { 384 lmdesc->chcfg = channel->chcfg; 385 lmdesc->header = HEADER_LV; 386 } 387 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 388 lmdesc = channel->lmdesc.base; 389 } 390 391 channel->lmdesc.tail = lmdesc; 392 393 if (dmac->has_icu) { 394 rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, 395 channel->index, channel->mid_rid); 396 } else { 397 rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); 398 } 399 400 channel->chctrl = CHCTRL_SETEN; 401 } 402 403 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan) 404 { 405 struct rz_dmac_desc *d = chan->desc; 406 struct virt_dma_desc *vd; 407 408 vd = vchan_next_desc(&chan->vc); 409 if (!vd) 410 return 0; 411 412 list_del(&vd->node); 413 414 switch (d->type) { 415 case RZ_DMAC_DESC_MEMCPY: 416 rz_dmac_prepare_desc_for_memcpy(chan); 417 break; 418 419 case RZ_DMAC_DESC_SLAVE_SG: 420 rz_dmac_prepare_descs_for_slave_sg(chan); 421 break; 422 423 default: 424 return -EINVAL; 425 } 426 427 rz_dmac_enable_hw(chan); 428 429 return 0; 430 } 431 432 /* 433 * ----------------------------------------------------------------------------- 434 * DMA engine operations 435 */ 436 437 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) 438 { 439 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 440 441 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { 442 struct rz_dmac_desc *desc; 443 444 desc = kzalloc_obj(*desc); 445 if (!desc) 446 break; 447 448 /* No need to lock. This is called only for the 1st client. */ 449 list_add_tail(&desc->node, &channel->ld_free); 450 channel->descs_allocated++; 451 } 452 453 if (!channel->descs_allocated) 454 return -ENOMEM; 455 456 return channel->descs_allocated; 457 } 458 459 static void rz_dmac_free_chan_resources(struct dma_chan *chan) 460 { 461 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 462 struct rz_dmac *dmac = to_rz_dmac(chan->device); 463 struct rz_lmdesc *lmdesc = channel->lmdesc.base; 464 struct rz_dmac_desc *desc, *_desc; 465 unsigned long flags; 466 unsigned int i; 467 468 spin_lock_irqsave(&channel->vc.lock, flags); 469 470 for (i = 0; i < DMAC_NR_LMDESC; i++) 471 lmdesc[i].header = 0; 472 473 rz_dmac_disable_hw(channel); 474 list_splice_tail_init(&channel->ld_active, &channel->ld_free); 475 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); 476 477 if (channel->mid_rid >= 0) { 478 clear_bit(channel->mid_rid, dmac->modules); 479 channel->mid_rid = -EINVAL; 480 } 481 482 spin_unlock_irqrestore(&channel->vc.lock, flags); 483 484 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { 485 kfree(desc); 486 channel->descs_allocated--; 487 } 488 489 INIT_LIST_HEAD(&channel->ld_free); 490 vchan_free_chan_resources(&channel->vc); 491 } 492 493 static struct dma_async_tx_descriptor * 494 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 495 size_t len, unsigned long flags) 496 { 497 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 498 struct rz_dmac *dmac = to_rz_dmac(chan->device); 499 struct rz_dmac_desc *desc; 500 501 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", 502 __func__, channel->index, &src, &dest, len); 503 504 scoped_guard(spinlock_irqsave, &channel->vc.lock) { 505 if (list_empty(&channel->ld_free)) 506 return NULL; 507 508 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); 509 510 desc->type = RZ_DMAC_DESC_MEMCPY; 511 desc->src = src; 512 desc->dest = dest; 513 desc->len = len; 514 desc->direction = DMA_MEM_TO_MEM; 515 516 list_move_tail(channel->ld_free.next, &channel->ld_queue); 517 } 518 519 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 520 } 521 522 static struct dma_async_tx_descriptor * 523 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 524 unsigned int sg_len, 525 enum dma_transfer_direction direction, 526 unsigned long flags, void *context) 527 { 528 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 529 struct rz_dmac_desc *desc; 530 struct scatterlist *sg; 531 int dma_length = 0; 532 int i = 0; 533 534 scoped_guard(spinlock_irqsave, &channel->vc.lock) { 535 if (list_empty(&channel->ld_free)) 536 return NULL; 537 538 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); 539 540 for_each_sg(sgl, sg, sg_len, i) 541 dma_length += sg_dma_len(sg); 542 543 desc->type = RZ_DMAC_DESC_SLAVE_SG; 544 desc->sg = sgl; 545 desc->sgcount = sg_len; 546 desc->len = dma_length; 547 desc->direction = direction; 548 549 if (direction == DMA_DEV_TO_MEM) 550 desc->src = channel->src_per_address; 551 else 552 desc->dest = channel->dst_per_address; 553 554 list_move_tail(channel->ld_free.next, &channel->ld_queue); 555 } 556 557 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 558 } 559 560 static int rz_dmac_terminate_all(struct dma_chan *chan) 561 { 562 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 563 struct rz_lmdesc *lmdesc = channel->lmdesc.base; 564 unsigned long flags; 565 unsigned int i; 566 LIST_HEAD(head); 567 568 spin_lock_irqsave(&channel->vc.lock, flags); 569 rz_dmac_disable_hw(channel); 570 for (i = 0; i < DMAC_NR_LMDESC; i++) 571 lmdesc[i].header = 0; 572 573 list_splice_tail_init(&channel->ld_active, &channel->ld_free); 574 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); 575 vchan_get_all_descriptors(&channel->vc, &head); 576 spin_unlock_irqrestore(&channel->vc.lock, flags); 577 vchan_dma_desc_free_list(&channel->vc, &head); 578 579 return 0; 580 } 581 582 static void rz_dmac_issue_pending(struct dma_chan *chan) 583 { 584 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 585 struct rz_dmac *dmac = to_rz_dmac(chan->device); 586 struct rz_dmac_desc *desc; 587 unsigned long flags; 588 589 spin_lock_irqsave(&channel->vc.lock, flags); 590 591 if (!list_empty(&channel->ld_queue)) { 592 desc = list_first_entry(&channel->ld_queue, 593 struct rz_dmac_desc, node); 594 channel->desc = desc; 595 if (vchan_issue_pending(&channel->vc)) { 596 if (rz_dmac_xfer_desc(channel) < 0) 597 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n", 598 channel->index); 599 else 600 list_move_tail(channel->ld_queue.next, 601 &channel->ld_active); 602 } 603 } 604 605 spin_unlock_irqrestore(&channel->vc.lock, flags); 606 } 607 608 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds) 609 { 610 u8 i; 611 static const enum dma_slave_buswidth ds_lut[] = { 612 DMA_SLAVE_BUSWIDTH_1_BYTE, 613 DMA_SLAVE_BUSWIDTH_2_BYTES, 614 DMA_SLAVE_BUSWIDTH_4_BYTES, 615 DMA_SLAVE_BUSWIDTH_8_BYTES, 616 DMA_SLAVE_BUSWIDTH_16_BYTES, 617 DMA_SLAVE_BUSWIDTH_32_BYTES, 618 DMA_SLAVE_BUSWIDTH_64_BYTES, 619 DMA_SLAVE_BUSWIDTH_128_BYTES, 620 }; 621 622 for (i = 0; i < ARRAY_SIZE(ds_lut); i++) { 623 if (ds_lut[i] == ds) 624 return i; 625 } 626 627 return CHCFG_DS_INVALID; 628 } 629 630 static int rz_dmac_config(struct dma_chan *chan, 631 struct dma_slave_config *config) 632 { 633 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 634 u32 val; 635 636 channel->dst_per_address = config->dst_addr; 637 channel->chcfg &= ~CHCFG_FILL_DDS_MASK; 638 if (channel->dst_per_address) { 639 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); 640 if (val == CHCFG_DS_INVALID) 641 return -EINVAL; 642 643 channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val); 644 } 645 646 channel->src_per_address = config->src_addr; 647 channel->chcfg &= ~CHCFG_FILL_SDS_MASK; 648 if (channel->src_per_address) { 649 val = rz_dmac_ds_to_val_mapping(config->src_addr_width); 650 if (val == CHCFG_DS_INVALID) 651 return -EINVAL; 652 653 channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val); 654 } 655 656 return 0; 657 } 658 659 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) 660 { 661 /* 662 * Place holder 663 * Descriptor allocation is done during alloc_chan_resources and 664 * get freed during free_chan_resources. 665 * list is used to manage the descriptors and avoid any memory 666 * allocation/free during DMA read/write. 667 */ 668 } 669 670 static void rz_dmac_device_synchronize(struct dma_chan *chan) 671 { 672 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 673 struct rz_dmac *dmac = to_rz_dmac(chan->device); 674 u32 chstat; 675 int ret; 676 677 ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), 678 100, 100000, false, channel, CHSTAT, 1); 679 if (ret < 0) 680 dev_warn(dmac->dev, "DMA Timeout"); 681 682 if (dmac->has_icu) { 683 rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, 684 channel->index, 685 RZV2H_ICU_DMAC_REQ_NO_DEFAULT); 686 } else { 687 rz_dmac_set_dmars_register(dmac, channel->index, 0); 688 } 689 } 690 691 /* 692 * ----------------------------------------------------------------------------- 693 * IRQ handling 694 */ 695 696 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) 697 { 698 struct dma_chan *chan = &channel->vc.chan; 699 struct rz_dmac *dmac = to_rz_dmac(chan->device); 700 u32 chstat, chctrl; 701 702 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 703 if (chstat & CHSTAT_ER) { 704 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", 705 channel->index, chstat); 706 707 scoped_guard(spinlock_irqsave, &channel->vc.lock) 708 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 709 goto done; 710 } 711 712 chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); 713 rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); 714 done: 715 return; 716 } 717 718 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) 719 { 720 struct rz_dmac_chan *channel = dev_id; 721 722 if (channel) { 723 rz_dmac_irq_handle_channel(channel); 724 return IRQ_WAKE_THREAD; 725 } 726 /* handle DMAERR irq */ 727 return IRQ_HANDLED; 728 } 729 730 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) 731 { 732 struct rz_dmac_chan *channel = dev_id; 733 struct rz_dmac_desc *desc = NULL; 734 unsigned long flags; 735 736 spin_lock_irqsave(&channel->vc.lock, flags); 737 738 if (list_empty(&channel->ld_active)) { 739 /* Someone might have called terminate all */ 740 goto out; 741 } 742 743 desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node); 744 vchan_cookie_complete(&desc->vd); 745 list_move_tail(channel->ld_active.next, &channel->ld_free); 746 if (!list_empty(&channel->ld_queue)) { 747 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc, 748 node); 749 channel->desc = desc; 750 if (rz_dmac_xfer_desc(channel) == 0) 751 list_move_tail(channel->ld_queue.next, &channel->ld_active); 752 } 753 out: 754 spin_unlock_irqrestore(&channel->vc.lock, flags); 755 756 return IRQ_HANDLED; 757 } 758 759 /* 760 * ----------------------------------------------------------------------------- 761 * OF xlate and channel filter 762 */ 763 764 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) 765 { 766 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 767 struct rz_dmac *dmac = to_rz_dmac(chan->device); 768 struct of_phandle_args *dma_spec = arg; 769 u32 ch_cfg; 770 771 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; 772 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; 773 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | 774 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); 775 776 return !test_and_set_bit(channel->mid_rid, dmac->modules); 777 } 778 779 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, 780 struct of_dma *ofdma) 781 { 782 dma_cap_mask_t mask; 783 784 if (dma_spec->args_count != 1) 785 return NULL; 786 787 /* Only slave DMA channels can be allocated via DT */ 788 dma_cap_zero(mask); 789 dma_cap_set(DMA_SLAVE, mask); 790 791 return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec, 792 ofdma->of_node); 793 } 794 795 /* 796 * ----------------------------------------------------------------------------- 797 * Probe and remove 798 */ 799 800 static int rz_dmac_chan_probe(struct rz_dmac *dmac, 801 struct rz_dmac_chan *channel, 802 u8 index) 803 { 804 struct platform_device *pdev = to_platform_device(dmac->dev); 805 struct rz_lmdesc *lmdesc; 806 char pdev_irqname[6]; 807 char *irqname; 808 int irq, ret; 809 810 channel->index = index; 811 channel->mid_rid = -EINVAL; 812 813 /* Request the channel interrupt. */ 814 scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); 815 irq = platform_get_irq_byname(pdev, pdev_irqname); 816 if (irq < 0) 817 return irq; 818 819 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 820 dev_name(dmac->dev), index); 821 if (!irqname) 822 return -ENOMEM; 823 824 ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, 825 rz_dmac_irq_handler_thread, 0, 826 irqname, channel); 827 if (ret) { 828 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); 829 return ret; 830 } 831 832 /* Set io base address for each channel */ 833 if (index < 8) { 834 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + 835 EACH_CHANNEL_OFFSET * index; 836 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; 837 } else { 838 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + 839 EACH_CHANNEL_OFFSET * (index - 8); 840 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; 841 } 842 843 /* Allocate descriptors */ 844 lmdesc = dma_alloc_coherent(&pdev->dev, 845 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 846 &channel->lmdesc.base_dma, GFP_KERNEL); 847 if (!lmdesc) { 848 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); 849 return -ENOMEM; 850 } 851 rz_lmdesc_setup(channel, lmdesc); 852 853 /* Initialize register for each channel */ 854 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 855 856 channel->vc.desc_free = rz_dmac_virt_desc_free; 857 vchan_init(&channel->vc, &dmac->engine); 858 INIT_LIST_HEAD(&channel->ld_queue); 859 INIT_LIST_HEAD(&channel->ld_free); 860 INIT_LIST_HEAD(&channel->ld_active); 861 862 return 0; 863 } 864 865 static void rz_dmac_put_device(void *_dev) 866 { 867 struct device *dev = _dev; 868 869 put_device(dev); 870 } 871 872 static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) 873 { 874 struct device_node *np = dev->of_node; 875 struct of_phandle_args args; 876 uint32_t dmac_index; 877 int ret; 878 879 ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); 880 if (ret == -ENOENT) 881 return 0; 882 if (ret) 883 return ret; 884 885 dmac->has_icu = true; 886 887 dmac->icu.pdev = of_find_device_by_node(args.np); 888 of_node_put(args.np); 889 if (!dmac->icu.pdev) { 890 dev_err(dev, "ICU device not found.\n"); 891 return -ENODEV; 892 } 893 894 ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev); 895 if (ret) 896 return ret; 897 898 dmac_index = args.args[0]; 899 if (dmac_index > RZV2H_MAX_DMAC_INDEX) { 900 dev_err(dev, "DMAC index %u invalid.\n", dmac_index); 901 return -EINVAL; 902 } 903 dmac->icu.dmac_index = dmac_index; 904 905 return 0; 906 } 907 908 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) 909 { 910 struct device_node *np = dev->of_node; 911 int ret; 912 913 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 914 if (ret < 0) { 915 dev_err(dev, "unable to read dma-channels property\n"); 916 return ret; 917 } 918 919 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { 920 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); 921 return -EINVAL; 922 } 923 924 return rz_dmac_parse_of_icu(dev, dmac); 925 } 926 927 static int rz_dmac_probe(struct platform_device *pdev) 928 { 929 const char *irqname = "error"; 930 struct dma_device *engine; 931 struct rz_dmac *dmac; 932 int channel_num; 933 int ret; 934 int irq; 935 u8 i; 936 937 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 938 if (!dmac) 939 return -ENOMEM; 940 941 dmac->dev = &pdev->dev; 942 platform_set_drvdata(pdev, dmac); 943 944 ret = rz_dmac_parse_of(&pdev->dev, dmac); 945 if (ret < 0) 946 return ret; 947 948 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 949 sizeof(*dmac->channels), GFP_KERNEL); 950 if (!dmac->channels) 951 return -ENOMEM; 952 953 /* Request resources */ 954 dmac->base = devm_platform_ioremap_resource(pdev, 0); 955 if (IS_ERR(dmac->base)) 956 return PTR_ERR(dmac->base); 957 958 if (!dmac->has_icu) { 959 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); 960 if (IS_ERR(dmac->ext_base)) 961 return PTR_ERR(dmac->ext_base); 962 } 963 964 /* Register interrupt handler for error */ 965 irq = platform_get_irq_byname(pdev, irqname); 966 if (irq < 0) 967 return irq; 968 969 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, 970 irqname, NULL); 971 if (ret) { 972 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", 973 irq, ret); 974 return ret; 975 } 976 977 /* Initialize the channels. */ 978 INIT_LIST_HEAD(&dmac->engine.channels); 979 980 dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev); 981 if (IS_ERR(dmac->rstc)) 982 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), 983 "failed to get resets\n"); 984 985 pm_runtime_enable(&pdev->dev); 986 ret = pm_runtime_resume_and_get(&pdev->dev); 987 if (ret < 0) { 988 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); 989 goto err_pm_disable; 990 } 991 992 ret = reset_control_deassert(dmac->rstc); 993 if (ret) 994 goto err_pm_runtime_put; 995 996 for (i = 0; i < dmac->n_channels; i++) { 997 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); 998 if (ret < 0) 999 goto err; 1000 } 1001 1002 /* Register the DMAC as a DMA provider for DT. */ 1003 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, 1004 NULL); 1005 if (ret < 0) 1006 goto err; 1007 1008 /* Register the DMA engine device. */ 1009 engine = &dmac->engine; 1010 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1011 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1012 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); 1013 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); 1014 1015 engine->dev = &pdev->dev; 1016 1017 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; 1018 engine->device_free_chan_resources = rz_dmac_free_chan_resources; 1019 engine->device_tx_status = dma_cookie_status; 1020 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; 1021 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; 1022 engine->device_config = rz_dmac_config; 1023 engine->device_terminate_all = rz_dmac_terminate_all; 1024 engine->device_issue_pending = rz_dmac_issue_pending; 1025 engine->device_synchronize = rz_dmac_device_synchronize; 1026 1027 engine->copy_align = DMAENGINE_ALIGN_1_BYTE; 1028 dma_set_max_seg_size(engine->dev, U32_MAX); 1029 1030 ret = dma_async_device_register(engine); 1031 if (ret < 0) { 1032 dev_err(&pdev->dev, "unable to register\n"); 1033 goto dma_register_err; 1034 } 1035 return 0; 1036 1037 dma_register_err: 1038 of_dma_controller_free(pdev->dev.of_node); 1039 err: 1040 channel_num = i ? i - 1 : 0; 1041 for (i = 0; i < channel_num; i++) { 1042 struct rz_dmac_chan *channel = &dmac->channels[i]; 1043 1044 dma_free_coherent(&pdev->dev, 1045 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 1046 channel->lmdesc.base, 1047 channel->lmdesc.base_dma); 1048 } 1049 1050 reset_control_assert(dmac->rstc); 1051 err_pm_runtime_put: 1052 pm_runtime_put(&pdev->dev); 1053 err_pm_disable: 1054 pm_runtime_disable(&pdev->dev); 1055 1056 return ret; 1057 } 1058 1059 static void rz_dmac_remove(struct platform_device *pdev) 1060 { 1061 struct rz_dmac *dmac = platform_get_drvdata(pdev); 1062 unsigned int i; 1063 1064 dma_async_device_unregister(&dmac->engine); 1065 of_dma_controller_free(pdev->dev.of_node); 1066 for (i = 0; i < dmac->n_channels; i++) { 1067 struct rz_dmac_chan *channel = &dmac->channels[i]; 1068 1069 dma_free_coherent(&pdev->dev, 1070 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 1071 channel->lmdesc.base, 1072 channel->lmdesc.base_dma); 1073 } 1074 reset_control_assert(dmac->rstc); 1075 pm_runtime_put(&pdev->dev); 1076 pm_runtime_disable(&pdev->dev); 1077 } 1078 1079 static const struct of_device_id of_rz_dmac_match[] = { 1080 { .compatible = "renesas,r9a09g057-dmac", }, 1081 { .compatible = "renesas,rz-dmac", }, 1082 { /* Sentinel */ } 1083 }; 1084 MODULE_DEVICE_TABLE(of, of_rz_dmac_match); 1085 1086 static struct platform_driver rz_dmac_driver = { 1087 .driver = { 1088 .name = "rz-dmac", 1089 .of_match_table = of_rz_dmac_match, 1090 }, 1091 .probe = rz_dmac_probe, 1092 .remove = rz_dmac_remove, 1093 }; 1094 1095 module_platform_driver(rz_dmac_driver); 1096 1097 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver"); 1098 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>"); 1099 MODULE_LICENSE("GPL v2"); 1100