1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas RZ/G2L DMA Controller Driver 4 * 5 * Based on imx-dma.c 6 * 7 * Copyright (C) 2021 Renesas Electronics Corp. 8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> 9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/cleanup.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/interrupt.h> 17 #include <linux/iopoll.h> 18 #include <linux/irqchip/irq-renesas-rzv2h.h> 19 #include <linux/irqchip/irq-renesas-rzt2h.h> 20 #include <linux/list.h> 21 #include <linux/lockdep.h> 22 #include <linux/module.h> 23 #include <linux/of.h> 24 #include <linux/of_dma.h> 25 #include <linux/of_platform.h> 26 #include <linux/platform_device.h> 27 #include <linux/pm_runtime.h> 28 #include <linux/reset.h> 29 #include <linux/slab.h> 30 #include <linux/spinlock.h> 31 32 #include "../dmaengine.h" 33 #include "../virt-dma.h" 34 35 enum rz_dmac_prep_type { 36 RZ_DMAC_DESC_MEMCPY, 37 RZ_DMAC_DESC_SLAVE_SG, 38 RZ_DMAC_DESC_CYCLIC, 39 }; 40 41 struct rz_lmdesc { 42 u32 header; 43 u32 sa; 44 u32 da; 45 u32 tb; 46 u32 chcfg; 47 u32 chitvl; 48 u32 chext; 49 u32 nxla; 50 }; 51 52 struct rz_dmac_desc { 53 struct virt_dma_desc vd; 54 dma_addr_t src; 55 dma_addr_t dest; 56 size_t len; 57 struct list_head node; 58 enum dma_transfer_direction direction; 59 enum rz_dmac_prep_type type; 60 /* For slave sg */ 61 struct scatterlist *sg; 62 unsigned int sgcount; 63 struct rz_lmdesc *start_lmdesc; 64 }; 65 66 #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) 67 68 /** 69 * enum rz_dmac_chan_status: RZ DMAC channel status 70 * @RZ_DMAC_CHAN_STATUS_PAUSED: Channel is paused though DMA engine callbacks 71 * @RZ_DMAC_CHAN_STATUS_CYCLIC: Channel is cyclic 72 * @RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL: Channel is paused through driver internal logic 73 */ 74 enum rz_dmac_chan_status { 75 RZ_DMAC_CHAN_STATUS_PAUSED, 76 RZ_DMAC_CHAN_STATUS_CYCLIC, 77 RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL, 78 }; 79 80 struct rz_dmac_chan { 81 struct virt_dma_chan vc; 82 void __iomem *ch_base; 83 void __iomem *ch_cmn_base; 84 unsigned int index; 85 struct rz_dmac_desc *desc; 86 int descs_allocated; 87 88 dma_addr_t src_per_address; 89 dma_addr_t dst_per_address; 90 91 unsigned long status; 92 93 u32 chcfg; 94 u32 chctrl; 95 int mid_rid; 96 int dmac_ack; 97 98 struct { 99 u32 nxla; 100 } pm_state; 101 102 struct list_head ld_free; 103 104 struct { 105 struct rz_lmdesc *base; 106 struct rz_lmdesc *head; 107 struct rz_lmdesc *tail; 108 dma_addr_t base_dma; 109 } lmdesc; 110 }; 111 112 #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) 113 114 struct rz_dmac_icu { 115 struct platform_device *pdev; 116 u8 dmac_index; 117 }; 118 119 struct rz_dmac_info { 120 void (*icu_register_dma_req)(struct platform_device *icu_dev, 121 u8 dmac_index, u8 dmac_channel, u16 req_no); 122 void (*icu_register_dma_ack)(struct platform_device *icu_dev, 123 u8 dmac_index, u8 dmac_channel, u16 ack_no); 124 u16 default_dma_ack_no; 125 u16 default_dma_req_no; 126 }; 127 128 struct rz_dmac { 129 struct dma_device engine; 130 struct rz_dmac_icu icu; 131 const struct rz_dmac_info *info; 132 struct device *dev; 133 struct reset_control *rstc; 134 void __iomem *base; 135 void __iomem *ext_base; 136 137 unsigned int n_channels; 138 struct rz_dmac_chan *channels; 139 140 DECLARE_BITMAP(modules, 1024); 141 }; 142 143 #define to_rz_dmac(d) container_of(d, struct rz_dmac, engine) 144 145 /* 146 * ----------------------------------------------------------------------------- 147 * Registers 148 */ 149 150 #define CRTB 0x0020 151 #define CHSTAT 0x0024 152 #define CHCTRL 0x0028 153 #define CHCFG 0x002c 154 #define NXLA 0x0038 155 #define CRLA 0x003c 156 157 #define DCTRL 0x0000 158 159 #define EACH_CHANNEL_OFFSET 0x0040 160 #define CHANNEL_0_7_OFFSET 0x0000 161 #define CHANNEL_0_7_COMMON_BASE 0x0300 162 #define CHANNEL_8_15_OFFSET 0x0400 163 #define CHANNEL_8_15_COMMON_BASE 0x0700 164 165 #define CHSTAT_ER BIT(4) 166 #define CHSTAT_SUS BIT(3) 167 #define CHSTAT_EN BIT(0) 168 169 #define CHCTRL_CLRINTMSK BIT(17) 170 #define CHCTRL_CLRSUS BIT(9) 171 #define CHCTRL_SETSUS BIT(8) 172 #define CHCTRL_CLRTC BIT(6) 173 #define CHCTRL_CLREND BIT(5) 174 #define CHCTRL_CLRRQ BIT(4) 175 #define CHCTRL_SWRST BIT(3) 176 #define CHCTRL_STG BIT(2) 177 #define CHCTRL_CLREN BIT(1) 178 #define CHCTRL_SETEN BIT(0) 179 #define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \ 180 CHCTRL_CLRTC | CHCTRL_CLREND | \ 181 CHCTRL_CLRRQ | CHCTRL_SWRST | \ 182 CHCTRL_CLREN) 183 184 #define CHCFG_DMS BIT(31) 185 #define CHCFG_DEM BIT(24) 186 #define CHCFG_DAD BIT(21) 187 #define CHCFG_SAD BIT(20) 188 #define CHCFG_REQD BIT(3) 189 #define CHCFG_SEL(bits) ((bits) & 0x07) 190 #define CHCFG_MEM_COPY (0x80400008) 191 #define CHCFG_FILL_DDS_MASK GENMASK(19, 16) 192 #define CHCFG_FILL_SDS_MASK GENMASK(15, 12) 193 #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) 194 #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) 195 #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) 196 #define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5) 197 198 #define MID_RID_MASK GENMASK(9, 0) 199 #define CHCFG_MASK GENMASK(15, 10) 200 #define CHCFG_DS_INVALID 0xFF 201 #define DCTRL_LVINT BIT(1) 202 #define DCTRL_PR BIT(0) 203 #define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR) 204 205 /* LINK MODE DESCRIPTOR */ 206 #define HEADER_LV BIT(0) 207 #define HEADER_LE BIT(1) 208 #define HEADER_WBD BIT(2) 209 210 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 211 #define RZ_DMAC_MAX_CHANNELS 16 212 #define DMAC_NR_LMDESC 64 213 214 /* RZ/V2H ICU related */ 215 #define RZV2H_MAX_DMAC_INDEX 4 216 217 /* 218 * ----------------------------------------------------------------------------- 219 * Device access 220 */ 221 222 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val, 223 unsigned int offset) 224 { 225 writel(val, dmac->base + offset); 226 } 227 228 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val, 229 unsigned int offset) 230 { 231 writel(val, dmac->ext_base + offset); 232 } 233 234 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset) 235 { 236 return readl(dmac->ext_base + offset); 237 } 238 239 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, 240 unsigned int offset, int which) 241 { 242 if (which) 243 writel(val, channel->ch_base + offset); 244 else 245 writel(val, channel->ch_cmn_base + offset); 246 } 247 248 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, 249 unsigned int offset, int which) 250 { 251 if (which) 252 return readl(channel->ch_base + offset); 253 else 254 return readl(channel->ch_cmn_base + offset); 255 } 256 257 /* 258 * ----------------------------------------------------------------------------- 259 * Initialization 260 */ 261 262 static void rz_lmdesc_setup(struct rz_dmac_chan *channel, 263 struct rz_lmdesc *lmdesc) 264 { 265 u32 nxla; 266 267 channel->lmdesc.base = lmdesc; 268 channel->lmdesc.head = lmdesc; 269 channel->lmdesc.tail = lmdesc; 270 nxla = channel->lmdesc.base_dma; 271 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { 272 lmdesc->header = 0; 273 nxla += sizeof(*lmdesc); 274 lmdesc->nxla = nxla; 275 lmdesc++; 276 } 277 278 lmdesc->header = 0; 279 lmdesc->nxla = channel->lmdesc.base_dma; 280 } 281 282 /* 283 * ----------------------------------------------------------------------------- 284 * Descriptors preparation 285 */ 286 287 static u32 rz_dmac_lmdesc_addr(struct rz_dmac_chan *channel, struct rz_lmdesc *lmdesc) 288 { 289 return channel->lmdesc.base_dma + 290 (sizeof(struct rz_lmdesc) * (lmdesc - channel->lmdesc.base)); 291 } 292 293 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) 294 { 295 struct rz_lmdesc *lmdesc = channel->lmdesc.head; 296 297 while (!(lmdesc->header & HEADER_LV)) { 298 lmdesc->header = 0; 299 lmdesc++; 300 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 301 lmdesc = channel->lmdesc.base; 302 } 303 channel->lmdesc.head = lmdesc; 304 } 305 306 static bool rz_dmac_chan_is_enabled(struct rz_dmac_chan *channel) 307 { 308 u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1); 309 310 return !!(val & CHSTAT_EN); 311 } 312 313 static bool rz_dmac_chan_is_paused(struct rz_dmac_chan *channel) 314 { 315 u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1); 316 317 return !!(val & CHSTAT_SUS); 318 } 319 320 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) 321 { 322 struct dma_chan *chan = &channel->vc.chan; 323 struct rz_dmac *dmac = to_rz_dmac(chan->device); 324 u32 nxla; 325 u32 chctrl; 326 327 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 328 329 rz_dmac_lmdesc_recycle(channel); 330 331 nxla = rz_dmac_lmdesc_addr(channel, channel->lmdesc.head); 332 333 chctrl = (channel->chctrl | CHCTRL_SETEN); 334 rz_dmac_ch_writel(channel, nxla, NXLA, 1); 335 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); 336 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); 337 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); 338 } 339 340 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) 341 { 342 struct dma_chan *chan = &channel->vc.chan; 343 struct rz_dmac *dmac = to_rz_dmac(chan->device); 344 345 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); 346 347 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); 348 } 349 350 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) 351 { 352 u32 dmars_offset = (nr / 2) * 4; 353 u32 shift = (nr % 2) * 16; 354 u32 dmars32; 355 356 dmars32 = rz_dmac_ext_readl(dmac, dmars_offset); 357 dmars32 &= ~(0xffff << shift); 358 dmars32 |= dmars << shift; 359 360 rz_dmac_ext_writel(dmac, dmars32, dmars_offset); 361 } 362 363 static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index, 364 int req_no) 365 { 366 if (dmac->info->icu_register_dma_req) 367 dmac->info->icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, 368 index, req_no); 369 else 370 rz_dmac_set_dmars_register(dmac, index, req_no); 371 } 372 373 /* 374 * Map MID/RID request number (bits[0:9] of DMA specifier) to the ICU 375 * DMA ACK signal number, per RZ/G3E hardware manual Table 4.6-28. 376 * 377 * Three peripheral groups cover all ACK-capable peripherals: 378 * 379 * PFC external DMA pins (DREQ0..DREQ4): 380 * req_no 0x000-0x004 -> ACK No. 84-88 (ack = req_no + 84) 381 * 382 * SSIU BUSIFs (ssip00..ssip93): 383 * req_no 0x161-0x198 -> ACK No. 28-83 (ack = req_no - 0x145) 384 * 385 * SPDIF (CH0..CH2) + SCU SRC (sr0..sr9) + DVC (cmd0..cmd1): 386 * req_no 0x199-0x1b4 -> ACK No. 0-27 (ack = req_no - 0x199) 387 */ 388 static int rz_dmac_get_ack_no(const struct rz_dmac_info *info, u16 req_no) 389 { 390 if (!info->icu_register_dma_ack) 391 return -EINVAL; 392 393 switch (req_no) { 394 case 0x000 ... 0x004: 395 /* PFC external DMA pins: ACK No. 84-88 */ 396 return req_no + 84; 397 case 0x161 ... 0x198: 398 /* SSIU BUSIFs: ACK No. 28-83 */ 399 return req_no - 0x145; 400 case 0x199 ... 0x1b4: 401 /* SPDIF + SCU SRC + DVC: ACK No. 0-27 */ 402 return req_no - 0x199; 403 default: 404 return -EINVAL; 405 } 406 } 407 408 static void rz_dmac_set_dma_ack_no(struct rz_dmac *dmac, unsigned int index, 409 int ack_no) 410 { 411 if (ack_no < 0 || !dmac->info->icu_register_dma_ack) 412 return; 413 414 dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index, 415 index, ack_no); 416 } 417 418 static void rz_dmac_reset_dma_ack_no(struct rz_dmac *dmac, int ack_no) 419 { 420 if (ack_no < 0 || !dmac->info->icu_register_dma_ack) 421 return; 422 423 dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index, 424 dmac->info->default_dma_ack_no, ack_no); 425 } 426 427 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) 428 { 429 struct dma_chan *chan = &channel->vc.chan; 430 struct rz_dmac *dmac = to_rz_dmac(chan->device); 431 struct rz_lmdesc *lmdesc = channel->lmdesc.tail; 432 struct rz_dmac_desc *d = channel->desc; 433 u32 chcfg = CHCFG_MEM_COPY; 434 435 d->start_lmdesc = lmdesc; 436 437 /* prepare descriptor */ 438 lmdesc->sa = d->src; 439 lmdesc->da = d->dest; 440 lmdesc->tb = d->len; 441 lmdesc->chcfg = chcfg; 442 lmdesc->chitvl = 0; 443 lmdesc->chext = 0; 444 lmdesc->header = HEADER_LV | HEADER_LE; 445 446 rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); 447 448 channel->chcfg = chcfg; 449 channel->chctrl = CHCTRL_STG; 450 } 451 452 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) 453 { 454 struct dma_chan *chan = &channel->vc.chan; 455 struct rz_dmac *dmac = to_rz_dmac(chan->device); 456 struct rz_dmac_desc *d = channel->desc; 457 struct scatterlist *sg, *sgl = d->sg; 458 struct rz_lmdesc *lmdesc; 459 unsigned int i, sg_len = d->sgcount; 460 461 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; 462 463 if (d->direction == DMA_DEV_TO_MEM) { 464 channel->chcfg |= CHCFG_SAD; 465 channel->chcfg &= ~CHCFG_REQD; 466 } else { 467 channel->chcfg |= CHCFG_DAD | CHCFG_REQD; 468 } 469 470 lmdesc = channel->lmdesc.tail; 471 d->start_lmdesc = lmdesc; 472 473 for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { 474 if (d->direction == DMA_DEV_TO_MEM) { 475 lmdesc->sa = channel->src_per_address; 476 lmdesc->da = sg_dma_address(sg); 477 } else { 478 lmdesc->sa = sg_dma_address(sg); 479 lmdesc->da = channel->dst_per_address; 480 } 481 482 lmdesc->tb = sg_dma_len(sg); 483 lmdesc->chitvl = 0; 484 lmdesc->chext = 0; 485 if (i == (sg_len - 1)) { 486 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); 487 lmdesc->header = HEADER_LV | HEADER_LE; 488 } else { 489 lmdesc->chcfg = channel->chcfg; 490 lmdesc->header = HEADER_LV; 491 } 492 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 493 lmdesc = channel->lmdesc.base; 494 } 495 496 channel->lmdesc.tail = lmdesc; 497 498 rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); 499 rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); 500 501 channel->chctrl = 0; 502 } 503 504 static void rz_dmac_prepare_descs_for_cyclic(struct rz_dmac_chan *channel) 505 { 506 struct dma_chan *chan = &channel->vc.chan; 507 struct rz_dmac *dmac = to_rz_dmac(chan->device); 508 struct rz_dmac_desc *d = channel->desc; 509 size_t period_len = d->sgcount; 510 struct rz_lmdesc *lmdesc; 511 size_t buf_len = d->len; 512 size_t periods = buf_len / period_len; 513 514 lockdep_assert_held(&channel->vc.lock); 515 516 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DMS; 517 518 if (d->direction == DMA_DEV_TO_MEM) { 519 channel->chcfg |= CHCFG_SAD; 520 channel->chcfg &= ~CHCFG_REQD; 521 } else { 522 channel->chcfg |= CHCFG_DAD | CHCFG_REQD; 523 } 524 525 lmdesc = channel->lmdesc.tail; 526 d->start_lmdesc = lmdesc; 527 528 for (size_t i = 0; i < periods; i++) { 529 if (d->direction == DMA_DEV_TO_MEM) { 530 lmdesc->sa = d->src; 531 lmdesc->da = d->dest + (i * period_len); 532 } else { 533 lmdesc->sa = d->src + (i * period_len); 534 lmdesc->da = d->dest; 535 } 536 537 lmdesc->tb = period_len; 538 lmdesc->chitvl = 0; 539 lmdesc->chext = 0; 540 lmdesc->chcfg = channel->chcfg; 541 lmdesc->header = HEADER_LV | HEADER_WBD; 542 543 if (i == periods - 1) 544 lmdesc->nxla = rz_dmac_lmdesc_addr(channel, d->start_lmdesc); 545 546 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) 547 lmdesc = channel->lmdesc.base; 548 } 549 550 channel->lmdesc.tail = lmdesc; 551 552 rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); 553 rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); 554 } 555 556 static void rz_dmac_xfer_desc(struct rz_dmac_chan *chan) 557 { 558 struct virt_dma_desc *vd; 559 560 vd = vchan_next_desc(&chan->vc); 561 if (!vd) { 562 chan->desc = NULL; 563 return; 564 } 565 566 list_del(&vd->node); 567 chan->desc = to_rz_dmac_desc(vd); 568 569 switch (chan->desc->type) { 570 case RZ_DMAC_DESC_MEMCPY: 571 rz_dmac_prepare_desc_for_memcpy(chan); 572 break; 573 574 case RZ_DMAC_DESC_SLAVE_SG: 575 rz_dmac_prepare_descs_for_slave_sg(chan); 576 break; 577 578 case RZ_DMAC_DESC_CYCLIC: 579 rz_dmac_prepare_descs_for_cyclic(chan); 580 break; 581 } 582 583 rz_dmac_enable_hw(chan); 584 } 585 586 /* 587 * ----------------------------------------------------------------------------- 588 * DMA engine operations 589 */ 590 591 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) 592 { 593 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 594 595 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { 596 struct rz_dmac_desc *desc; 597 598 desc = kzalloc_obj(*desc); 599 if (!desc) 600 break; 601 602 /* No need to lock. This is called only for the 1st client. */ 603 list_add_tail(&desc->node, &channel->ld_free); 604 channel->descs_allocated++; 605 } 606 607 if (!channel->descs_allocated) 608 return -ENOMEM; 609 610 return channel->descs_allocated; 611 } 612 613 static void rz_dmac_free_chan_resources(struct dma_chan *chan) 614 { 615 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 616 struct rz_dmac *dmac = to_rz_dmac(chan->device); 617 struct rz_dmac_desc *desc, *_desc; 618 unsigned long flags; 619 int ret; 620 621 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 622 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 623 if (ret) { 624 dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!", 625 dma_chan_name(chan), ret); 626 } 627 628 spin_lock_irqsave(&channel->vc.lock, flags); 629 630 rz_lmdesc_setup(channel, channel->lmdesc.base); 631 632 /* Skip touching HW if RPM resume failed. Let the cleanup do its jobs. */ 633 if (!ret) 634 rz_dmac_disable_hw(channel); 635 636 if (channel->mid_rid >= 0) { 637 clear_bit(channel->mid_rid, dmac->modules); 638 channel->mid_rid = -EINVAL; 639 } 640 641 channel->status = 0; 642 rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack); 643 channel->dmac_ack = -EINVAL; 644 645 spin_unlock_irqrestore(&channel->vc.lock, flags); 646 647 vchan_free_chan_resources(&channel->vc); 648 649 spin_lock_irqsave(&channel->vc.lock, flags); 650 651 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { 652 list_del(&desc->node); 653 kfree(desc); 654 channel->descs_allocated--; 655 } 656 657 INIT_LIST_HEAD(&channel->ld_free); 658 659 spin_unlock_irqrestore(&channel->vc.lock, flags); 660 } 661 662 static struct dma_async_tx_descriptor * 663 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 664 size_t len, unsigned long flags) 665 { 666 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 667 struct rz_dmac *dmac = to_rz_dmac(chan->device); 668 struct rz_dmac_desc *desc; 669 670 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", 671 __func__, channel->index, &src, &dest, len); 672 673 scoped_guard(spinlock_irqsave, &channel->vc.lock) { 674 desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); 675 if (!desc) 676 return NULL; 677 678 list_del(&desc->node); 679 } 680 681 desc->type = RZ_DMAC_DESC_MEMCPY; 682 desc->src = src; 683 desc->dest = dest; 684 desc->len = len; 685 desc->direction = DMA_MEM_TO_MEM; 686 687 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 688 } 689 690 static struct dma_async_tx_descriptor * 691 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 692 unsigned int sg_len, 693 enum dma_transfer_direction direction, 694 unsigned long flags, void *context) 695 { 696 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 697 struct rz_dmac_desc *desc; 698 struct scatterlist *sg; 699 int dma_length = 0; 700 int i = 0; 701 702 scoped_guard(spinlock_irqsave, &channel->vc.lock) { 703 desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); 704 if (!desc) 705 return NULL; 706 707 list_del(&desc->node); 708 } 709 710 for_each_sg(sgl, sg, sg_len, i) 711 dma_length += sg_dma_len(sg); 712 713 desc->type = RZ_DMAC_DESC_SLAVE_SG; 714 desc->sg = sgl; 715 desc->sgcount = sg_len; 716 desc->len = dma_length; 717 desc->direction = direction; 718 719 if (direction == DMA_DEV_TO_MEM) 720 desc->src = channel->src_per_address; 721 else 722 desc->dest = channel->dst_per_address; 723 724 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 725 } 726 727 static struct dma_async_tx_descriptor * 728 rz_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 729 size_t buf_len, size_t period_len, 730 enum dma_transfer_direction direction, 731 unsigned long flags) 732 { 733 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 734 struct rz_dmac_desc *desc; 735 size_t periods; 736 737 if (!is_slave_direction(direction)) 738 return NULL; 739 740 if (!period_len || !buf_len) 741 return NULL; 742 743 periods = buf_len / period_len; 744 if (!periods || periods > DMAC_NR_LMDESC) 745 return NULL; 746 747 scoped_guard(spinlock_irqsave, &channel->vc.lock) { 748 if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) 749 return NULL; 750 751 desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); 752 if (!desc) 753 return NULL; 754 755 list_del(&desc->node); 756 757 channel->status |= BIT(RZ_DMAC_CHAN_STATUS_CYCLIC); 758 } 759 760 desc->type = RZ_DMAC_DESC_CYCLIC; 761 desc->sgcount = period_len; 762 desc->len = buf_len; 763 desc->direction = direction; 764 765 if (direction == DMA_DEV_TO_MEM) { 766 desc->src = channel->src_per_address; 767 desc->dest = buf_addr; 768 } else { 769 desc->src = buf_addr; 770 desc->dest = channel->dst_per_address; 771 } 772 773 return vchan_tx_prep(&channel->vc, &desc->vd, flags); 774 } 775 776 static int rz_dmac_terminate_all(struct dma_chan *chan) 777 { 778 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 779 struct rz_dmac *dmac = to_rz_dmac(chan->device); 780 unsigned long flags; 781 LIST_HEAD(head); 782 int ret; 783 784 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 785 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 786 if (ret) { 787 dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!", 788 dma_chan_name(chan), ret); 789 } 790 791 spin_lock_irqsave(&channel->vc.lock, flags); 792 /* Don't return if RPM failed. Let the cleanup do its jobs. */ 793 if (!ret) 794 rz_dmac_disable_hw(channel); 795 rz_lmdesc_setup(channel, channel->lmdesc.base); 796 797 if (channel->desc) { 798 vchan_terminate_vdesc(&channel->desc->vd); 799 channel->desc = NULL; 800 } 801 802 vchan_get_all_descriptors(&channel->vc, &head); 803 804 channel->status = 0; 805 806 spin_unlock_irqrestore(&channel->vc.lock, flags); 807 vchan_dma_desc_free_list(&channel->vc, &head); 808 809 return ret; 810 } 811 812 static void rz_dmac_issue_pending(struct dma_chan *chan) 813 { 814 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 815 struct rz_dmac *dmac = to_rz_dmac(chan->device); 816 unsigned long flags; 817 int ret; 818 819 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 820 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 821 if (ret) 822 return; 823 824 spin_lock_irqsave(&channel->vc.lock, flags); 825 826 /* 827 * Issue the descriptor. If another transfer is already in progress, the 828 * issued descriptor will be handled after the current transfer finishes. 829 */ 830 if (vchan_issue_pending(&channel->vc) && !channel->desc) 831 rz_dmac_xfer_desc(channel); 832 833 spin_unlock_irqrestore(&channel->vc.lock, flags); 834 } 835 836 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds) 837 { 838 u8 i; 839 static const enum dma_slave_buswidth ds_lut[] = { 840 DMA_SLAVE_BUSWIDTH_1_BYTE, 841 DMA_SLAVE_BUSWIDTH_2_BYTES, 842 DMA_SLAVE_BUSWIDTH_4_BYTES, 843 DMA_SLAVE_BUSWIDTH_8_BYTES, 844 DMA_SLAVE_BUSWIDTH_16_BYTES, 845 DMA_SLAVE_BUSWIDTH_32_BYTES, 846 DMA_SLAVE_BUSWIDTH_64_BYTES, 847 DMA_SLAVE_BUSWIDTH_128_BYTES, 848 }; 849 850 for (i = 0; i < ARRAY_SIZE(ds_lut); i++) { 851 if (ds_lut[i] == ds) 852 return i; 853 } 854 855 return CHCFG_DS_INVALID; 856 } 857 858 static int rz_dmac_config(struct dma_chan *chan, 859 struct dma_slave_config *config) 860 { 861 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 862 u32 val; 863 864 channel->dst_per_address = config->dst_addr; 865 channel->chcfg &= ~CHCFG_FILL_DDS_MASK; 866 if (channel->dst_per_address) { 867 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); 868 if (val == CHCFG_DS_INVALID) 869 return -EINVAL; 870 871 channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val); 872 } 873 874 channel->src_per_address = config->src_addr; 875 channel->chcfg &= ~CHCFG_FILL_SDS_MASK; 876 if (channel->src_per_address) { 877 val = rz_dmac_ds_to_val_mapping(config->src_addr_width); 878 if (val == CHCFG_DS_INVALID) 879 return -EINVAL; 880 881 channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val); 882 } 883 884 return 0; 885 } 886 887 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) 888 { 889 struct rz_dmac_chan *channel = to_rz_dmac_chan(vd->tx.chan); 890 struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); 891 struct rz_dmac_desc *desc = to_rz_dmac_desc(vd); 892 893 guard(spinlock_irqsave)(&vc->lock); 894 895 list_add_tail(&desc->node, &channel->ld_free); 896 } 897 898 static void rz_dmac_device_synchronize(struct dma_chan *chan) 899 { 900 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 901 struct rz_dmac *dmac = to_rz_dmac(chan->device); 902 u32 chstat; 903 int ret; 904 905 vchan_synchronize(&channel->vc); 906 907 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 908 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 909 if (ret) 910 return; 911 912 ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), 913 100, 100000, false, channel, CHSTAT, 1); 914 if (ret < 0) 915 dev_warn(dmac->dev, "DMA Timeout"); 916 917 rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); 918 rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack); 919 } 920 921 static struct rz_lmdesc * 922 rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc) 923 { 924 struct rz_lmdesc *next = ++lmdesc; 925 926 if (next >= base + DMAC_NR_LMDESC) 927 next = base; 928 929 return next; 930 } 931 932 static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, 933 struct rz_dmac_desc *desc, u32 crla) 934 { 935 struct rz_lmdesc *lmdesc = desc->start_lmdesc; 936 struct dma_chan *chan = &channel->vc.chan; 937 struct rz_dmac *dmac = to_rz_dmac(chan->device); 938 u32 residue = 0, i = 0; 939 940 while (lmdesc->nxla != crla) { 941 lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); 942 if (++i >= DMAC_NR_LMDESC) 943 return 0; 944 } 945 946 /* Calculate residue from next lmdesc to end of virtual desc */ 947 if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) { 948 u32 start_lmdesc_addr = rz_dmac_lmdesc_addr(channel, desc->start_lmdesc); 949 950 while (lmdesc->nxla != start_lmdesc_addr) { 951 residue += lmdesc->tb; 952 lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); 953 } 954 } else { 955 while (lmdesc->chcfg & CHCFG_DEM) { 956 residue += lmdesc->tb; 957 lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); 958 } 959 } 960 961 dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue); 962 963 return residue; 964 } 965 966 static int rz_dmac_chan_get_residue(struct device *dev, struct rz_dmac_chan *channel, 967 dma_cookie_t cookie, u32 *residue) 968 { 969 struct rz_dmac_desc *desc = NULL; 970 struct virt_dma_desc *vd; 971 u32 crla, crtb, i; 972 int ret; 973 974 vd = vchan_find_desc(&channel->vc, cookie); 975 if (vd) { 976 /* Descriptor has been issued but not yet processed. */ 977 desc = to_rz_dmac_desc(vd); 978 *residue = desc->len; 979 return 0; 980 } else if (channel->desc && channel->desc->vd.tx.cookie == cookie) { 981 /* Descriptor is currently processed. */ 982 desc = channel->desc; 983 } 984 985 if (!desc) { 986 /* Descriptor was not found. May be already completed by now. */ 987 *residue = 0; 988 return 0; 989 } 990 991 PM_RUNTIME_ACQUIRE_IF_ENABLED(dev, pm); 992 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 993 if (ret) 994 return ret; 995 996 /* 997 * We need to read two registers. Make sure the hardware does not move 998 * to next lmdesc while reading the current lmdesc. Trying it 3 times 999 * should be enough: initial read, retry, retry for the paranoid. 1000 */ 1001 for (i = 0; i < 3; i++) { 1002 crla = rz_dmac_ch_readl(channel, CRLA, 1); 1003 crtb = rz_dmac_ch_readl(channel, CRTB, 1); 1004 /* Still the same? */ 1005 if (crla == rz_dmac_ch_readl(channel, CRLA, 1)) 1006 break; 1007 } 1008 1009 WARN_ONCE(i >= 3, "residue might not be continuous!"); 1010 1011 /* 1012 * Calculate number of bytes transferred in processing virtual descriptor. 1013 * One virtual descriptor can have many lmdesc. 1014 */ 1015 *residue = crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, desc, crla); 1016 1017 return 0; 1018 } 1019 1020 static enum dma_status rz_dmac_tx_status(struct dma_chan *chan, 1021 dma_cookie_t cookie, 1022 struct dma_tx_state *txstate) 1023 { 1024 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 1025 struct rz_dmac *dmac = to_rz_dmac(chan->device); 1026 enum dma_status status; 1027 u32 residue; 1028 1029 scoped_guard(spinlock_irqsave, &channel->vc.lock) { 1030 int ret; 1031 1032 status = dma_cookie_status(chan, cookie, txstate); 1033 if (status == DMA_COMPLETE || !txstate) 1034 return status; 1035 1036 ret = rz_dmac_chan_get_residue(dmac->dev, channel, cookie, &residue); 1037 if (ret) 1038 return DMA_ERROR; 1039 1040 if (status == DMA_IN_PROGRESS && rz_dmac_chan_is_paused(channel)) 1041 status = DMA_PAUSED; 1042 } 1043 1044 dma_set_residue(txstate, residue); 1045 1046 return status; 1047 } 1048 1049 static int rz_dmac_device_pause_set(struct rz_dmac_chan *channel, 1050 unsigned long set_bitmask) 1051 { 1052 int ret = 0; 1053 u32 val; 1054 1055 lockdep_assert_held(&channel->vc.lock); 1056 1057 if (!rz_dmac_chan_is_enabled(channel)) 1058 return 0; 1059 1060 if (rz_dmac_chan_is_paused(channel)) 1061 goto set_bit; 1062 1063 rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1); 1064 ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val, 1065 (val & CHSTAT_SUS), 1, 1024, false, 1066 channel, CHSTAT, 1); 1067 1068 set_bit: 1069 channel->status |= set_bitmask; 1070 1071 return ret; 1072 } 1073 1074 static int rz_dmac_device_pause(struct dma_chan *chan) 1075 { 1076 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 1077 struct rz_dmac *dmac = to_rz_dmac(chan->device); 1078 int ret; 1079 1080 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 1081 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 1082 if (ret) 1083 return ret; 1084 1085 guard(spinlock_irqsave)(&channel->vc.lock); 1086 1087 return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED)); 1088 } 1089 1090 static int rz_dmac_device_pause_internal(struct rz_dmac_chan *channel) 1091 { 1092 lockdep_assert_held(&channel->vc.lock); 1093 1094 /* Skip channels explicitly paused by consummers or disabled. */ 1095 if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED) || 1096 !rz_dmac_chan_is_enabled(channel)) 1097 return 0; 1098 1099 return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)); 1100 } 1101 1102 static int rz_dmac_device_resume_set(struct rz_dmac_chan *channel, 1103 unsigned long clear_bitmask) 1104 { 1105 u32 val; 1106 int ret; 1107 1108 lockdep_assert_held(&channel->vc.lock); 1109 1110 /* 1111 * We can be: 1112 * 1113 * 1/ after the channel was paused by a consummer and now it 1114 * needs to be resummed 1115 * 2/ after the channel was paused internally (as a result of 1116 * a system suspend with power loss or not) 1117 * 3/ after the channel was paused by a consummer, the system 1118 * went through a system suspend (with power loss or not) 1119 * and the consummer wants to resume the channel 1120 * 1121 * To cover all the above cases we set both CLRSUS and SETEN. 1122 * 1123 * In case 1/ setting SETEN while the channel is still enabled 1124 * is harmless for the controller. 1125 * 1126 * In case 2/ the channel is disabled when calling this function 1127 * and setting CLRSUS is harmless for the controller as the 1128 * channel is disabled anyway. 1129 * 1130 * In case 3/ the channel is disabled/enabled if the system 1131 * went though a suspend with power loss/or not and setting 1132 * CLRSUS/SETEN is harmless for the controller as the channel 1133 * is enabled/disabled anyway. 1134 */ 1135 1136 rz_dmac_ch_writel(channel, CHCTRL_CLRSUS | CHCTRL_SETEN, CHCTRL, 1); 1137 1138 ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val, 1139 ((val & (CHSTAT_SUS | CHSTAT_EN)) == CHSTAT_EN), 1140 1, 1024, false, channel, CHSTAT, 1); 1141 1142 channel->status &= ~clear_bitmask; 1143 1144 return ret; 1145 } 1146 1147 static int rz_dmac_device_resume(struct dma_chan *chan) 1148 { 1149 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 1150 struct rz_dmac *dmac = to_rz_dmac(chan->device); 1151 int ret; 1152 1153 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 1154 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 1155 if (ret) 1156 return ret; 1157 1158 guard(spinlock_irqsave)(&channel->vc.lock); 1159 1160 if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED))) 1161 return 0; 1162 1163 return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED)); 1164 } 1165 1166 static int rz_dmac_device_resume_internal(struct rz_dmac_chan *channel) 1167 { 1168 lockdep_assert_held(&channel->vc.lock); 1169 1170 if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL))) 1171 return 0; 1172 1173 return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)); 1174 } 1175 1176 /* 1177 * ----------------------------------------------------------------------------- 1178 * IRQ handling 1179 */ 1180 1181 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) 1182 { 1183 struct dma_chan *chan = &channel->vc.chan; 1184 struct rz_dmac *dmac = to_rz_dmac(chan->device); 1185 u32 chstat; 1186 1187 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); 1188 if (chstat & CHSTAT_ER) { 1189 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", 1190 channel->index, chstat); 1191 1192 scoped_guard(spinlock_irqsave, &channel->vc.lock) 1193 rz_dmac_disable_hw(channel); 1194 return; 1195 } 1196 1197 /* 1198 * No need to lock. This just clears the END interrupt. Writing 1199 * zeros to CHCTRL is just ignored by HW. 1200 */ 1201 rz_dmac_ch_writel(channel, CHCTRL_CLREND, CHCTRL, 1); 1202 } 1203 1204 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) 1205 { 1206 struct rz_dmac_chan *channel = dev_id; 1207 1208 if (channel) { 1209 rz_dmac_irq_handle_channel(channel); 1210 return IRQ_WAKE_THREAD; 1211 } 1212 /* handle DMAERR irq */ 1213 return IRQ_HANDLED; 1214 } 1215 1216 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) 1217 { 1218 struct rz_dmac_chan *channel = dev_id; 1219 struct rz_dmac_desc *desc; 1220 1221 guard(spinlock_irqsave)(&channel->vc.lock); 1222 1223 desc = channel->desc; 1224 if (!desc) 1225 return IRQ_HANDLED; 1226 1227 if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) { 1228 vchan_cyclic_callback(&desc->vd); 1229 } else { 1230 vchan_cookie_complete(&desc->vd); 1231 channel->desc = NULL; 1232 1233 rz_dmac_xfer_desc(channel); 1234 } 1235 1236 return IRQ_HANDLED; 1237 } 1238 1239 /* 1240 * ----------------------------------------------------------------------------- 1241 * OF xlate and channel filter 1242 */ 1243 1244 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) 1245 { 1246 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); 1247 struct rz_dmac *dmac = to_rz_dmac(chan->device); 1248 struct of_phandle_args *dma_spec = arg; 1249 u32 ch_cfg; 1250 1251 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; 1252 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; 1253 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | 1254 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); 1255 1256 channel->dmac_ack = rz_dmac_get_ack_no(dmac->info, channel->mid_rid); 1257 1258 return !test_and_set_bit(channel->mid_rid, dmac->modules); 1259 } 1260 1261 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, 1262 struct of_dma *ofdma) 1263 { 1264 dma_cap_mask_t mask; 1265 1266 if (dma_spec->args_count != 1) 1267 return NULL; 1268 1269 /* Only slave DMA channels can be allocated via DT */ 1270 dma_cap_zero(mask); 1271 dma_cap_set(DMA_SLAVE, mask); 1272 1273 return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec, 1274 ofdma->of_node); 1275 } 1276 1277 /* 1278 * ----------------------------------------------------------------------------- 1279 * Probe and remove 1280 */ 1281 1282 static int rz_dmac_chan_probe(struct rz_dmac *dmac, 1283 struct rz_dmac_chan *channel, 1284 u8 index) 1285 { 1286 struct platform_device *pdev = to_platform_device(dmac->dev); 1287 struct rz_lmdesc *lmdesc; 1288 char pdev_irqname[6]; 1289 char *irqname; 1290 int irq, ret; 1291 1292 channel->index = index; 1293 channel->mid_rid = -EINVAL; 1294 channel->dmac_ack = -EINVAL; 1295 1296 /* Set io base address for each channel */ 1297 if (index < 8) { 1298 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + 1299 EACH_CHANNEL_OFFSET * index; 1300 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; 1301 } else { 1302 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + 1303 EACH_CHANNEL_OFFSET * (index - 8); 1304 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; 1305 } 1306 1307 /* Allocate descriptors */ 1308 lmdesc = dmam_alloc_coherent(&pdev->dev, 1309 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, 1310 &channel->lmdesc.base_dma, GFP_KERNEL); 1311 if (!lmdesc) { 1312 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); 1313 return -ENOMEM; 1314 } 1315 rz_lmdesc_setup(channel, lmdesc); 1316 1317 channel->vc.desc_free = rz_dmac_virt_desc_free; 1318 vchan_init(&channel->vc, &dmac->engine); 1319 INIT_LIST_HEAD(&channel->ld_free); 1320 1321 /* Initialize register for each channel */ 1322 rz_dmac_disable_hw(channel); 1323 1324 /* Request the channel interrupt. */ 1325 scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); 1326 irq = platform_get_irq_byname(pdev, pdev_irqname); 1327 if (irq < 0) 1328 return irq; 1329 1330 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 1331 dev_name(dmac->dev), index); 1332 if (!irqname) 1333 return -ENOMEM; 1334 1335 ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, 1336 rz_dmac_irq_handler_thread, 0, 1337 irqname, channel); 1338 if (ret) 1339 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); 1340 1341 return ret; 1342 } 1343 1344 static void rz_dmac_put_device(void *_dev) 1345 { 1346 struct device *dev = _dev; 1347 1348 put_device(dev); 1349 } 1350 1351 static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) 1352 { 1353 struct device_node *np = dev->of_node; 1354 struct of_phandle_args args; 1355 uint32_t dmac_index; 1356 int ret; 1357 1358 if (!dmac->info->icu_register_dma_req) 1359 return 0; 1360 1361 ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); 1362 if (ret) 1363 return ret; 1364 1365 dmac->icu.pdev = of_find_device_by_node(args.np); 1366 of_node_put(args.np); 1367 if (!dmac->icu.pdev) { 1368 dev_err(dev, "ICU device not found.\n"); 1369 return -ENODEV; 1370 } 1371 1372 ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev); 1373 if (ret) 1374 return ret; 1375 1376 dmac_index = args.args[0]; 1377 if (dmac_index > RZV2H_MAX_DMAC_INDEX) { 1378 dev_err(dev, "DMAC index %u invalid.\n", dmac_index); 1379 return -EINVAL; 1380 } 1381 dmac->icu.dmac_index = dmac_index; 1382 1383 return 0; 1384 } 1385 1386 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) 1387 { 1388 struct device_node *np = dev->of_node; 1389 int ret; 1390 1391 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 1392 if (ret < 0) { 1393 dev_err(dev, "unable to read dma-channels property\n"); 1394 return ret; 1395 } 1396 1397 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { 1398 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); 1399 return -EINVAL; 1400 } 1401 1402 return rz_dmac_parse_of_icu(dev, dmac); 1403 } 1404 1405 static int rz_dmac_probe(struct platform_device *pdev) 1406 { 1407 const char *irqname = "error"; 1408 struct dma_device *engine; 1409 struct rz_dmac *dmac; 1410 int ret; 1411 int irq; 1412 u8 i; 1413 1414 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1415 if (!dmac) 1416 return -ENOMEM; 1417 1418 dmac->info = device_get_match_data(&pdev->dev); 1419 dmac->dev = &pdev->dev; 1420 platform_set_drvdata(pdev, dmac); 1421 1422 ret = rz_dmac_parse_of(&pdev->dev, dmac); 1423 if (ret < 0) 1424 return ret; 1425 1426 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 1427 sizeof(*dmac->channels), GFP_KERNEL); 1428 if (!dmac->channels) 1429 return -ENOMEM; 1430 1431 /* Request resources */ 1432 dmac->base = devm_platform_ioremap_resource(pdev, 0); 1433 if (IS_ERR(dmac->base)) 1434 return PTR_ERR(dmac->base); 1435 1436 if (!dmac->info->icu_register_dma_req) { 1437 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); 1438 if (IS_ERR(dmac->ext_base)) 1439 return PTR_ERR(dmac->ext_base); 1440 } 1441 1442 /* Initialize the channels. */ 1443 INIT_LIST_HEAD(&dmac->engine.channels); 1444 1445 dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev); 1446 if (IS_ERR(dmac->rstc)) 1447 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), 1448 "failed to get resets\n"); 1449 1450 pm_runtime_irq_safe(&pdev->dev); 1451 pm_runtime_enable(&pdev->dev); 1452 ret = pm_runtime_resume_and_get(&pdev->dev); 1453 if (ret < 0) { 1454 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); 1455 goto err_pm_disable; 1456 } 1457 1458 ret = reset_control_deassert(dmac->rstc); 1459 if (ret) 1460 goto err_pm_runtime_put; 1461 1462 for (i = 0; i < dmac->n_channels; i++) { 1463 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); 1464 if (ret < 0) 1465 goto err; 1466 } 1467 1468 /* Register interrupt handler for error */ 1469 irq = platform_get_irq_byname_optional(pdev, irqname); 1470 if (irq > 0) { 1471 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, 1472 irqname, NULL); 1473 if (ret) { 1474 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", 1475 irq, ret); 1476 goto err; 1477 } 1478 } 1479 1480 /* Register the DMAC as a DMA provider for DT. */ 1481 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, 1482 NULL); 1483 if (ret < 0) 1484 goto err; 1485 1486 /* Register the DMA engine device. */ 1487 engine = &dmac->engine; 1488 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1489 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1490 dma_cap_set(DMA_CYCLIC, engine->cap_mask); 1491 engine->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1492 engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1493 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); 1494 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); 1495 1496 engine->dev = &pdev->dev; 1497 1498 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; 1499 engine->device_free_chan_resources = rz_dmac_free_chan_resources; 1500 engine->device_tx_status = rz_dmac_tx_status; 1501 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; 1502 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; 1503 engine->device_prep_dma_cyclic = rz_dmac_prep_dma_cyclic; 1504 engine->device_config = rz_dmac_config; 1505 engine->device_terminate_all = rz_dmac_terminate_all; 1506 engine->device_issue_pending = rz_dmac_issue_pending; 1507 engine->device_synchronize = rz_dmac_device_synchronize; 1508 engine->device_pause = rz_dmac_device_pause; 1509 engine->device_resume = rz_dmac_device_resume; 1510 1511 engine->copy_align = DMAENGINE_ALIGN_1_BYTE; 1512 dma_set_max_seg_size(engine->dev, U32_MAX); 1513 1514 ret = dma_async_device_register(engine); 1515 if (ret < 0) { 1516 dev_err(&pdev->dev, "unable to register\n"); 1517 goto dma_register_err; 1518 } 1519 return 0; 1520 1521 dma_register_err: 1522 of_dma_controller_free(pdev->dev.of_node); 1523 err: 1524 reset_control_assert(dmac->rstc); 1525 err_pm_runtime_put: 1526 pm_runtime_put(&pdev->dev); 1527 err_pm_disable: 1528 pm_runtime_disable(&pdev->dev); 1529 1530 return ret; 1531 } 1532 1533 static void rz_dmac_remove(struct platform_device *pdev) 1534 { 1535 struct rz_dmac *dmac = platform_get_drvdata(pdev); 1536 1537 dma_async_device_unregister(&dmac->engine); 1538 of_dma_controller_free(pdev->dev.of_node); 1539 reset_control_assert(dmac->rstc); 1540 pm_runtime_put(&pdev->dev); 1541 pm_runtime_disable(&pdev->dev); 1542 } 1543 1544 static void rz_dmac_suspend_recover(struct rz_dmac *dmac) 1545 { 1546 int ret; 1547 1548 PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); 1549 ret = PM_RUNTIME_ACQUIRE_ERR(&pm); 1550 if (ret) 1551 return; 1552 1553 for (unsigned int i = 0; i < dmac->n_channels; i++) { 1554 struct rz_dmac_chan *channel = &dmac->channels[i]; 1555 1556 guard(spinlock_irqsave)(&channel->vc.lock); 1557 1558 if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) 1559 continue; 1560 1561 rz_dmac_device_resume_internal(channel); 1562 } 1563 } 1564 1565 static int rz_dmac_suspend(struct device *dev) 1566 { 1567 struct rz_dmac *dmac = dev_get_drvdata(dev); 1568 int ret = 0; 1569 1570 for (unsigned int i = 0; i < dmac->n_channels; i++) { 1571 struct rz_dmac_chan *channel = &dmac->channels[i]; 1572 1573 guard(spinlock_irqsave)(&channel->vc.lock); 1574 1575 if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) 1576 continue; 1577 1578 ret = rz_dmac_device_pause_internal(channel); 1579 if (ret) { 1580 dev_err(dev, "Failed to suspend channel %s\n", 1581 dma_chan_name(&channel->vc.chan)); 1582 break; 1583 } 1584 1585 channel->pm_state.nxla = rz_dmac_ch_readl(channel, NXLA, 1); 1586 } 1587 1588 if (ret) 1589 goto suspend_recover; 1590 1591 ret = reset_control_assert(dmac->rstc); 1592 if (ret) 1593 goto suspend_recover; 1594 1595 ret = pm_runtime_put_sync(dev); 1596 if (ret < 0) 1597 goto reset_deassert; 1598 1599 return 0; 1600 1601 reset_deassert: 1602 reset_control_deassert(dmac->rstc); 1603 suspend_recover: 1604 rz_dmac_suspend_recover(dmac); 1605 return ret; 1606 } 1607 1608 static int rz_dmac_resume(struct device *dev) 1609 { 1610 struct rz_dmac *dmac = dev_get_drvdata(dev); 1611 int errors = 0, ret; 1612 1613 ret = pm_runtime_resume_and_get(dev); 1614 if (ret) 1615 return ret; 1616 1617 ret = reset_control_deassert(dmac->rstc); 1618 if (ret) { 1619 /* 1620 * Do not put runtime PM here and keep the same state as in 1621 * probe. As subsequent suspend/resume cycles may follow, leave 1622 * the runtime PM as is, here, to avoid imbalances. 1623 */ 1624 return ret; 1625 } 1626 1627 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); 1628 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); 1629 1630 for (unsigned int i = 0; i < dmac->n_channels; i++) { 1631 struct rz_dmac_chan *channel = &dmac->channels[i]; 1632 1633 guard(spinlock_irqsave)(&channel->vc.lock); 1634 1635 rz_dmac_disable_hw(&dmac->channels[i]); 1636 1637 if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) 1638 continue; 1639 1640 rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); 1641 rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); 1642 1643 rz_dmac_ch_writel(channel, channel->pm_state.nxla, NXLA, 1); 1644 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); 1645 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); 1646 rz_dmac_ch_writel(channel, channel->chctrl, CHCTRL, 1); 1647 1648 ret = rz_dmac_device_resume_internal(channel); 1649 if (ret) { 1650 errors = ret; 1651 dev_err(dev, "Failed to resume channel %s, ret=%d\n", 1652 dma_chan_name(&channel->vc.chan), ret); 1653 } 1654 } 1655 1656 return errors ? : 0; 1657 } 1658 1659 static DEFINE_SIMPLE_DEV_PM_OPS(rz_dmac_pm_ops, rz_dmac_suspend, rz_dmac_resume); 1660 1661 static const struct rz_dmac_info rz_dmac_v2h_info = { 1662 .icu_register_dma_req = rzv2h_icu_register_dma_req, 1663 .icu_register_dma_ack = rzv2h_icu_register_dma_ack, 1664 .default_dma_ack_no = RZV2H_ICU_DMAC_ACK_NO_DEFAULT, 1665 .default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT, 1666 }; 1667 1668 static const struct rz_dmac_info rz_dmac_t2h_info = { 1669 .icu_register_dma_req = rzt2h_icu_register_dma_req, 1670 .default_dma_req_no = RZT2H_ICU_DMAC_REQ_NO_DEFAULT, 1671 }; 1672 1673 static const struct rz_dmac_info rz_dmac_generic_info = { 1674 .default_dma_req_no = 0, 1675 }; 1676 1677 static const struct of_device_id of_rz_dmac_match[] = { 1678 { .compatible = "renesas,r9a09g057-dmac", .data = &rz_dmac_v2h_info }, 1679 { .compatible = "renesas,r9a09g077-dmac", .data = &rz_dmac_t2h_info }, 1680 { .compatible = "renesas,rz-dmac", .data = &rz_dmac_generic_info }, 1681 { /* Sentinel */ } 1682 }; 1683 MODULE_DEVICE_TABLE(of, of_rz_dmac_match); 1684 1685 static struct platform_driver rz_dmac_driver = { 1686 .driver = { 1687 .name = "rz-dmac", 1688 .of_match_table = of_rz_dmac_match, 1689 .pm = pm_ptr(&rz_dmac_pm_ops), 1690 }, 1691 .probe = rz_dmac_probe, 1692 .remove = rz_dmac_remove, 1693 }; 1694 1695 module_platform_driver(rz_dmac_driver); 1696 1697 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver"); 1698 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>"); 1699 MODULE_LICENSE("GPL v2"); 1700