1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2012 Marvell International Ltd. 4 */ 5 6 #include <linux/err.h> 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/types.h> 10 #include <linux/interrupt.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/slab.h> 13 #include <linux/dmaengine.h> 14 #include <linux/platform_device.h> 15 #include <linux/device.h> 16 #include <linux/platform_data/mmp_dma.h> 17 #include <linux/dmapool.h> 18 #include <linux/of_device.h> 19 #include <linux/of_dma.h> 20 #include <linux/of.h> 21 22 #include "dmaengine.h" 23 24 #define DCSR 0x0000 25 #define DALGN 0x00a0 26 #define DINT 0x00f0 27 #define DDADR 0x0200 28 #define DSADR(n) (0x0204 + ((n) << 4)) 29 #define DTADR(n) (0x0208 + ((n) << 4)) 30 #define DCMD 0x020c 31 32 #define DCSR_RUN BIT(31) /* Run Bit (read / write) */ 33 #define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */ 34 #define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */ 35 #define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */ 36 #define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */ 37 #define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */ 38 #define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */ 39 #define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */ 40 41 #define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */ 42 #define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */ 43 #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ 44 #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ 45 #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ 46 #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ 47 #define DCSR_EORINTR BIT(9) /* The end of Receive */ 48 49 #define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2)) 50 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ 51 #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ 52 53 #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */ 54 #define DDADR_STOP BIT(0) /* Stop (read / write) */ 55 56 #define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */ 57 #define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */ 58 #define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */ 59 #define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */ 60 #define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */ 61 #define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */ 62 #define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */ 63 #define DCMD_BURST8 (1 << 16) /* 8 byte burst */ 64 #define DCMD_BURST16 (2 << 16) /* 16 byte burst */ 65 #define DCMD_BURST32 (3 << 16) /* 32 byte burst */ 66 #define DCMD_WIDTH1 (1 << 14) /* 1 byte width */ 67 #define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */ 68 #define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ 69 #define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ 70 71 #define PDMA_MAX_DESC_BYTES DCMD_LENGTH 72 73 struct mmp_pdma_desc_hw { 74 u32 ddadr; /* Points to the next descriptor + flags */ 75 u32 dsadr; /* DSADR value for the current transfer */ 76 u32 dtadr; /* DTADR value for the current transfer */ 77 u32 dcmd; /* DCMD value for the current transfer */ 78 } __aligned(32); 79 80 struct mmp_pdma_desc_sw { 81 struct mmp_pdma_desc_hw desc; 82 struct list_head node; 83 struct list_head tx_list; 84 struct dma_async_tx_descriptor async_tx; 85 }; 86 87 struct mmp_pdma_phy; 88 89 struct mmp_pdma_chan { 90 struct device *dev; 91 struct dma_chan chan; 92 struct dma_async_tx_descriptor desc; 93 struct mmp_pdma_phy *phy; 94 enum dma_transfer_direction dir; 95 struct dma_slave_config slave_config; 96 97 struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel 98 * is in cyclic mode */ 99 100 /* channel's basic info */ 101 struct tasklet_struct tasklet; 102 u32 dcmd; 103 u32 drcmr; 104 u32 dev_addr; 105 106 /* list for desc */ 107 spinlock_t desc_lock; /* Descriptor list lock */ 108 struct list_head chain_pending; /* Link descriptors queue for pending */ 109 struct list_head chain_running; /* Link descriptors queue for running */ 110 bool idle; /* channel statue machine */ 111 bool byte_align; 112 113 struct dma_pool *desc_pool; /* Descriptors pool */ 114 }; 115 116 struct mmp_pdma_phy { 117 int idx; 118 void __iomem *base; 119 struct mmp_pdma_chan *vchan; 120 }; 121 122 struct mmp_pdma_device { 123 int dma_channels; 124 void __iomem *base; 125 struct device *dev; 126 struct dma_device device; 127 struct mmp_pdma_phy *phy; 128 spinlock_t phy_lock; /* protect alloc/free phy channels */ 129 }; 130 131 #define tx_to_mmp_pdma_desc(tx) \ 132 container_of(tx, struct mmp_pdma_desc_sw, async_tx) 133 #define to_mmp_pdma_desc(lh) \ 134 container_of(lh, struct mmp_pdma_desc_sw, node) 135 #define to_mmp_pdma_chan(dchan) \ 136 container_of(dchan, struct mmp_pdma_chan, chan) 137 #define to_mmp_pdma_dev(dmadev) \ 138 container_of(dmadev, struct mmp_pdma_device, device) 139 140 static int mmp_pdma_config_write(struct dma_chan *dchan, 141 struct dma_slave_config *cfg, 142 enum dma_transfer_direction direction); 143 144 static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) 145 { 146 u32 reg = (phy->idx << 4) + DDADR; 147 148 writel(addr, phy->base + reg); 149 } 150 151 static void enable_chan(struct mmp_pdma_phy *phy) 152 { 153 u32 reg, dalgn; 154 155 if (!phy->vchan) 156 return; 157 158 reg = DRCMR(phy->vchan->drcmr); 159 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); 160 161 dalgn = readl(phy->base + DALGN); 162 if (phy->vchan->byte_align) 163 dalgn |= 1 << phy->idx; 164 else 165 dalgn &= ~(1 << phy->idx); 166 writel(dalgn, phy->base + DALGN); 167 168 reg = (phy->idx << 2) + DCSR; 169 writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); 170 } 171 172 static void disable_chan(struct mmp_pdma_phy *phy) 173 { 174 u32 reg; 175 176 if (!phy) 177 return; 178 179 reg = (phy->idx << 2) + DCSR; 180 writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); 181 } 182 183 static int clear_chan_irq(struct mmp_pdma_phy *phy) 184 { 185 u32 dcsr; 186 u32 dint = readl(phy->base + DINT); 187 u32 reg = (phy->idx << 2) + DCSR; 188 189 if (!(dint & BIT(phy->idx))) 190 return -EAGAIN; 191 192 /* clear irq */ 193 dcsr = readl(phy->base + reg); 194 writel(dcsr, phy->base + reg); 195 if ((dcsr & DCSR_BUSERR) && (phy->vchan)) 196 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); 197 198 return 0; 199 } 200 201 static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id) 202 { 203 struct mmp_pdma_phy *phy = dev_id; 204 205 if (clear_chan_irq(phy) != 0) 206 return IRQ_NONE; 207 208 tasklet_schedule(&phy->vchan->tasklet); 209 return IRQ_HANDLED; 210 } 211 212 static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) 213 { 214 struct mmp_pdma_device *pdev = dev_id; 215 struct mmp_pdma_phy *phy; 216 u32 dint = readl(pdev->base + DINT); 217 int i, ret; 218 int irq_num = 0; 219 220 while (dint) { 221 i = __ffs(dint); 222 /* only handle interrupts belonging to pdma driver*/ 223 if (i >= pdev->dma_channels) 224 break; 225 dint &= (dint - 1); 226 phy = &pdev->phy[i]; 227 ret = mmp_pdma_chan_handler(irq, phy); 228 if (ret == IRQ_HANDLED) 229 irq_num++; 230 } 231 232 if (irq_num) 233 return IRQ_HANDLED; 234 235 return IRQ_NONE; 236 } 237 238 /* lookup free phy channel as descending priority */ 239 static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan) 240 { 241 int prio, i; 242 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 243 struct mmp_pdma_phy *phy, *found = NULL; 244 unsigned long flags; 245 246 /* 247 * dma channel priorities 248 * ch 0 - 3, 16 - 19 <--> (0) 249 * ch 4 - 7, 20 - 23 <--> (1) 250 * ch 8 - 11, 24 - 27 <--> (2) 251 * ch 12 - 15, 28 - 31 <--> (3) 252 */ 253 254 spin_lock_irqsave(&pdev->phy_lock, flags); 255 for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) { 256 for (i = 0; i < pdev->dma_channels; i++) { 257 if (prio != (i & 0xf) >> 2) 258 continue; 259 phy = &pdev->phy[i]; 260 if (!phy->vchan) { 261 phy->vchan = pchan; 262 found = phy; 263 goto out_unlock; 264 } 265 } 266 } 267 268 out_unlock: 269 spin_unlock_irqrestore(&pdev->phy_lock, flags); 270 return found; 271 } 272 273 static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) 274 { 275 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 276 unsigned long flags; 277 u32 reg; 278 279 if (!pchan->phy) 280 return; 281 282 /* clear the channel mapping in DRCMR */ 283 reg = DRCMR(pchan->drcmr); 284 writel(0, pchan->phy->base + reg); 285 286 spin_lock_irqsave(&pdev->phy_lock, flags); 287 pchan->phy->vchan = NULL; 288 pchan->phy = NULL; 289 spin_unlock_irqrestore(&pdev->phy_lock, flags); 290 } 291 292 /* 293 * start_pending_queue - transfer any pending transactions 294 * pending list ==> running list 295 */ 296 static void start_pending_queue(struct mmp_pdma_chan *chan) 297 { 298 struct mmp_pdma_desc_sw *desc; 299 300 /* still in running, irq will start the pending list */ 301 if (!chan->idle) { 302 dev_dbg(chan->dev, "DMA controller still busy\n"); 303 return; 304 } 305 306 if (list_empty(&chan->chain_pending)) { 307 /* chance to re-fetch phy channel with higher prio */ 308 mmp_pdma_free_phy(chan); 309 dev_dbg(chan->dev, "no pending list\n"); 310 return; 311 } 312 313 if (!chan->phy) { 314 chan->phy = lookup_phy(chan); 315 if (!chan->phy) { 316 dev_dbg(chan->dev, "no free dma channel\n"); 317 return; 318 } 319 } 320 321 /* 322 * pending -> running 323 * reintilize pending list 324 */ 325 desc = list_first_entry(&chan->chain_pending, 326 struct mmp_pdma_desc_sw, node); 327 list_splice_tail_init(&chan->chain_pending, &chan->chain_running); 328 329 /* 330 * Program the descriptor's address into the DMA controller, 331 * then start the DMA transaction 332 */ 333 set_desc(chan->phy, desc->async_tx.phys); 334 enable_chan(chan->phy); 335 chan->idle = false; 336 } 337 338 339 /* desc->tx_list ==> pending list */ 340 static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx) 341 { 342 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan); 343 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx); 344 struct mmp_pdma_desc_sw *child; 345 unsigned long flags; 346 dma_cookie_t cookie = -EBUSY; 347 348 spin_lock_irqsave(&chan->desc_lock, flags); 349 350 list_for_each_entry(child, &desc->tx_list, node) { 351 cookie = dma_cookie_assign(&child->async_tx); 352 } 353 354 /* softly link to pending list - desc->tx_list ==> pending list */ 355 list_splice_tail_init(&desc->tx_list, &chan->chain_pending); 356 357 spin_unlock_irqrestore(&chan->desc_lock, flags); 358 359 return cookie; 360 } 361 362 static struct mmp_pdma_desc_sw * 363 mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan) 364 { 365 struct mmp_pdma_desc_sw *desc; 366 dma_addr_t pdesc; 367 368 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 369 if (!desc) { 370 dev_err(chan->dev, "out of memory for link descriptor\n"); 371 return NULL; 372 } 373 374 INIT_LIST_HEAD(&desc->tx_list); 375 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 376 /* each desc has submit */ 377 desc->async_tx.tx_submit = mmp_pdma_tx_submit; 378 desc->async_tx.phys = pdesc; 379 380 return desc; 381 } 382 383 /* 384 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel. 385 * 386 * This function will create a dma pool for descriptor allocation. 387 * Request irq only when channel is requested 388 * Return - The number of allocated descriptors. 389 */ 390 391 static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) 392 { 393 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 394 395 if (chan->desc_pool) 396 return 1; 397 398 chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), 399 chan->dev, 400 sizeof(struct mmp_pdma_desc_sw), 401 __alignof__(struct mmp_pdma_desc_sw), 402 0); 403 if (!chan->desc_pool) { 404 dev_err(chan->dev, "unable to allocate descriptor pool\n"); 405 return -ENOMEM; 406 } 407 408 mmp_pdma_free_phy(chan); 409 chan->idle = true; 410 chan->dev_addr = 0; 411 return 1; 412 } 413 414 static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan, 415 struct list_head *list) 416 { 417 struct mmp_pdma_desc_sw *desc, *_desc; 418 419 list_for_each_entry_safe(desc, _desc, list, node) { 420 list_del(&desc->node); 421 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 422 } 423 } 424 425 static void mmp_pdma_free_chan_resources(struct dma_chan *dchan) 426 { 427 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 428 unsigned long flags; 429 430 spin_lock_irqsave(&chan->desc_lock, flags); 431 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 432 mmp_pdma_free_desc_list(chan, &chan->chain_running); 433 spin_unlock_irqrestore(&chan->desc_lock, flags); 434 435 dma_pool_destroy(chan->desc_pool); 436 chan->desc_pool = NULL; 437 chan->idle = true; 438 chan->dev_addr = 0; 439 mmp_pdma_free_phy(chan); 440 return; 441 } 442 443 static struct dma_async_tx_descriptor * 444 mmp_pdma_prep_memcpy(struct dma_chan *dchan, 445 dma_addr_t dma_dst, dma_addr_t dma_src, 446 size_t len, unsigned long flags) 447 { 448 struct mmp_pdma_chan *chan; 449 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 450 size_t copy = 0; 451 452 if (!dchan) 453 return NULL; 454 455 if (!len) 456 return NULL; 457 458 chan = to_mmp_pdma_chan(dchan); 459 chan->byte_align = false; 460 461 if (!chan->dir) { 462 chan->dir = DMA_MEM_TO_MEM; 463 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR; 464 chan->dcmd |= DCMD_BURST32; 465 } 466 467 do { 468 /* Allocate the link descriptor from DMA pool */ 469 new = mmp_pdma_alloc_descriptor(chan); 470 if (!new) { 471 dev_err(chan->dev, "no memory for desc\n"); 472 goto fail; 473 } 474 475 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES); 476 if (dma_src & 0x7 || dma_dst & 0x7) 477 chan->byte_align = true; 478 479 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); 480 new->desc.dsadr = dma_src; 481 new->desc.dtadr = dma_dst; 482 483 if (!first) 484 first = new; 485 else 486 prev->desc.ddadr = new->async_tx.phys; 487 488 new->async_tx.cookie = 0; 489 async_tx_ack(&new->async_tx); 490 491 prev = new; 492 len -= copy; 493 494 if (chan->dir == DMA_MEM_TO_DEV) { 495 dma_src += copy; 496 } else if (chan->dir == DMA_DEV_TO_MEM) { 497 dma_dst += copy; 498 } else if (chan->dir == DMA_MEM_TO_MEM) { 499 dma_src += copy; 500 dma_dst += copy; 501 } 502 503 /* Insert the link descriptor to the LD ring */ 504 list_add_tail(&new->node, &first->tx_list); 505 } while (len); 506 507 first->async_tx.flags = flags; /* client is in control of this ack */ 508 first->async_tx.cookie = -EBUSY; 509 510 /* last desc and fire IRQ */ 511 new->desc.ddadr = DDADR_STOP; 512 new->desc.dcmd |= DCMD_ENDIRQEN; 513 514 chan->cyclic_first = NULL; 515 516 return &first->async_tx; 517 518 fail: 519 if (first) 520 mmp_pdma_free_desc_list(chan, &first->tx_list); 521 return NULL; 522 } 523 524 static struct dma_async_tx_descriptor * 525 mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 526 unsigned int sg_len, enum dma_transfer_direction dir, 527 unsigned long flags, void *context) 528 { 529 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 530 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; 531 size_t len, avail; 532 struct scatterlist *sg; 533 dma_addr_t addr; 534 int i; 535 536 if ((sgl == NULL) || (sg_len == 0)) 537 return NULL; 538 539 chan->byte_align = false; 540 541 mmp_pdma_config_write(dchan, &chan->slave_config, dir); 542 543 for_each_sg(sgl, sg, sg_len, i) { 544 addr = sg_dma_address(sg); 545 avail = sg_dma_len(sgl); 546 547 do { 548 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); 549 if (addr & 0x7) 550 chan->byte_align = true; 551 552 /* allocate and populate the descriptor */ 553 new = mmp_pdma_alloc_descriptor(chan); 554 if (!new) { 555 dev_err(chan->dev, "no memory for desc\n"); 556 goto fail; 557 } 558 559 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); 560 if (dir == DMA_MEM_TO_DEV) { 561 new->desc.dsadr = addr; 562 new->desc.dtadr = chan->dev_addr; 563 } else { 564 new->desc.dsadr = chan->dev_addr; 565 new->desc.dtadr = addr; 566 } 567 568 if (!first) 569 first = new; 570 else 571 prev->desc.ddadr = new->async_tx.phys; 572 573 new->async_tx.cookie = 0; 574 async_tx_ack(&new->async_tx); 575 prev = new; 576 577 /* Insert the link descriptor to the LD ring */ 578 list_add_tail(&new->node, &first->tx_list); 579 580 /* update metadata */ 581 addr += len; 582 avail -= len; 583 } while (avail); 584 } 585 586 first->async_tx.cookie = -EBUSY; 587 first->async_tx.flags = flags; 588 589 /* last desc and fire IRQ */ 590 new->desc.ddadr = DDADR_STOP; 591 new->desc.dcmd |= DCMD_ENDIRQEN; 592 593 chan->dir = dir; 594 chan->cyclic_first = NULL; 595 596 return &first->async_tx; 597 598 fail: 599 if (first) 600 mmp_pdma_free_desc_list(chan, &first->tx_list); 601 return NULL; 602 } 603 604 static struct dma_async_tx_descriptor * 605 mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, 606 dma_addr_t buf_addr, size_t len, size_t period_len, 607 enum dma_transfer_direction direction, 608 unsigned long flags) 609 { 610 struct mmp_pdma_chan *chan; 611 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 612 dma_addr_t dma_src, dma_dst; 613 614 if (!dchan || !len || !period_len) 615 return NULL; 616 617 /* the buffer length must be a multiple of period_len */ 618 if (len % period_len != 0) 619 return NULL; 620 621 if (period_len > PDMA_MAX_DESC_BYTES) 622 return NULL; 623 624 chan = to_mmp_pdma_chan(dchan); 625 mmp_pdma_config_write(dchan, &chan->slave_config, direction); 626 627 switch (direction) { 628 case DMA_MEM_TO_DEV: 629 dma_src = buf_addr; 630 dma_dst = chan->dev_addr; 631 break; 632 case DMA_DEV_TO_MEM: 633 dma_dst = buf_addr; 634 dma_src = chan->dev_addr; 635 break; 636 default: 637 dev_err(chan->dev, "Unsupported direction for cyclic DMA\n"); 638 return NULL; 639 } 640 641 chan->dir = direction; 642 643 do { 644 /* Allocate the link descriptor from DMA pool */ 645 new = mmp_pdma_alloc_descriptor(chan); 646 if (!new) { 647 dev_err(chan->dev, "no memory for desc\n"); 648 goto fail; 649 } 650 651 new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | 652 (DCMD_LENGTH & period_len)); 653 new->desc.dsadr = dma_src; 654 new->desc.dtadr = dma_dst; 655 656 if (!first) 657 first = new; 658 else 659 prev->desc.ddadr = new->async_tx.phys; 660 661 new->async_tx.cookie = 0; 662 async_tx_ack(&new->async_tx); 663 664 prev = new; 665 len -= period_len; 666 667 if (chan->dir == DMA_MEM_TO_DEV) 668 dma_src += period_len; 669 else 670 dma_dst += period_len; 671 672 /* Insert the link descriptor to the LD ring */ 673 list_add_tail(&new->node, &first->tx_list); 674 } while (len); 675 676 first->async_tx.flags = flags; /* client is in control of this ack */ 677 first->async_tx.cookie = -EBUSY; 678 679 /* make the cyclic link */ 680 new->desc.ddadr = first->async_tx.phys; 681 chan->cyclic_first = first; 682 683 return &first->async_tx; 684 685 fail: 686 if (first) 687 mmp_pdma_free_desc_list(chan, &first->tx_list); 688 return NULL; 689 } 690 691 static int mmp_pdma_config_write(struct dma_chan *dchan, 692 struct dma_slave_config *cfg, 693 enum dma_transfer_direction direction) 694 { 695 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 696 u32 maxburst = 0, addr = 0; 697 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 698 699 if (!dchan) 700 return -EINVAL; 701 702 if (direction == DMA_DEV_TO_MEM) { 703 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC; 704 maxburst = cfg->src_maxburst; 705 width = cfg->src_addr_width; 706 addr = cfg->src_addr; 707 } else if (direction == DMA_MEM_TO_DEV) { 708 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG; 709 maxburst = cfg->dst_maxburst; 710 width = cfg->dst_addr_width; 711 addr = cfg->dst_addr; 712 } 713 714 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) 715 chan->dcmd |= DCMD_WIDTH1; 716 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 717 chan->dcmd |= DCMD_WIDTH2; 718 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES) 719 chan->dcmd |= DCMD_WIDTH4; 720 721 if (maxburst == 8) 722 chan->dcmd |= DCMD_BURST8; 723 else if (maxburst == 16) 724 chan->dcmd |= DCMD_BURST16; 725 else if (maxburst == 32) 726 chan->dcmd |= DCMD_BURST32; 727 728 chan->dir = direction; 729 chan->dev_addr = addr; 730 731 return 0; 732 } 733 734 static int mmp_pdma_config(struct dma_chan *dchan, 735 struct dma_slave_config *cfg) 736 { 737 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 738 739 memcpy(&chan->slave_config, cfg, sizeof(*cfg)); 740 return 0; 741 } 742 743 static int mmp_pdma_terminate_all(struct dma_chan *dchan) 744 { 745 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 746 unsigned long flags; 747 748 if (!dchan) 749 return -EINVAL; 750 751 disable_chan(chan->phy); 752 mmp_pdma_free_phy(chan); 753 spin_lock_irqsave(&chan->desc_lock, flags); 754 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 755 mmp_pdma_free_desc_list(chan, &chan->chain_running); 756 spin_unlock_irqrestore(&chan->desc_lock, flags); 757 chan->idle = true; 758 759 return 0; 760 } 761 762 static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, 763 dma_cookie_t cookie) 764 { 765 struct mmp_pdma_desc_sw *sw; 766 u32 curr, residue = 0; 767 bool passed = false; 768 bool cyclic = chan->cyclic_first != NULL; 769 770 /* 771 * If the channel does not have a phy pointer anymore, it has already 772 * been completed. Therefore, its residue is 0. 773 */ 774 if (!chan->phy) 775 return 0; 776 777 if (chan->dir == DMA_DEV_TO_MEM) 778 curr = readl(chan->phy->base + DTADR(chan->phy->idx)); 779 else 780 curr = readl(chan->phy->base + DSADR(chan->phy->idx)); 781 782 list_for_each_entry(sw, &chan->chain_running, node) { 783 u32 start, end, len; 784 785 if (chan->dir == DMA_DEV_TO_MEM) 786 start = sw->desc.dtadr; 787 else 788 start = sw->desc.dsadr; 789 790 len = sw->desc.dcmd & DCMD_LENGTH; 791 end = start + len; 792 793 /* 794 * 'passed' will be latched once we found the descriptor which 795 * lies inside the boundaries of the curr pointer. All 796 * descriptors that occur in the list _after_ we found that 797 * partially handled descriptor are still to be processed and 798 * are hence added to the residual bytes counter. 799 */ 800 801 if (passed) { 802 residue += len; 803 } else if (curr >= start && curr <= end) { 804 residue += end - curr; 805 passed = true; 806 } 807 808 /* 809 * Descriptors that have the ENDIRQEN bit set mark the end of a 810 * transaction chain, and the cookie assigned with it has been 811 * returned previously from mmp_pdma_tx_submit(). 812 * 813 * In case we have multiple transactions in the running chain, 814 * and the cookie does not match the one the user asked us 815 * about, reset the state variables and start over. 816 * 817 * This logic does not apply to cyclic transactions, where all 818 * descriptors have the ENDIRQEN bit set, and for which we 819 * can't have multiple transactions on one channel anyway. 820 */ 821 if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN)) 822 continue; 823 824 if (sw->async_tx.cookie == cookie) { 825 return residue; 826 } else { 827 residue = 0; 828 passed = false; 829 } 830 } 831 832 /* We should only get here in case of cyclic transactions */ 833 return residue; 834 } 835 836 static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan, 837 dma_cookie_t cookie, 838 struct dma_tx_state *txstate) 839 { 840 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 841 enum dma_status ret; 842 843 ret = dma_cookie_status(dchan, cookie, txstate); 844 if (likely(ret != DMA_ERROR)) 845 dma_set_residue(txstate, mmp_pdma_residue(chan, cookie)); 846 847 return ret; 848 } 849 850 /* 851 * mmp_pdma_issue_pending - Issue the DMA start command 852 * pending list ==> running list 853 */ 854 static void mmp_pdma_issue_pending(struct dma_chan *dchan) 855 { 856 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 857 unsigned long flags; 858 859 spin_lock_irqsave(&chan->desc_lock, flags); 860 start_pending_queue(chan); 861 spin_unlock_irqrestore(&chan->desc_lock, flags); 862 } 863 864 /* 865 * dma_do_tasklet 866 * Do call back 867 * Start pending list 868 */ 869 static void dma_do_tasklet(struct tasklet_struct *t) 870 { 871 struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet); 872 struct mmp_pdma_desc_sw *desc, *_desc; 873 LIST_HEAD(chain_cleanup); 874 unsigned long flags; 875 struct dmaengine_desc_callback cb; 876 877 if (chan->cyclic_first) { 878 spin_lock_irqsave(&chan->desc_lock, flags); 879 desc = chan->cyclic_first; 880 dmaengine_desc_get_callback(&desc->async_tx, &cb); 881 spin_unlock_irqrestore(&chan->desc_lock, flags); 882 883 dmaengine_desc_callback_invoke(&cb, NULL); 884 885 return; 886 } 887 888 /* submit pending list; callback for each desc; free desc */ 889 spin_lock_irqsave(&chan->desc_lock, flags); 890 891 list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) { 892 /* 893 * move the descriptors to a temporary list so we can drop 894 * the lock during the entire cleanup operation 895 */ 896 list_move(&desc->node, &chain_cleanup); 897 898 /* 899 * Look for the first list entry which has the ENDIRQEN flag 900 * set. That is the descriptor we got an interrupt for, so 901 * complete that transaction and its cookie. 902 */ 903 if (desc->desc.dcmd & DCMD_ENDIRQEN) { 904 dma_cookie_t cookie = desc->async_tx.cookie; 905 dma_cookie_complete(&desc->async_tx); 906 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie); 907 break; 908 } 909 } 910 911 /* 912 * The hardware is idle and ready for more when the 913 * chain_running list is empty. 914 */ 915 chan->idle = list_empty(&chan->chain_running); 916 917 /* Start any pending transactions automatically */ 918 start_pending_queue(chan); 919 spin_unlock_irqrestore(&chan->desc_lock, flags); 920 921 /* Run the callback for each descriptor, in order */ 922 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) { 923 struct dma_async_tx_descriptor *txd = &desc->async_tx; 924 925 /* Remove from the list of transactions */ 926 list_del(&desc->node); 927 /* Run the link descriptor callback function */ 928 dmaengine_desc_get_callback(txd, &cb); 929 dmaengine_desc_callback_invoke(&cb, NULL); 930 931 dma_pool_free(chan->desc_pool, desc, txd->phys); 932 } 933 } 934 935 static int mmp_pdma_remove(struct platform_device *op) 936 { 937 struct mmp_pdma_device *pdev = platform_get_drvdata(op); 938 struct mmp_pdma_phy *phy; 939 int i, irq = 0, irq_num = 0; 940 941 if (op->dev.of_node) 942 of_dma_controller_free(op->dev.of_node); 943 944 for (i = 0; i < pdev->dma_channels; i++) { 945 if (platform_get_irq(op, i) > 0) 946 irq_num++; 947 } 948 949 if (irq_num != pdev->dma_channels) { 950 irq = platform_get_irq(op, 0); 951 devm_free_irq(&op->dev, irq, pdev); 952 } else { 953 for (i = 0; i < pdev->dma_channels; i++) { 954 phy = &pdev->phy[i]; 955 irq = platform_get_irq(op, i); 956 devm_free_irq(&op->dev, irq, phy); 957 } 958 } 959 960 dma_async_device_unregister(&pdev->device); 961 return 0; 962 } 963 964 static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) 965 { 966 struct mmp_pdma_phy *phy = &pdev->phy[idx]; 967 struct mmp_pdma_chan *chan; 968 int ret; 969 970 chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL); 971 if (chan == NULL) 972 return -ENOMEM; 973 974 phy->idx = idx; 975 phy->base = pdev->base; 976 977 if (irq) { 978 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler, 979 IRQF_SHARED, "pdma", phy); 980 if (ret) { 981 dev_err(pdev->dev, "channel request irq fail!\n"); 982 return ret; 983 } 984 } 985 986 spin_lock_init(&chan->desc_lock); 987 chan->dev = pdev->dev; 988 chan->chan.device = &pdev->device; 989 tasklet_setup(&chan->tasklet, dma_do_tasklet); 990 INIT_LIST_HEAD(&chan->chain_pending); 991 INIT_LIST_HEAD(&chan->chain_running); 992 993 /* register virt channel to dma engine */ 994 list_add_tail(&chan->chan.device_node, &pdev->device.channels); 995 996 return 0; 997 } 998 999 static const struct of_device_id mmp_pdma_dt_ids[] = { 1000 { .compatible = "marvell,pdma-1.0", }, 1001 {} 1002 }; 1003 MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); 1004 1005 static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, 1006 struct of_dma *ofdma) 1007 { 1008 struct mmp_pdma_device *d = ofdma->of_dma_data; 1009 struct dma_chan *chan; 1010 1011 chan = dma_get_any_slave_channel(&d->device); 1012 if (!chan) 1013 return NULL; 1014 1015 to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0]; 1016 1017 return chan; 1018 } 1019 1020 static int mmp_pdma_probe(struct platform_device *op) 1021 { 1022 struct mmp_pdma_device *pdev; 1023 const struct of_device_id *of_id; 1024 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); 1025 struct resource *iores; 1026 int i, ret, irq = 0; 1027 int dma_channels = 0, irq_num = 0; 1028 const enum dma_slave_buswidth widths = 1029 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 1030 DMA_SLAVE_BUSWIDTH_4_BYTES; 1031 1032 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL); 1033 if (!pdev) 1034 return -ENOMEM; 1035 1036 pdev->dev = &op->dev; 1037 1038 spin_lock_init(&pdev->phy_lock); 1039 1040 iores = platform_get_resource(op, IORESOURCE_MEM, 0); 1041 pdev->base = devm_ioremap_resource(pdev->dev, iores); 1042 if (IS_ERR(pdev->base)) 1043 return PTR_ERR(pdev->base); 1044 1045 of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); 1046 if (of_id) { 1047 /* Parse new and deprecated dma-channels properties */ 1048 if (of_property_read_u32(pdev->dev->of_node, "dma-channels", 1049 &dma_channels)) 1050 of_property_read_u32(pdev->dev->of_node, "#dma-channels", 1051 &dma_channels); 1052 } else if (pdata && pdata->dma_channels) { 1053 dma_channels = pdata->dma_channels; 1054 } else { 1055 dma_channels = 32; /* default 32 channel */ 1056 } 1057 pdev->dma_channels = dma_channels; 1058 1059 for (i = 0; i < dma_channels; i++) { 1060 if (platform_get_irq_optional(op, i) > 0) 1061 irq_num++; 1062 } 1063 1064 pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy), 1065 GFP_KERNEL); 1066 if (pdev->phy == NULL) 1067 return -ENOMEM; 1068 1069 INIT_LIST_HEAD(&pdev->device.channels); 1070 1071 if (irq_num != dma_channels) { 1072 /* all chan share one irq, demux inside */ 1073 irq = platform_get_irq(op, 0); 1074 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler, 1075 IRQF_SHARED, "pdma", pdev); 1076 if (ret) 1077 return ret; 1078 } 1079 1080 for (i = 0; i < dma_channels; i++) { 1081 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i); 1082 ret = mmp_pdma_chan_init(pdev, i, irq); 1083 if (ret) 1084 return ret; 1085 } 1086 1087 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask); 1088 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask); 1089 dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask); 1090 dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask); 1091 pdev->device.dev = &op->dev; 1092 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources; 1093 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources; 1094 pdev->device.device_tx_status = mmp_pdma_tx_status; 1095 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy; 1096 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg; 1097 pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic; 1098 pdev->device.device_issue_pending = mmp_pdma_issue_pending; 1099 pdev->device.device_config = mmp_pdma_config; 1100 pdev->device.device_terminate_all = mmp_pdma_terminate_all; 1101 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; 1102 pdev->device.src_addr_widths = widths; 1103 pdev->device.dst_addr_widths = widths; 1104 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1105 pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1106 1107 if (pdev->dev->coherent_dma_mask) 1108 dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); 1109 else 1110 dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); 1111 1112 ret = dma_async_device_register(&pdev->device); 1113 if (ret) { 1114 dev_err(pdev->device.dev, "unable to register\n"); 1115 return ret; 1116 } 1117 1118 if (op->dev.of_node) { 1119 /* Device-tree DMA controller registration */ 1120 ret = of_dma_controller_register(op->dev.of_node, 1121 mmp_pdma_dma_xlate, pdev); 1122 if (ret < 0) { 1123 dev_err(&op->dev, "of_dma_controller_register failed\n"); 1124 dma_async_device_unregister(&pdev->device); 1125 return ret; 1126 } 1127 } 1128 1129 platform_set_drvdata(op, pdev); 1130 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels); 1131 return 0; 1132 } 1133 1134 static const struct platform_device_id mmp_pdma_id_table[] = { 1135 { "mmp-pdma", }, 1136 { }, 1137 }; 1138 1139 static struct platform_driver mmp_pdma_driver = { 1140 .driver = { 1141 .name = "mmp-pdma", 1142 .of_match_table = mmp_pdma_dt_ids, 1143 }, 1144 .id_table = mmp_pdma_id_table, 1145 .probe = mmp_pdma_probe, 1146 .remove = mmp_pdma_remove, 1147 }; 1148 1149 module_platform_driver(mmp_pdma_driver); 1150 1151 MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); 1152 MODULE_AUTHOR("Marvell International Ltd."); 1153 MODULE_LICENSE("GPL v2"); 1154