1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/dmaengine.h> 14 #include <linux/err.h> 15 #include <linux/interrupt.h> 16 #include <linux/irq.h> 17 #include <linux/dma/edma.h> 18 #include <linux/dma-mapping.h> 19 20 #include "dw-edma-core.h" 21 #include "dw-edma-v0-core.h" 22 #include "../dmaengine.h" 23 #include "../virt-dma.h" 24 25 static inline 26 struct device *dchan2dev(struct dma_chan *dchan) 27 { 28 return &dchan->dev->device; 29 } 30 31 static inline 32 struct device *chan2dev(struct dw_edma_chan *chan) 33 { 34 return &chan->vc.chan.dev->device; 35 } 36 37 static inline 38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 39 { 40 return container_of(vd, struct dw_edma_desc, vd); 41 } 42 43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 44 { 45 struct dw_edma_burst *burst; 46 47 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 48 if (unlikely(!burst)) 49 return NULL; 50 51 INIT_LIST_HEAD(&burst->list); 52 if (chunk->burst) { 53 /* Create and add new element into the linked list */ 54 chunk->bursts_alloc++; 55 list_add_tail(&burst->list, &chunk->burst->list); 56 } else { 57 /* List head */ 58 chunk->bursts_alloc = 0; 59 chunk->burst = burst; 60 } 61 62 return burst; 63 } 64 65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 66 { 67 struct dw_edma_chan *chan = desc->chan; 68 struct dw_edma *dw = chan->chip->dw; 69 struct dw_edma_chunk *chunk; 70 71 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 72 if (unlikely(!chunk)) 73 return NULL; 74 75 INIT_LIST_HEAD(&chunk->list); 76 chunk->chan = chan; 77 /* Toggling change bit (CB) in each chunk, this is a mechanism to 78 * inform the eDMA HW block that this is a new linked list ready 79 * to be consumed. 80 * - Odd chunks originate CB equal to 0 81 * - Even chunks originate CB equal to 1 82 */ 83 chunk->cb = !(desc->chunks_alloc % 2); 84 if (chan->dir == EDMA_DIR_WRITE) { 85 chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr; 86 chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr; 87 } else { 88 chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr; 89 chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr; 90 } 91 92 if (desc->chunk) { 93 /* Create and add new element into the linked list */ 94 if (!dw_edma_alloc_burst(chunk)) { 95 kfree(chunk); 96 return NULL; 97 } 98 desc->chunks_alloc++; 99 list_add_tail(&chunk->list, &desc->chunk->list); 100 } else { 101 /* List head */ 102 chunk->burst = NULL; 103 desc->chunks_alloc = 0; 104 desc->chunk = chunk; 105 } 106 107 return chunk; 108 } 109 110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 111 { 112 struct dw_edma_desc *desc; 113 114 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 115 if (unlikely(!desc)) 116 return NULL; 117 118 desc->chan = chan; 119 if (!dw_edma_alloc_chunk(desc)) { 120 kfree(desc); 121 return NULL; 122 } 123 124 return desc; 125 } 126 127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 128 { 129 struct dw_edma_burst *child, *_next; 130 131 /* Remove all the list elements */ 132 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 133 list_del(&child->list); 134 kfree(child); 135 chunk->bursts_alloc--; 136 } 137 138 /* Remove the list head */ 139 kfree(child); 140 chunk->burst = NULL; 141 } 142 143 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 144 { 145 struct dw_edma_chunk *child, *_next; 146 147 if (!desc->chunk) 148 return; 149 150 /* Remove all the list elements */ 151 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 152 dw_edma_free_burst(child); 153 list_del(&child->list); 154 kfree(child); 155 desc->chunks_alloc--; 156 } 157 158 /* Remove the list head */ 159 kfree(child); 160 desc->chunk = NULL; 161 } 162 163 static void dw_edma_free_desc(struct dw_edma_desc *desc) 164 { 165 dw_edma_free_chunk(desc); 166 kfree(desc); 167 } 168 169 static void vchan_free_desc(struct virt_dma_desc *vdesc) 170 { 171 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 172 } 173 174 static void dw_edma_start_transfer(struct dw_edma_chan *chan) 175 { 176 struct dw_edma_chunk *child; 177 struct dw_edma_desc *desc; 178 struct virt_dma_desc *vd; 179 180 vd = vchan_next_desc(&chan->vc); 181 if (!vd) 182 return; 183 184 desc = vd2dw_edma_desc(vd); 185 if (!desc) 186 return; 187 188 child = list_first_entry_or_null(&desc->chunk->list, 189 struct dw_edma_chunk, list); 190 if (!child) 191 return; 192 193 dw_edma_v0_core_start(child, !desc->xfer_sz); 194 desc->xfer_sz += child->ll_region.sz; 195 dw_edma_free_burst(child); 196 list_del(&child->list); 197 kfree(child); 198 desc->chunks_alloc--; 199 } 200 201 static int dw_edma_device_config(struct dma_chan *dchan, 202 struct dma_slave_config *config) 203 { 204 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 205 206 memcpy(&chan->config, config, sizeof(*config)); 207 chan->configured = true; 208 209 return 0; 210 } 211 212 static int dw_edma_device_pause(struct dma_chan *dchan) 213 { 214 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 215 int err = 0; 216 217 if (!chan->configured) 218 err = -EPERM; 219 else if (chan->status != EDMA_ST_BUSY) 220 err = -EPERM; 221 else if (chan->request != EDMA_REQ_NONE) 222 err = -EPERM; 223 else 224 chan->request = EDMA_REQ_PAUSE; 225 226 return err; 227 } 228 229 static int dw_edma_device_resume(struct dma_chan *dchan) 230 { 231 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 232 int err = 0; 233 234 if (!chan->configured) { 235 err = -EPERM; 236 } else if (chan->status != EDMA_ST_PAUSE) { 237 err = -EPERM; 238 } else if (chan->request != EDMA_REQ_NONE) { 239 err = -EPERM; 240 } else { 241 chan->status = EDMA_ST_BUSY; 242 dw_edma_start_transfer(chan); 243 } 244 245 return err; 246 } 247 248 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 249 { 250 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 251 int err = 0; 252 253 if (!chan->configured) { 254 /* Do nothing */ 255 } else if (chan->status == EDMA_ST_PAUSE) { 256 chan->status = EDMA_ST_IDLE; 257 chan->configured = false; 258 } else if (chan->status == EDMA_ST_IDLE) { 259 chan->configured = false; 260 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { 261 /* 262 * The channel is in a false BUSY state, probably didn't 263 * receive or lost an interrupt 264 */ 265 chan->status = EDMA_ST_IDLE; 266 chan->configured = false; 267 } else if (chan->request > EDMA_REQ_PAUSE) { 268 err = -EPERM; 269 } else { 270 chan->request = EDMA_REQ_STOP; 271 } 272 273 return err; 274 } 275 276 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 277 { 278 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 279 unsigned long flags; 280 281 spin_lock_irqsave(&chan->vc.lock, flags); 282 if (chan->configured && chan->request == EDMA_REQ_NONE && 283 chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) { 284 chan->status = EDMA_ST_BUSY; 285 dw_edma_start_transfer(chan); 286 } 287 spin_unlock_irqrestore(&chan->vc.lock, flags); 288 } 289 290 static enum dma_status 291 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 292 struct dma_tx_state *txstate) 293 { 294 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 295 struct dw_edma_desc *desc; 296 struct virt_dma_desc *vd; 297 unsigned long flags; 298 enum dma_status ret; 299 u32 residue = 0; 300 301 ret = dma_cookie_status(dchan, cookie, txstate); 302 if (ret == DMA_COMPLETE) 303 return ret; 304 305 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 306 ret = DMA_PAUSED; 307 308 if (!txstate) 309 goto ret_residue; 310 311 spin_lock_irqsave(&chan->vc.lock, flags); 312 vd = vchan_find_desc(&chan->vc, cookie); 313 if (vd) { 314 desc = vd2dw_edma_desc(vd); 315 if (desc) 316 residue = desc->alloc_sz - desc->xfer_sz; 317 } 318 spin_unlock_irqrestore(&chan->vc.lock, flags); 319 320 ret_residue: 321 dma_set_residue(txstate, residue); 322 323 return ret; 324 } 325 326 static struct dma_async_tx_descriptor * 327 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 328 { 329 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 330 enum dma_transfer_direction dir = xfer->direction; 331 phys_addr_t src_addr, dst_addr; 332 struct scatterlist *sg = NULL; 333 struct dw_edma_chunk *chunk; 334 struct dw_edma_burst *burst; 335 struct dw_edma_desc *desc; 336 u32 cnt = 0; 337 int i; 338 339 if (!chan->configured) 340 return NULL; 341 342 switch (chan->config.direction) { 343 case DMA_DEV_TO_MEM: /* local DMA */ 344 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ) 345 break; 346 return NULL; 347 case DMA_MEM_TO_DEV: /* local DMA */ 348 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) 349 break; 350 return NULL; 351 default: /* remote DMA */ 352 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ) 353 break; 354 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE) 355 break; 356 return NULL; 357 } 358 359 if (xfer->type == EDMA_XFER_CYCLIC) { 360 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 361 return NULL; 362 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 363 if (xfer->xfer.sg.len < 1) 364 return NULL; 365 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 366 if (!xfer->xfer.il->numf) 367 return NULL; 368 if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0) 369 return NULL; 370 } else { 371 return NULL; 372 } 373 374 desc = dw_edma_alloc_desc(chan); 375 if (unlikely(!desc)) 376 goto err_alloc; 377 378 chunk = dw_edma_alloc_chunk(desc); 379 if (unlikely(!chunk)) 380 goto err_alloc; 381 382 if (xfer->type == EDMA_XFER_INTERLEAVED) { 383 src_addr = xfer->xfer.il->src_start; 384 dst_addr = xfer->xfer.il->dst_start; 385 } else { 386 src_addr = chan->config.src_addr; 387 dst_addr = chan->config.dst_addr; 388 } 389 390 if (xfer->type == EDMA_XFER_CYCLIC) { 391 cnt = xfer->xfer.cyclic.cnt; 392 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 393 cnt = xfer->xfer.sg.len; 394 sg = xfer->xfer.sg.sgl; 395 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 396 if (xfer->xfer.il->numf > 0) 397 cnt = xfer->xfer.il->numf; 398 else 399 cnt = xfer->xfer.il->frame_size; 400 } 401 402 for (i = 0; i < cnt; i++) { 403 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 404 break; 405 406 if (chunk->bursts_alloc == chan->ll_max) { 407 chunk = dw_edma_alloc_chunk(desc); 408 if (unlikely(!chunk)) 409 goto err_alloc; 410 } 411 412 burst = dw_edma_alloc_burst(chunk); 413 if (unlikely(!burst)) 414 goto err_alloc; 415 416 if (xfer->type == EDMA_XFER_CYCLIC) 417 burst->sz = xfer->xfer.cyclic.len; 418 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 419 burst->sz = sg_dma_len(sg); 420 else if (xfer->type == EDMA_XFER_INTERLEAVED) 421 burst->sz = xfer->xfer.il->sgl[i].size; 422 423 chunk->ll_region.sz += burst->sz; 424 desc->alloc_sz += burst->sz; 425 426 if (chan->dir == EDMA_DIR_WRITE) { 427 burst->sar = src_addr; 428 if (xfer->type == EDMA_XFER_CYCLIC) { 429 burst->dar = xfer->xfer.cyclic.paddr; 430 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 431 src_addr += sg_dma_len(sg); 432 burst->dar = sg_dma_address(sg); 433 /* Unlike the typical assumption by other 434 * drivers/IPs the peripheral memory isn't 435 * a FIFO memory, in this case, it's a 436 * linear memory and that why the source 437 * and destination addresses are increased 438 * by the same portion (data length) 439 */ 440 } 441 } else { 442 burst->dar = dst_addr; 443 if (xfer->type == EDMA_XFER_CYCLIC) { 444 burst->sar = xfer->xfer.cyclic.paddr; 445 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 446 dst_addr += sg_dma_len(sg); 447 burst->sar = sg_dma_address(sg); 448 /* Unlike the typical assumption by other 449 * drivers/IPs the peripheral memory isn't 450 * a FIFO memory, in this case, it's a 451 * linear memory and that why the source 452 * and destination addresses are increased 453 * by the same portion (data length) 454 */ 455 } 456 } 457 458 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 459 sg = sg_next(sg); 460 } else if (xfer->type == EDMA_XFER_INTERLEAVED && 461 xfer->xfer.il->frame_size > 0) { 462 struct dma_interleaved_template *il = xfer->xfer.il; 463 struct data_chunk *dc = &il->sgl[i]; 464 465 if (il->src_sgl) { 466 src_addr += burst->sz; 467 src_addr += dmaengine_get_src_icg(il, dc); 468 } 469 470 if (il->dst_sgl) { 471 dst_addr += burst->sz; 472 dst_addr += dmaengine_get_dst_icg(il, dc); 473 } 474 } 475 } 476 477 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 478 479 err_alloc: 480 if (desc) 481 dw_edma_free_desc(desc); 482 483 return NULL; 484 } 485 486 static struct dma_async_tx_descriptor * 487 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 488 unsigned int len, 489 enum dma_transfer_direction direction, 490 unsigned long flags, void *context) 491 { 492 struct dw_edma_transfer xfer; 493 494 xfer.dchan = dchan; 495 xfer.direction = direction; 496 xfer.xfer.sg.sgl = sgl; 497 xfer.xfer.sg.len = len; 498 xfer.flags = flags; 499 xfer.type = EDMA_XFER_SCATTER_GATHER; 500 501 return dw_edma_device_transfer(&xfer); 502 } 503 504 static struct dma_async_tx_descriptor * 505 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 506 size_t len, size_t count, 507 enum dma_transfer_direction direction, 508 unsigned long flags) 509 { 510 struct dw_edma_transfer xfer; 511 512 xfer.dchan = dchan; 513 xfer.direction = direction; 514 xfer.xfer.cyclic.paddr = paddr; 515 xfer.xfer.cyclic.len = len; 516 xfer.xfer.cyclic.cnt = count; 517 xfer.flags = flags; 518 xfer.type = EDMA_XFER_CYCLIC; 519 520 return dw_edma_device_transfer(&xfer); 521 } 522 523 static struct dma_async_tx_descriptor * 524 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 525 struct dma_interleaved_template *ilt, 526 unsigned long flags) 527 { 528 struct dw_edma_transfer xfer; 529 530 xfer.dchan = dchan; 531 xfer.direction = ilt->dir; 532 xfer.xfer.il = ilt; 533 xfer.flags = flags; 534 xfer.type = EDMA_XFER_INTERLEAVED; 535 536 return dw_edma_device_transfer(&xfer); 537 } 538 539 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 540 { 541 struct dw_edma_desc *desc; 542 struct virt_dma_desc *vd; 543 unsigned long flags; 544 545 dw_edma_v0_core_clear_done_int(chan); 546 547 spin_lock_irqsave(&chan->vc.lock, flags); 548 vd = vchan_next_desc(&chan->vc); 549 if (vd) { 550 switch (chan->request) { 551 case EDMA_REQ_NONE: 552 desc = vd2dw_edma_desc(vd); 553 if (desc->chunks_alloc) { 554 chan->status = EDMA_ST_BUSY; 555 dw_edma_start_transfer(chan); 556 } else { 557 list_del(&vd->node); 558 vchan_cookie_complete(vd); 559 chan->status = EDMA_ST_IDLE; 560 } 561 break; 562 563 case EDMA_REQ_STOP: 564 list_del(&vd->node); 565 vchan_cookie_complete(vd); 566 chan->request = EDMA_REQ_NONE; 567 chan->status = EDMA_ST_IDLE; 568 break; 569 570 case EDMA_REQ_PAUSE: 571 chan->request = EDMA_REQ_NONE; 572 chan->status = EDMA_ST_PAUSE; 573 break; 574 575 default: 576 break; 577 } 578 } 579 spin_unlock_irqrestore(&chan->vc.lock, flags); 580 } 581 582 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 583 { 584 struct virt_dma_desc *vd; 585 unsigned long flags; 586 587 dw_edma_v0_core_clear_abort_int(chan); 588 589 spin_lock_irqsave(&chan->vc.lock, flags); 590 vd = vchan_next_desc(&chan->vc); 591 if (vd) { 592 list_del(&vd->node); 593 vchan_cookie_complete(vd); 594 } 595 spin_unlock_irqrestore(&chan->vc.lock, flags); 596 chan->request = EDMA_REQ_NONE; 597 chan->status = EDMA_ST_IDLE; 598 } 599 600 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) 601 { 602 struct dw_edma_irq *dw_irq = data; 603 struct dw_edma *dw = dw_irq->dw; 604 unsigned long total, pos, val; 605 unsigned long off; 606 u32 mask; 607 608 if (write) { 609 total = dw->wr_ch_cnt; 610 off = 0; 611 mask = dw_irq->wr_mask; 612 } else { 613 total = dw->rd_ch_cnt; 614 off = dw->wr_ch_cnt; 615 mask = dw_irq->rd_mask; 616 } 617 618 val = dw_edma_v0_core_status_done_int(dw, write ? 619 EDMA_DIR_WRITE : 620 EDMA_DIR_READ); 621 val &= mask; 622 for_each_set_bit(pos, &val, total) { 623 struct dw_edma_chan *chan = &dw->chan[pos + off]; 624 625 dw_edma_done_interrupt(chan); 626 } 627 628 val = dw_edma_v0_core_status_abort_int(dw, write ? 629 EDMA_DIR_WRITE : 630 EDMA_DIR_READ); 631 val &= mask; 632 for_each_set_bit(pos, &val, total) { 633 struct dw_edma_chan *chan = &dw->chan[pos + off]; 634 635 dw_edma_abort_interrupt(chan); 636 } 637 638 return IRQ_HANDLED; 639 } 640 641 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 642 { 643 return dw_edma_interrupt(irq, data, true); 644 } 645 646 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 647 { 648 return dw_edma_interrupt(irq, data, false); 649 } 650 651 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 652 { 653 dw_edma_interrupt(irq, data, true); 654 dw_edma_interrupt(irq, data, false); 655 656 return IRQ_HANDLED; 657 } 658 659 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 660 { 661 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 662 663 if (chan->status != EDMA_ST_IDLE) 664 return -EBUSY; 665 666 pm_runtime_get(chan->chip->dev); 667 668 return 0; 669 } 670 671 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 672 { 673 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 674 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 675 int ret; 676 677 while (time_before(jiffies, timeout)) { 678 ret = dw_edma_device_terminate_all(dchan); 679 if (!ret) 680 break; 681 682 if (time_after_eq(jiffies, timeout)) 683 return; 684 685 cpu_relax(); 686 } 687 688 pm_runtime_put(chan->chip->dev); 689 } 690 691 static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, 692 u32 wr_alloc, u32 rd_alloc) 693 { 694 struct dw_edma_region *dt_region; 695 struct device *dev = chip->dev; 696 struct dw_edma *dw = chip->dw; 697 struct dw_edma_chan *chan; 698 struct dw_edma_irq *irq; 699 struct dma_device *dma; 700 u32 alloc, off_alloc; 701 u32 i, j, cnt; 702 int err = 0; 703 u32 pos; 704 705 if (write) { 706 i = 0; 707 cnt = dw->wr_ch_cnt; 708 dma = &dw->wr_edma; 709 alloc = wr_alloc; 710 off_alloc = 0; 711 } else { 712 i = dw->wr_ch_cnt; 713 cnt = dw->rd_ch_cnt; 714 dma = &dw->rd_edma; 715 alloc = rd_alloc; 716 off_alloc = wr_alloc; 717 } 718 719 INIT_LIST_HEAD(&dma->channels); 720 for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) { 721 chan = &dw->chan[i]; 722 723 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL); 724 if (!dt_region) 725 return -ENOMEM; 726 727 chan->vc.chan.private = dt_region; 728 729 chan->chip = chip; 730 chan->id = j; 731 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; 732 chan->configured = false; 733 chan->request = EDMA_REQ_NONE; 734 chan->status = EDMA_ST_IDLE; 735 736 if (write) 737 chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ); 738 else 739 chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ); 740 chan->ll_max -= 1; 741 742 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 743 write ? "write" : "read", j, chan->ll_max); 744 745 if (dw->nr_irqs == 1) 746 pos = 0; 747 else 748 pos = off_alloc + (j % alloc); 749 750 irq = &dw->irq[pos]; 751 752 if (write) 753 irq->wr_mask |= BIT(j); 754 else 755 irq->rd_mask |= BIT(j); 756 757 irq->dw = dw; 758 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 759 760 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 761 write ? "write" : "read", j, 762 chan->msi.address_hi, chan->msi.address_lo, 763 chan->msi.data); 764 765 chan->vc.desc_free = vchan_free_desc; 766 vchan_init(&chan->vc, dma); 767 768 if (write) { 769 dt_region->paddr = dw->dt_region_wr[j].paddr; 770 dt_region->vaddr = dw->dt_region_wr[j].vaddr; 771 dt_region->sz = dw->dt_region_wr[j].sz; 772 } else { 773 dt_region->paddr = dw->dt_region_rd[j].paddr; 774 dt_region->vaddr = dw->dt_region_rd[j].vaddr; 775 dt_region->sz = dw->dt_region_rd[j].sz; 776 } 777 778 dw_edma_v0_core_device_config(chan); 779 } 780 781 /* Set DMA channel capabilities */ 782 dma_cap_zero(dma->cap_mask); 783 dma_cap_set(DMA_SLAVE, dma->cap_mask); 784 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 785 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 786 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 787 dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV); 788 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 789 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 790 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 791 dma->chancnt = cnt; 792 793 /* Set DMA channel callbacks */ 794 dma->dev = chip->dev; 795 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 796 dma->device_free_chan_resources = dw_edma_free_chan_resources; 797 dma->device_config = dw_edma_device_config; 798 dma->device_pause = dw_edma_device_pause; 799 dma->device_resume = dw_edma_device_resume; 800 dma->device_terminate_all = dw_edma_device_terminate_all; 801 dma->device_issue_pending = dw_edma_device_issue_pending; 802 dma->device_tx_status = dw_edma_device_tx_status; 803 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 804 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 805 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 806 807 dma_set_max_seg_size(dma->dev, U32_MAX); 808 809 /* Register DMA device */ 810 err = dma_async_device_register(dma); 811 812 return err; 813 } 814 815 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 816 { 817 if (*nr_irqs && *alloc < cnt) { 818 (*alloc)++; 819 (*nr_irqs)--; 820 } 821 } 822 823 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 824 { 825 while (*mask * alloc < cnt) 826 (*mask)++; 827 } 828 829 static int dw_edma_irq_request(struct dw_edma_chip *chip, 830 u32 *wr_alloc, u32 *rd_alloc) 831 { 832 struct device *dev = chip->dev; 833 struct dw_edma *dw = chip->dw; 834 u32 wr_mask = 1; 835 u32 rd_mask = 1; 836 int i, err = 0; 837 u32 ch_cnt; 838 int irq; 839 840 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 841 842 if (dw->nr_irqs < 1) 843 return -EINVAL; 844 845 if (dw->nr_irqs == 1) { 846 /* Common IRQ shared among all channels */ 847 irq = dw->ops->irq_vector(dev, 0); 848 err = request_irq(irq, dw_edma_interrupt_common, 849 IRQF_SHARED, dw->name, &dw->irq[0]); 850 if (err) { 851 dw->nr_irqs = 0; 852 return err; 853 } 854 855 if (irq_get_msi_desc(irq)) 856 get_cached_msi_msg(irq, &dw->irq[0].msi); 857 } else { 858 /* Distribute IRQs equally among all channels */ 859 int tmp = dw->nr_irqs; 860 861 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 862 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 863 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 864 } 865 866 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 867 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 868 869 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 870 irq = dw->ops->irq_vector(dev, i); 871 err = request_irq(irq, 872 i < *wr_alloc ? 873 dw_edma_interrupt_write : 874 dw_edma_interrupt_read, 875 IRQF_SHARED, dw->name, 876 &dw->irq[i]); 877 if (err) { 878 dw->nr_irqs = i; 879 return err; 880 } 881 882 if (irq_get_msi_desc(irq)) 883 get_cached_msi_msg(irq, &dw->irq[i].msi); 884 } 885 886 dw->nr_irqs = i; 887 } 888 889 return err; 890 } 891 892 int dw_edma_probe(struct dw_edma_chip *chip) 893 { 894 struct device *dev; 895 struct dw_edma *dw; 896 u32 wr_alloc = 0; 897 u32 rd_alloc = 0; 898 int i, err; 899 900 if (!chip) 901 return -EINVAL; 902 903 dev = chip->dev; 904 if (!dev) 905 return -EINVAL; 906 907 dw = chip->dw; 908 if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector) 909 return -EINVAL; 910 911 raw_spin_lock_init(&dw->lock); 912 913 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, 914 dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); 915 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 916 917 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, 918 dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); 919 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 920 921 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 922 return -EINVAL; 923 924 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 925 dw->wr_ch_cnt, dw->rd_ch_cnt); 926 927 /* Allocate channels */ 928 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 929 sizeof(*dw->chan), GFP_KERNEL); 930 if (!dw->chan) 931 return -ENOMEM; 932 933 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id); 934 935 /* Disable eDMA, only to establish the ideal initial conditions */ 936 dw_edma_v0_core_off(dw); 937 938 /* Request IRQs */ 939 err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); 940 if (err) 941 return err; 942 943 /* Setup write channels */ 944 err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); 945 if (err) 946 goto err_irq_free; 947 948 /* Setup read channels */ 949 err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); 950 if (err) 951 goto err_irq_free; 952 953 /* Power management */ 954 pm_runtime_enable(dev); 955 956 /* Turn debugfs on */ 957 dw_edma_v0_core_debugfs_on(chip); 958 959 return 0; 960 961 err_irq_free: 962 for (i = (dw->nr_irqs - 1); i >= 0; i--) 963 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); 964 965 dw->nr_irqs = 0; 966 967 return err; 968 } 969 EXPORT_SYMBOL_GPL(dw_edma_probe); 970 971 int dw_edma_remove(struct dw_edma_chip *chip) 972 { 973 struct dw_edma_chan *chan, *_chan; 974 struct device *dev = chip->dev; 975 struct dw_edma *dw = chip->dw; 976 int i; 977 978 /* Disable eDMA */ 979 dw_edma_v0_core_off(dw); 980 981 /* Free irqs */ 982 for (i = (dw->nr_irqs - 1); i >= 0; i--) 983 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); 984 985 /* Power management */ 986 pm_runtime_disable(dev); 987 988 /* Deregister eDMA device */ 989 dma_async_device_unregister(&dw->wr_edma); 990 list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, 991 vc.chan.device_node) { 992 tasklet_kill(&chan->vc.task); 993 list_del(&chan->vc.chan.device_node); 994 } 995 996 dma_async_device_unregister(&dw->rd_edma); 997 list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, 998 vc.chan.device_node) { 999 tasklet_kill(&chan->vc.task); 1000 list_del(&chan->vc.chan.device_node); 1001 } 1002 1003 /* Turn debugfs off */ 1004 dw_edma_v0_core_debugfs_off(chip); 1005 1006 return 0; 1007 } 1008 EXPORT_SYMBOL_GPL(dw_edma_remove); 1009 1010 MODULE_LICENSE("GPL v2"); 1011 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1012 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1013