1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/dmaengine.h> 14 #include <linux/err.h> 15 #include <linux/interrupt.h> 16 #include <linux/irq.h> 17 #include <linux/dma/edma.h> 18 #include <linux/dma-mapping.h> 19 20 #include "dw-edma-core.h" 21 #include "dw-edma-v0-core.h" 22 #include "../dmaengine.h" 23 #include "../virt-dma.h" 24 25 static inline 26 struct device *dchan2dev(struct dma_chan *dchan) 27 { 28 return &dchan->dev->device; 29 } 30 31 static inline 32 struct device *chan2dev(struct dw_edma_chan *chan) 33 { 34 return &chan->vc.chan.dev->device; 35 } 36 37 static inline 38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 39 { 40 return container_of(vd, struct dw_edma_desc, vd); 41 } 42 43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 44 { 45 struct dw_edma_burst *burst; 46 47 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 48 if (unlikely(!burst)) 49 return NULL; 50 51 INIT_LIST_HEAD(&burst->list); 52 if (chunk->burst) { 53 /* Create and add new element into the linked list */ 54 chunk->bursts_alloc++; 55 list_add_tail(&burst->list, &chunk->burst->list); 56 } else { 57 /* List head */ 58 chunk->bursts_alloc = 0; 59 chunk->burst = burst; 60 } 61 62 return burst; 63 } 64 65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 66 { 67 struct dw_edma_chan *chan = desc->chan; 68 struct dw_edma *dw = chan->chip->dw; 69 struct dw_edma_chunk *chunk; 70 71 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 72 if (unlikely(!chunk)) 73 return NULL; 74 75 INIT_LIST_HEAD(&chunk->list); 76 chunk->chan = chan; 77 /* Toggling change bit (CB) in each chunk, this is a mechanism to 78 * inform the eDMA HW block that this is a new linked list ready 79 * to be consumed. 80 * - Odd chunks originate CB equal to 0 81 * - Even chunks originate CB equal to 1 82 */ 83 chunk->cb = !(desc->chunks_alloc % 2); 84 if (chan->dir == EDMA_DIR_WRITE) { 85 chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr; 86 chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr; 87 } else { 88 chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr; 89 chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr; 90 } 91 92 if (desc->chunk) { 93 /* Create and add new element into the linked list */ 94 if (!dw_edma_alloc_burst(chunk)) { 95 kfree(chunk); 96 return NULL; 97 } 98 desc->chunks_alloc++; 99 list_add_tail(&chunk->list, &desc->chunk->list); 100 } else { 101 /* List head */ 102 chunk->burst = NULL; 103 desc->chunks_alloc = 0; 104 desc->chunk = chunk; 105 } 106 107 return chunk; 108 } 109 110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 111 { 112 struct dw_edma_desc *desc; 113 114 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 115 if (unlikely(!desc)) 116 return NULL; 117 118 desc->chan = chan; 119 if (!dw_edma_alloc_chunk(desc)) { 120 kfree(desc); 121 return NULL; 122 } 123 124 return desc; 125 } 126 127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 128 { 129 struct dw_edma_burst *child, *_next; 130 131 /* Remove all the list elements */ 132 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 133 list_del(&child->list); 134 kfree(child); 135 chunk->bursts_alloc--; 136 } 137 138 /* Remove the list head */ 139 kfree(child); 140 chunk->burst = NULL; 141 } 142 143 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 144 { 145 struct dw_edma_chunk *child, *_next; 146 147 if (!desc->chunk) 148 return; 149 150 /* Remove all the list elements */ 151 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 152 dw_edma_free_burst(child); 153 list_del(&child->list); 154 kfree(child); 155 desc->chunks_alloc--; 156 } 157 158 /* Remove the list head */ 159 kfree(child); 160 desc->chunk = NULL; 161 } 162 163 static void dw_edma_free_desc(struct dw_edma_desc *desc) 164 { 165 dw_edma_free_chunk(desc); 166 kfree(desc); 167 } 168 169 static void vchan_free_desc(struct virt_dma_desc *vdesc) 170 { 171 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 172 } 173 174 static void dw_edma_start_transfer(struct dw_edma_chan *chan) 175 { 176 struct dw_edma_chunk *child; 177 struct dw_edma_desc *desc; 178 struct virt_dma_desc *vd; 179 180 vd = vchan_next_desc(&chan->vc); 181 if (!vd) 182 return; 183 184 desc = vd2dw_edma_desc(vd); 185 if (!desc) 186 return; 187 188 child = list_first_entry_or_null(&desc->chunk->list, 189 struct dw_edma_chunk, list); 190 if (!child) 191 return; 192 193 dw_edma_v0_core_start(child, !desc->xfer_sz); 194 desc->xfer_sz += child->ll_region.sz; 195 dw_edma_free_burst(child); 196 list_del(&child->list); 197 kfree(child); 198 desc->chunks_alloc--; 199 } 200 201 static int dw_edma_device_config(struct dma_chan *dchan, 202 struct dma_slave_config *config) 203 { 204 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 205 206 memcpy(&chan->config, config, sizeof(*config)); 207 chan->configured = true; 208 209 return 0; 210 } 211 212 static int dw_edma_device_pause(struct dma_chan *dchan) 213 { 214 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 215 int err = 0; 216 217 if (!chan->configured) 218 err = -EPERM; 219 else if (chan->status != EDMA_ST_BUSY) 220 err = -EPERM; 221 else if (chan->request != EDMA_REQ_NONE) 222 err = -EPERM; 223 else 224 chan->request = EDMA_REQ_PAUSE; 225 226 return err; 227 } 228 229 static int dw_edma_device_resume(struct dma_chan *dchan) 230 { 231 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 232 int err = 0; 233 234 if (!chan->configured) { 235 err = -EPERM; 236 } else if (chan->status != EDMA_ST_PAUSE) { 237 err = -EPERM; 238 } else if (chan->request != EDMA_REQ_NONE) { 239 err = -EPERM; 240 } else { 241 chan->status = EDMA_ST_BUSY; 242 dw_edma_start_transfer(chan); 243 } 244 245 return err; 246 } 247 248 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 249 { 250 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 251 int err = 0; 252 LIST_HEAD(head); 253 254 if (!chan->configured) { 255 /* Do nothing */ 256 } else if (chan->status == EDMA_ST_PAUSE) { 257 chan->status = EDMA_ST_IDLE; 258 chan->configured = false; 259 } else if (chan->status == EDMA_ST_IDLE) { 260 chan->configured = false; 261 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { 262 /* 263 * The channel is in a false BUSY state, probably didn't 264 * receive or lost an interrupt 265 */ 266 chan->status = EDMA_ST_IDLE; 267 chan->configured = false; 268 } else if (chan->request > EDMA_REQ_PAUSE) { 269 err = -EPERM; 270 } else { 271 chan->request = EDMA_REQ_STOP; 272 } 273 274 return err; 275 } 276 277 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 278 { 279 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 280 unsigned long flags; 281 282 spin_lock_irqsave(&chan->vc.lock, flags); 283 if (chan->configured && chan->request == EDMA_REQ_NONE && 284 chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) { 285 chan->status = EDMA_ST_BUSY; 286 dw_edma_start_transfer(chan); 287 } 288 spin_unlock_irqrestore(&chan->vc.lock, flags); 289 } 290 291 static enum dma_status 292 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 293 struct dma_tx_state *txstate) 294 { 295 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 296 struct dw_edma_desc *desc; 297 struct virt_dma_desc *vd; 298 unsigned long flags; 299 enum dma_status ret; 300 u32 residue = 0; 301 302 ret = dma_cookie_status(dchan, cookie, txstate); 303 if (ret == DMA_COMPLETE) 304 return ret; 305 306 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 307 ret = DMA_PAUSED; 308 309 if (!txstate) 310 goto ret_residue; 311 312 spin_lock_irqsave(&chan->vc.lock, flags); 313 vd = vchan_find_desc(&chan->vc, cookie); 314 if (vd) { 315 desc = vd2dw_edma_desc(vd); 316 if (desc) 317 residue = desc->alloc_sz - desc->xfer_sz; 318 } 319 spin_unlock_irqrestore(&chan->vc.lock, flags); 320 321 ret_residue: 322 dma_set_residue(txstate, residue); 323 324 return ret; 325 } 326 327 static struct dma_async_tx_descriptor * 328 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 329 { 330 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 331 enum dma_transfer_direction dir = xfer->direction; 332 phys_addr_t src_addr, dst_addr; 333 struct scatterlist *sg = NULL; 334 struct dw_edma_chunk *chunk; 335 struct dw_edma_burst *burst; 336 struct dw_edma_desc *desc; 337 u32 cnt = 0; 338 int i; 339 340 if (!chan->configured) 341 return NULL; 342 343 switch (chan->config.direction) { 344 case DMA_DEV_TO_MEM: /* local DMA */ 345 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ) 346 break; 347 return NULL; 348 case DMA_MEM_TO_DEV: /* local DMA */ 349 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) 350 break; 351 return NULL; 352 default: /* remote DMA */ 353 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ) 354 break; 355 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE) 356 break; 357 return NULL; 358 } 359 360 if (xfer->type == EDMA_XFER_CYCLIC) { 361 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 362 return NULL; 363 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 364 if (xfer->xfer.sg.len < 1) 365 return NULL; 366 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 367 if (!xfer->xfer.il->numf) 368 return NULL; 369 if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0) 370 return NULL; 371 } else { 372 return NULL; 373 } 374 375 desc = dw_edma_alloc_desc(chan); 376 if (unlikely(!desc)) 377 goto err_alloc; 378 379 chunk = dw_edma_alloc_chunk(desc); 380 if (unlikely(!chunk)) 381 goto err_alloc; 382 383 if (xfer->type == EDMA_XFER_INTERLEAVED) { 384 src_addr = xfer->xfer.il->src_start; 385 dst_addr = xfer->xfer.il->dst_start; 386 } else { 387 src_addr = chan->config.src_addr; 388 dst_addr = chan->config.dst_addr; 389 } 390 391 if (xfer->type == EDMA_XFER_CYCLIC) { 392 cnt = xfer->xfer.cyclic.cnt; 393 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 394 cnt = xfer->xfer.sg.len; 395 sg = xfer->xfer.sg.sgl; 396 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 397 if (xfer->xfer.il->numf > 0) 398 cnt = xfer->xfer.il->numf; 399 else 400 cnt = xfer->xfer.il->frame_size; 401 } 402 403 for (i = 0; i < cnt; i++) { 404 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 405 break; 406 407 if (chunk->bursts_alloc == chan->ll_max) { 408 chunk = dw_edma_alloc_chunk(desc); 409 if (unlikely(!chunk)) 410 goto err_alloc; 411 } 412 413 burst = dw_edma_alloc_burst(chunk); 414 if (unlikely(!burst)) 415 goto err_alloc; 416 417 if (xfer->type == EDMA_XFER_CYCLIC) 418 burst->sz = xfer->xfer.cyclic.len; 419 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 420 burst->sz = sg_dma_len(sg); 421 else if (xfer->type == EDMA_XFER_INTERLEAVED) 422 burst->sz = xfer->xfer.il->sgl[i].size; 423 424 chunk->ll_region.sz += burst->sz; 425 desc->alloc_sz += burst->sz; 426 427 if (chan->dir == EDMA_DIR_WRITE) { 428 burst->sar = src_addr; 429 if (xfer->type == EDMA_XFER_CYCLIC) { 430 burst->dar = xfer->xfer.cyclic.paddr; 431 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 432 src_addr += sg_dma_len(sg); 433 burst->dar = sg_dma_address(sg); 434 /* Unlike the typical assumption by other 435 * drivers/IPs the peripheral memory isn't 436 * a FIFO memory, in this case, it's a 437 * linear memory and that why the source 438 * and destination addresses are increased 439 * by the same portion (data length) 440 */ 441 } 442 } else { 443 burst->dar = dst_addr; 444 if (xfer->type == EDMA_XFER_CYCLIC) { 445 burst->sar = xfer->xfer.cyclic.paddr; 446 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 447 dst_addr += sg_dma_len(sg); 448 burst->sar = sg_dma_address(sg); 449 /* Unlike the typical assumption by other 450 * drivers/IPs the peripheral memory isn't 451 * a FIFO memory, in this case, it's a 452 * linear memory and that why the source 453 * and destination addresses are increased 454 * by the same portion (data length) 455 */ 456 } 457 } 458 459 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 460 sg = sg_next(sg); 461 } else if (xfer->type == EDMA_XFER_INTERLEAVED && 462 xfer->xfer.il->frame_size > 0) { 463 struct dma_interleaved_template *il = xfer->xfer.il; 464 struct data_chunk *dc = &il->sgl[i]; 465 466 if (il->src_sgl) { 467 src_addr += burst->sz; 468 src_addr += dmaengine_get_src_icg(il, dc); 469 } 470 471 if (il->dst_sgl) { 472 dst_addr += burst->sz; 473 dst_addr += dmaengine_get_dst_icg(il, dc); 474 } 475 } 476 } 477 478 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 479 480 err_alloc: 481 if (desc) 482 dw_edma_free_desc(desc); 483 484 return NULL; 485 } 486 487 static struct dma_async_tx_descriptor * 488 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 489 unsigned int len, 490 enum dma_transfer_direction direction, 491 unsigned long flags, void *context) 492 { 493 struct dw_edma_transfer xfer; 494 495 xfer.dchan = dchan; 496 xfer.direction = direction; 497 xfer.xfer.sg.sgl = sgl; 498 xfer.xfer.sg.len = len; 499 xfer.flags = flags; 500 xfer.type = EDMA_XFER_SCATTER_GATHER; 501 502 return dw_edma_device_transfer(&xfer); 503 } 504 505 static struct dma_async_tx_descriptor * 506 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 507 size_t len, size_t count, 508 enum dma_transfer_direction direction, 509 unsigned long flags) 510 { 511 struct dw_edma_transfer xfer; 512 513 xfer.dchan = dchan; 514 xfer.direction = direction; 515 xfer.xfer.cyclic.paddr = paddr; 516 xfer.xfer.cyclic.len = len; 517 xfer.xfer.cyclic.cnt = count; 518 xfer.flags = flags; 519 xfer.type = EDMA_XFER_CYCLIC; 520 521 return dw_edma_device_transfer(&xfer); 522 } 523 524 static struct dma_async_tx_descriptor * 525 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 526 struct dma_interleaved_template *ilt, 527 unsigned long flags) 528 { 529 struct dw_edma_transfer xfer; 530 531 xfer.dchan = dchan; 532 xfer.direction = ilt->dir; 533 xfer.xfer.il = ilt; 534 xfer.flags = flags; 535 xfer.type = EDMA_XFER_INTERLEAVED; 536 537 return dw_edma_device_transfer(&xfer); 538 } 539 540 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 541 { 542 struct dw_edma_desc *desc; 543 struct virt_dma_desc *vd; 544 unsigned long flags; 545 546 dw_edma_v0_core_clear_done_int(chan); 547 548 spin_lock_irqsave(&chan->vc.lock, flags); 549 vd = vchan_next_desc(&chan->vc); 550 if (vd) { 551 switch (chan->request) { 552 case EDMA_REQ_NONE: 553 desc = vd2dw_edma_desc(vd); 554 if (desc->chunks_alloc) { 555 chan->status = EDMA_ST_BUSY; 556 dw_edma_start_transfer(chan); 557 } else { 558 list_del(&vd->node); 559 vchan_cookie_complete(vd); 560 chan->status = EDMA_ST_IDLE; 561 } 562 break; 563 564 case EDMA_REQ_STOP: 565 list_del(&vd->node); 566 vchan_cookie_complete(vd); 567 chan->request = EDMA_REQ_NONE; 568 chan->status = EDMA_ST_IDLE; 569 break; 570 571 case EDMA_REQ_PAUSE: 572 chan->request = EDMA_REQ_NONE; 573 chan->status = EDMA_ST_PAUSE; 574 break; 575 576 default: 577 break; 578 } 579 } 580 spin_unlock_irqrestore(&chan->vc.lock, flags); 581 } 582 583 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 584 { 585 struct virt_dma_desc *vd; 586 unsigned long flags; 587 588 dw_edma_v0_core_clear_abort_int(chan); 589 590 spin_lock_irqsave(&chan->vc.lock, flags); 591 vd = vchan_next_desc(&chan->vc); 592 if (vd) { 593 list_del(&vd->node); 594 vchan_cookie_complete(vd); 595 } 596 spin_unlock_irqrestore(&chan->vc.lock, flags); 597 chan->request = EDMA_REQ_NONE; 598 chan->status = EDMA_ST_IDLE; 599 } 600 601 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) 602 { 603 struct dw_edma_irq *dw_irq = data; 604 struct dw_edma *dw = dw_irq->dw; 605 unsigned long total, pos, val; 606 unsigned long off; 607 u32 mask; 608 609 if (write) { 610 total = dw->wr_ch_cnt; 611 off = 0; 612 mask = dw_irq->wr_mask; 613 } else { 614 total = dw->rd_ch_cnt; 615 off = dw->wr_ch_cnt; 616 mask = dw_irq->rd_mask; 617 } 618 619 val = dw_edma_v0_core_status_done_int(dw, write ? 620 EDMA_DIR_WRITE : 621 EDMA_DIR_READ); 622 val &= mask; 623 for_each_set_bit(pos, &val, total) { 624 struct dw_edma_chan *chan = &dw->chan[pos + off]; 625 626 dw_edma_done_interrupt(chan); 627 } 628 629 val = dw_edma_v0_core_status_abort_int(dw, write ? 630 EDMA_DIR_WRITE : 631 EDMA_DIR_READ); 632 val &= mask; 633 for_each_set_bit(pos, &val, total) { 634 struct dw_edma_chan *chan = &dw->chan[pos + off]; 635 636 dw_edma_abort_interrupt(chan); 637 } 638 639 return IRQ_HANDLED; 640 } 641 642 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 643 { 644 return dw_edma_interrupt(irq, data, true); 645 } 646 647 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 648 { 649 return dw_edma_interrupt(irq, data, false); 650 } 651 652 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 653 { 654 dw_edma_interrupt(irq, data, true); 655 dw_edma_interrupt(irq, data, false); 656 657 return IRQ_HANDLED; 658 } 659 660 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 661 { 662 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 663 664 if (chan->status != EDMA_ST_IDLE) 665 return -EBUSY; 666 667 pm_runtime_get(chan->chip->dev); 668 669 return 0; 670 } 671 672 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 673 { 674 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 675 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 676 int ret; 677 678 while (time_before(jiffies, timeout)) { 679 ret = dw_edma_device_terminate_all(dchan); 680 if (!ret) 681 break; 682 683 if (time_after_eq(jiffies, timeout)) 684 return; 685 686 cpu_relax(); 687 } 688 689 pm_runtime_put(chan->chip->dev); 690 } 691 692 static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, 693 u32 wr_alloc, u32 rd_alloc) 694 { 695 struct dw_edma_region *dt_region; 696 struct device *dev = chip->dev; 697 struct dw_edma *dw = chip->dw; 698 struct dw_edma_chan *chan; 699 struct dw_edma_irq *irq; 700 struct dma_device *dma; 701 u32 alloc, off_alloc; 702 u32 i, j, cnt; 703 int err = 0; 704 u32 pos; 705 706 if (write) { 707 i = 0; 708 cnt = dw->wr_ch_cnt; 709 dma = &dw->wr_edma; 710 alloc = wr_alloc; 711 off_alloc = 0; 712 } else { 713 i = dw->wr_ch_cnt; 714 cnt = dw->rd_ch_cnt; 715 dma = &dw->rd_edma; 716 alloc = rd_alloc; 717 off_alloc = wr_alloc; 718 } 719 720 INIT_LIST_HEAD(&dma->channels); 721 for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) { 722 chan = &dw->chan[i]; 723 724 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL); 725 if (!dt_region) 726 return -ENOMEM; 727 728 chan->vc.chan.private = dt_region; 729 730 chan->chip = chip; 731 chan->id = j; 732 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; 733 chan->configured = false; 734 chan->request = EDMA_REQ_NONE; 735 chan->status = EDMA_ST_IDLE; 736 737 if (write) 738 chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ); 739 else 740 chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ); 741 chan->ll_max -= 1; 742 743 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 744 write ? "write" : "read", j, chan->ll_max); 745 746 if (dw->nr_irqs == 1) 747 pos = 0; 748 else 749 pos = off_alloc + (j % alloc); 750 751 irq = &dw->irq[pos]; 752 753 if (write) 754 irq->wr_mask |= BIT(j); 755 else 756 irq->rd_mask |= BIT(j); 757 758 irq->dw = dw; 759 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 760 761 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 762 write ? "write" : "read", j, 763 chan->msi.address_hi, chan->msi.address_lo, 764 chan->msi.data); 765 766 chan->vc.desc_free = vchan_free_desc; 767 vchan_init(&chan->vc, dma); 768 769 if (write) { 770 dt_region->paddr = dw->dt_region_wr[j].paddr; 771 dt_region->vaddr = dw->dt_region_wr[j].vaddr; 772 dt_region->sz = dw->dt_region_wr[j].sz; 773 } else { 774 dt_region->paddr = dw->dt_region_rd[j].paddr; 775 dt_region->vaddr = dw->dt_region_rd[j].vaddr; 776 dt_region->sz = dw->dt_region_rd[j].sz; 777 } 778 779 dw_edma_v0_core_device_config(chan); 780 } 781 782 /* Set DMA channel capabilities */ 783 dma_cap_zero(dma->cap_mask); 784 dma_cap_set(DMA_SLAVE, dma->cap_mask); 785 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 786 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 787 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 788 dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV); 789 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 790 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 791 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 792 dma->chancnt = cnt; 793 794 /* Set DMA channel callbacks */ 795 dma->dev = chip->dev; 796 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 797 dma->device_free_chan_resources = dw_edma_free_chan_resources; 798 dma->device_config = dw_edma_device_config; 799 dma->device_pause = dw_edma_device_pause; 800 dma->device_resume = dw_edma_device_resume; 801 dma->device_terminate_all = dw_edma_device_terminate_all; 802 dma->device_issue_pending = dw_edma_device_issue_pending; 803 dma->device_tx_status = dw_edma_device_tx_status; 804 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 805 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 806 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 807 808 dma_set_max_seg_size(dma->dev, U32_MAX); 809 810 /* Register DMA device */ 811 err = dma_async_device_register(dma); 812 813 return err; 814 } 815 816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 817 { 818 if (*nr_irqs && *alloc < cnt) { 819 (*alloc)++; 820 (*nr_irqs)--; 821 } 822 } 823 824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 825 { 826 while (*mask * alloc < cnt) 827 (*mask)++; 828 } 829 830 static int dw_edma_irq_request(struct dw_edma_chip *chip, 831 u32 *wr_alloc, u32 *rd_alloc) 832 { 833 struct device *dev = chip->dev; 834 struct dw_edma *dw = chip->dw; 835 u32 wr_mask = 1; 836 u32 rd_mask = 1; 837 int i, err = 0; 838 u32 ch_cnt; 839 int irq; 840 841 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 842 843 if (dw->nr_irqs < 1) 844 return -EINVAL; 845 846 if (dw->nr_irqs == 1) { 847 /* Common IRQ shared among all channels */ 848 irq = dw->ops->irq_vector(dev, 0); 849 err = request_irq(irq, dw_edma_interrupt_common, 850 IRQF_SHARED, dw->name, &dw->irq[0]); 851 if (err) { 852 dw->nr_irqs = 0; 853 return err; 854 } 855 856 if (irq_get_msi_desc(irq)) 857 get_cached_msi_msg(irq, &dw->irq[0].msi); 858 } else { 859 /* Distribute IRQs equally among all channels */ 860 int tmp = dw->nr_irqs; 861 862 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 863 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 864 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 865 } 866 867 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 868 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 869 870 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 871 irq = dw->ops->irq_vector(dev, i); 872 err = request_irq(irq, 873 i < *wr_alloc ? 874 dw_edma_interrupt_write : 875 dw_edma_interrupt_read, 876 IRQF_SHARED, dw->name, 877 &dw->irq[i]); 878 if (err) { 879 dw->nr_irqs = i; 880 return err; 881 } 882 883 if (irq_get_msi_desc(irq)) 884 get_cached_msi_msg(irq, &dw->irq[i].msi); 885 } 886 887 dw->nr_irqs = i; 888 } 889 890 return err; 891 } 892 893 int dw_edma_probe(struct dw_edma_chip *chip) 894 { 895 struct device *dev; 896 struct dw_edma *dw; 897 u32 wr_alloc = 0; 898 u32 rd_alloc = 0; 899 int i, err; 900 901 if (!chip) 902 return -EINVAL; 903 904 dev = chip->dev; 905 if (!dev) 906 return -EINVAL; 907 908 dw = chip->dw; 909 if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector) 910 return -EINVAL; 911 912 raw_spin_lock_init(&dw->lock); 913 914 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, 915 dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); 916 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 917 918 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, 919 dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); 920 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 921 922 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 923 return -EINVAL; 924 925 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 926 dw->wr_ch_cnt, dw->rd_ch_cnt); 927 928 /* Allocate channels */ 929 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 930 sizeof(*dw->chan), GFP_KERNEL); 931 if (!dw->chan) 932 return -ENOMEM; 933 934 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id); 935 936 /* Disable eDMA, only to establish the ideal initial conditions */ 937 dw_edma_v0_core_off(dw); 938 939 /* Request IRQs */ 940 err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); 941 if (err) 942 return err; 943 944 /* Setup write channels */ 945 err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); 946 if (err) 947 goto err_irq_free; 948 949 /* Setup read channels */ 950 err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); 951 if (err) 952 goto err_irq_free; 953 954 /* Power management */ 955 pm_runtime_enable(dev); 956 957 /* Turn debugfs on */ 958 dw_edma_v0_core_debugfs_on(chip); 959 960 return 0; 961 962 err_irq_free: 963 for (i = (dw->nr_irqs - 1); i >= 0; i--) 964 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); 965 966 dw->nr_irqs = 0; 967 968 return err; 969 } 970 EXPORT_SYMBOL_GPL(dw_edma_probe); 971 972 int dw_edma_remove(struct dw_edma_chip *chip) 973 { 974 struct dw_edma_chan *chan, *_chan; 975 struct device *dev = chip->dev; 976 struct dw_edma *dw = chip->dw; 977 int i; 978 979 /* Disable eDMA */ 980 dw_edma_v0_core_off(dw); 981 982 /* Free irqs */ 983 for (i = (dw->nr_irqs - 1); i >= 0; i--) 984 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); 985 986 /* Power management */ 987 pm_runtime_disable(dev); 988 989 /* Deregister eDMA device */ 990 dma_async_device_unregister(&dw->wr_edma); 991 list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, 992 vc.chan.device_node) { 993 tasklet_kill(&chan->vc.task); 994 list_del(&chan->vc.chan.device_node); 995 } 996 997 dma_async_device_unregister(&dw->rd_edma); 998 list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, 999 vc.chan.device_node) { 1000 tasklet_kill(&chan->vc.task); 1001 list_del(&chan->vc.chan.device_node); 1002 } 1003 1004 /* Turn debugfs off */ 1005 dw_edma_v0_core_debugfs_off(chip); 1006 1007 return 0; 1008 } 1009 EXPORT_SYMBOL_GPL(dw_edma_remove); 1010 1011 MODULE_LICENSE("GPL v2"); 1012 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1013 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1014