1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/dmaengine.h> 13 #include <linux/err.h> 14 #include <linux/interrupt.h> 15 #include <linux/irq.h> 16 #include <linux/dma/edma.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/string_choices.h> 19 20 #include "dw-edma-core.h" 21 #include "dw-edma-v0-core.h" 22 #include "dw-hdma-v0-core.h" 23 #include "../dmaengine.h" 24 #include "../virt-dma.h" 25 26 static inline 27 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 28 { 29 return container_of(vd, struct dw_edma_desc, vd); 30 } 31 32 static inline 33 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) 34 { 35 struct dw_edma_chip *chip = chan->dw->chip; 36 37 if (chip->ops->pci_address) 38 return chip->ops->pci_address(chip->dev, cpu_addr); 39 40 return cpu_addr; 41 } 42 43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 44 { 45 struct dw_edma_burst *burst; 46 47 burst = kzalloc(sizeof(*burst), GFP_NOWAIT); 48 if (unlikely(!burst)) 49 return NULL; 50 51 INIT_LIST_HEAD(&burst->list); 52 if (chunk->burst) { 53 /* Create and add new element into the linked list */ 54 chunk->bursts_alloc++; 55 list_add_tail(&burst->list, &chunk->burst->list); 56 } else { 57 /* List head */ 58 chunk->bursts_alloc = 0; 59 chunk->burst = burst; 60 } 61 62 return burst; 63 } 64 65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 66 { 67 struct dw_edma_chip *chip = desc->chan->dw->chip; 68 struct dw_edma_chan *chan = desc->chan; 69 struct dw_edma_chunk *chunk; 70 71 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); 72 if (unlikely(!chunk)) 73 return NULL; 74 75 INIT_LIST_HEAD(&chunk->list); 76 chunk->chan = chan; 77 /* Toggling change bit (CB) in each chunk, this is a mechanism to 78 * inform the eDMA HW block that this is a new linked list ready 79 * to be consumed. 80 * - Odd chunks originate CB equal to 0 81 * - Even chunks originate CB equal to 1 82 */ 83 chunk->cb = !(desc->chunks_alloc % 2); 84 if (chan->dir == EDMA_DIR_WRITE) { 85 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; 86 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; 87 } else { 88 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; 89 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; 90 } 91 92 if (desc->chunk) { 93 /* Create and add new element into the linked list */ 94 if (!dw_edma_alloc_burst(chunk)) { 95 kfree(chunk); 96 return NULL; 97 } 98 desc->chunks_alloc++; 99 list_add_tail(&chunk->list, &desc->chunk->list); 100 } else { 101 /* List head */ 102 chunk->burst = NULL; 103 desc->chunks_alloc = 0; 104 desc->chunk = chunk; 105 } 106 107 return chunk; 108 } 109 110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 111 { 112 struct dw_edma_desc *desc; 113 114 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 115 if (unlikely(!desc)) 116 return NULL; 117 118 desc->chan = chan; 119 if (!dw_edma_alloc_chunk(desc)) { 120 kfree(desc); 121 return NULL; 122 } 123 124 return desc; 125 } 126 127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 128 { 129 struct dw_edma_burst *child, *_next; 130 131 /* Remove all the list elements */ 132 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 133 list_del(&child->list); 134 kfree(child); 135 chunk->bursts_alloc--; 136 } 137 138 /* Remove the list head */ 139 kfree(child); 140 chunk->burst = NULL; 141 } 142 143 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 144 { 145 struct dw_edma_chunk *child, *_next; 146 147 if (!desc->chunk) 148 return; 149 150 /* Remove all the list elements */ 151 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 152 dw_edma_free_burst(child); 153 list_del(&child->list); 154 kfree(child); 155 desc->chunks_alloc--; 156 } 157 158 /* Remove the list head */ 159 kfree(child); 160 desc->chunk = NULL; 161 } 162 163 static void dw_edma_free_desc(struct dw_edma_desc *desc) 164 { 165 dw_edma_free_chunk(desc); 166 kfree(desc); 167 } 168 169 static void vchan_free_desc(struct virt_dma_desc *vdesc) 170 { 171 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 172 } 173 174 static int dw_edma_start_transfer(struct dw_edma_chan *chan) 175 { 176 struct dw_edma *dw = chan->dw; 177 struct dw_edma_chunk *child; 178 struct dw_edma_desc *desc; 179 struct virt_dma_desc *vd; 180 181 vd = vchan_next_desc(&chan->vc); 182 if (!vd) 183 return 0; 184 185 desc = vd2dw_edma_desc(vd); 186 if (!desc) 187 return 0; 188 189 child = list_first_entry_or_null(&desc->chunk->list, 190 struct dw_edma_chunk, list); 191 if (!child) 192 return 0; 193 194 dw_edma_core_start(dw, child, !desc->xfer_sz); 195 desc->xfer_sz += child->ll_region.sz; 196 dw_edma_free_burst(child); 197 list_del(&child->list); 198 kfree(child); 199 desc->chunks_alloc--; 200 201 return 1; 202 } 203 204 static void dw_edma_device_caps(struct dma_chan *dchan, 205 struct dma_slave_caps *caps) 206 { 207 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 208 209 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 210 if (chan->dir == EDMA_DIR_READ) 211 caps->directions = BIT(DMA_DEV_TO_MEM); 212 else 213 caps->directions = BIT(DMA_MEM_TO_DEV); 214 } else { 215 if (chan->dir == EDMA_DIR_WRITE) 216 caps->directions = BIT(DMA_DEV_TO_MEM); 217 else 218 caps->directions = BIT(DMA_MEM_TO_DEV); 219 } 220 } 221 222 static int dw_edma_device_config(struct dma_chan *dchan, 223 struct dma_slave_config *config) 224 { 225 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 226 227 memcpy(&chan->config, config, sizeof(*config)); 228 chan->configured = true; 229 230 return 0; 231 } 232 233 static int dw_edma_device_pause(struct dma_chan *dchan) 234 { 235 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 236 int err = 0; 237 238 if (!chan->configured) 239 err = -EPERM; 240 else if (chan->status != EDMA_ST_BUSY) 241 err = -EPERM; 242 else if (chan->request != EDMA_REQ_NONE) 243 err = -EPERM; 244 else 245 chan->request = EDMA_REQ_PAUSE; 246 247 return err; 248 } 249 250 static int dw_edma_device_resume(struct dma_chan *dchan) 251 { 252 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 253 int err = 0; 254 255 if (!chan->configured) { 256 err = -EPERM; 257 } else if (chan->status != EDMA_ST_PAUSE) { 258 err = -EPERM; 259 } else if (chan->request != EDMA_REQ_NONE) { 260 err = -EPERM; 261 } else { 262 chan->status = EDMA_ST_BUSY; 263 dw_edma_start_transfer(chan); 264 } 265 266 return err; 267 } 268 269 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 270 { 271 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 272 int err = 0; 273 274 if (!chan->configured) { 275 /* Do nothing */ 276 } else if (chan->status == EDMA_ST_PAUSE) { 277 chan->status = EDMA_ST_IDLE; 278 chan->configured = false; 279 } else if (chan->status == EDMA_ST_IDLE) { 280 chan->configured = false; 281 } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { 282 /* 283 * The channel is in a false BUSY state, probably didn't 284 * receive or lost an interrupt 285 */ 286 chan->status = EDMA_ST_IDLE; 287 chan->configured = false; 288 } else if (chan->request > EDMA_REQ_PAUSE) { 289 err = -EPERM; 290 } else { 291 chan->request = EDMA_REQ_STOP; 292 } 293 294 return err; 295 } 296 297 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 298 { 299 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 300 unsigned long flags; 301 302 if (!chan->configured) 303 return; 304 305 spin_lock_irqsave(&chan->vc.lock, flags); 306 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && 307 chan->status == EDMA_ST_IDLE) { 308 chan->status = EDMA_ST_BUSY; 309 dw_edma_start_transfer(chan); 310 } 311 spin_unlock_irqrestore(&chan->vc.lock, flags); 312 } 313 314 static enum dma_status 315 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 316 struct dma_tx_state *txstate) 317 { 318 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 319 struct dw_edma_desc *desc; 320 struct virt_dma_desc *vd; 321 unsigned long flags; 322 enum dma_status ret; 323 u32 residue = 0; 324 325 ret = dma_cookie_status(dchan, cookie, txstate); 326 if (ret == DMA_COMPLETE) 327 return ret; 328 329 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 330 ret = DMA_PAUSED; 331 332 if (!txstate) 333 goto ret_residue; 334 335 spin_lock_irqsave(&chan->vc.lock, flags); 336 vd = vchan_find_desc(&chan->vc, cookie); 337 if (vd) { 338 desc = vd2dw_edma_desc(vd); 339 if (desc) 340 residue = desc->alloc_sz - desc->xfer_sz; 341 } 342 spin_unlock_irqrestore(&chan->vc.lock, flags); 343 344 ret_residue: 345 dma_set_residue(txstate, residue); 346 347 return ret; 348 } 349 350 static struct dma_async_tx_descriptor * 351 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 352 { 353 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 354 enum dma_transfer_direction dir = xfer->direction; 355 struct scatterlist *sg = NULL; 356 struct dw_edma_chunk *chunk; 357 struct dw_edma_burst *burst; 358 struct dw_edma_desc *desc; 359 u64 src_addr, dst_addr; 360 size_t fsz = 0; 361 u32 cnt = 0; 362 int i; 363 364 if (!chan->configured) 365 return NULL; 366 367 /* 368 * Local Root Port/End-point Remote End-point 369 * +-----------------------+ PCIe bus +----------------------+ 370 * | | +-+ | | 371 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | 372 * | | | | | | 373 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | 374 * | | +-+ | | 375 * +-----------------------+ +----------------------+ 376 * 377 * 1. Normal logic: 378 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the 379 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used 380 * for the device read operations (DEV_TO_MEM) and the Tx channel 381 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). 382 * 383 * 2. Inverted logic: 384 * If eDMA is embedded into a Remote PCIe EP and is controlled by the 385 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx 386 * channel (EDMA_DIR_WRITE) will be used for the device read operations 387 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write 388 * operations (MEM_TO_DEV). 389 * 390 * It is the client driver responsibility to choose a proper channel 391 * for the DMA transfers. 392 */ 393 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 394 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || 395 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) 396 return NULL; 397 } else { 398 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || 399 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) 400 return NULL; 401 } 402 403 if (xfer->type == EDMA_XFER_CYCLIC) { 404 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 405 return NULL; 406 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 407 if (xfer->xfer.sg.len < 1) 408 return NULL; 409 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 410 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) 411 return NULL; 412 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) 413 return NULL; 414 } else { 415 return NULL; 416 } 417 418 desc = dw_edma_alloc_desc(chan); 419 if (unlikely(!desc)) 420 goto err_alloc; 421 422 chunk = dw_edma_alloc_chunk(desc); 423 if (unlikely(!chunk)) 424 goto err_alloc; 425 426 if (xfer->type == EDMA_XFER_INTERLEAVED) { 427 src_addr = xfer->xfer.il->src_start; 428 dst_addr = xfer->xfer.il->dst_start; 429 } else { 430 src_addr = chan->config.src_addr; 431 dst_addr = chan->config.dst_addr; 432 } 433 434 if (dir == DMA_DEV_TO_MEM) 435 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr); 436 else 437 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr); 438 439 if (xfer->type == EDMA_XFER_CYCLIC) { 440 cnt = xfer->xfer.cyclic.cnt; 441 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 442 cnt = xfer->xfer.sg.len; 443 sg = xfer->xfer.sg.sgl; 444 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 445 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; 446 fsz = xfer->xfer.il->frame_size; 447 } 448 449 for (i = 0; i < cnt; i++) { 450 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 451 break; 452 453 if (chunk->bursts_alloc == chan->ll_max) { 454 chunk = dw_edma_alloc_chunk(desc); 455 if (unlikely(!chunk)) 456 goto err_alloc; 457 } 458 459 burst = dw_edma_alloc_burst(chunk); 460 if (unlikely(!burst)) 461 goto err_alloc; 462 463 if (xfer->type == EDMA_XFER_CYCLIC) 464 burst->sz = xfer->xfer.cyclic.len; 465 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 466 burst->sz = sg_dma_len(sg); 467 else if (xfer->type == EDMA_XFER_INTERLEAVED) 468 burst->sz = xfer->xfer.il->sgl[i % fsz].size; 469 470 chunk->ll_region.sz += burst->sz; 471 desc->alloc_sz += burst->sz; 472 473 if (dir == DMA_DEV_TO_MEM) { 474 burst->sar = src_addr; 475 if (xfer->type == EDMA_XFER_CYCLIC) { 476 burst->dar = xfer->xfer.cyclic.paddr; 477 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 478 src_addr += sg_dma_len(sg); 479 burst->dar = sg_dma_address(sg); 480 /* Unlike the typical assumption by other 481 * drivers/IPs the peripheral memory isn't 482 * a FIFO memory, in this case, it's a 483 * linear memory and that why the source 484 * and destination addresses are increased 485 * by the same portion (data length) 486 */ 487 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 488 burst->dar = dst_addr; 489 } 490 } else { 491 burst->dar = dst_addr; 492 if (xfer->type == EDMA_XFER_CYCLIC) { 493 burst->sar = xfer->xfer.cyclic.paddr; 494 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 495 dst_addr += sg_dma_len(sg); 496 burst->sar = sg_dma_address(sg); 497 /* Unlike the typical assumption by other 498 * drivers/IPs the peripheral memory isn't 499 * a FIFO memory, in this case, it's a 500 * linear memory and that why the source 501 * and destination addresses are increased 502 * by the same portion (data length) 503 */ 504 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 505 burst->sar = src_addr; 506 } 507 } 508 509 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 510 sg = sg_next(sg); 511 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 512 struct dma_interleaved_template *il = xfer->xfer.il; 513 struct data_chunk *dc = &il->sgl[i % fsz]; 514 515 src_addr += burst->sz; 516 if (il->src_sgl) 517 src_addr += dmaengine_get_src_icg(il, dc); 518 519 dst_addr += burst->sz; 520 if (il->dst_sgl) 521 dst_addr += dmaengine_get_dst_icg(il, dc); 522 } 523 } 524 525 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 526 527 err_alloc: 528 if (desc) 529 dw_edma_free_desc(desc); 530 531 return NULL; 532 } 533 534 static struct dma_async_tx_descriptor * 535 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 536 unsigned int len, 537 enum dma_transfer_direction direction, 538 unsigned long flags, void *context) 539 { 540 struct dw_edma_transfer xfer; 541 542 xfer.dchan = dchan; 543 xfer.direction = direction; 544 xfer.xfer.sg.sgl = sgl; 545 xfer.xfer.sg.len = len; 546 xfer.flags = flags; 547 xfer.type = EDMA_XFER_SCATTER_GATHER; 548 549 return dw_edma_device_transfer(&xfer); 550 } 551 552 static struct dma_async_tx_descriptor * 553 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 554 size_t len, size_t count, 555 enum dma_transfer_direction direction, 556 unsigned long flags) 557 { 558 struct dw_edma_transfer xfer; 559 560 xfer.dchan = dchan; 561 xfer.direction = direction; 562 xfer.xfer.cyclic.paddr = paddr; 563 xfer.xfer.cyclic.len = len; 564 xfer.xfer.cyclic.cnt = count; 565 xfer.flags = flags; 566 xfer.type = EDMA_XFER_CYCLIC; 567 568 return dw_edma_device_transfer(&xfer); 569 } 570 571 static struct dma_async_tx_descriptor * 572 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 573 struct dma_interleaved_template *ilt, 574 unsigned long flags) 575 { 576 struct dw_edma_transfer xfer; 577 578 xfer.dchan = dchan; 579 xfer.direction = ilt->dir; 580 xfer.xfer.il = ilt; 581 xfer.flags = flags; 582 xfer.type = EDMA_XFER_INTERLEAVED; 583 584 return dw_edma_device_transfer(&xfer); 585 } 586 587 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, 588 enum dmaengine_tx_result result) 589 { 590 u32 residue = 0; 591 struct dw_edma_desc *desc; 592 struct dmaengine_result *res; 593 594 if (!vd->tx.callback_result) 595 return; 596 597 desc = vd2dw_edma_desc(vd); 598 if (desc) 599 residue = desc->alloc_sz - desc->xfer_sz; 600 601 res = &vd->tx_result; 602 res->result = result; 603 res->residue = residue; 604 } 605 606 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 607 { 608 struct dw_edma_desc *desc; 609 struct virt_dma_desc *vd; 610 unsigned long flags; 611 612 spin_lock_irqsave(&chan->vc.lock, flags); 613 vd = vchan_next_desc(&chan->vc); 614 if (vd) { 615 switch (chan->request) { 616 case EDMA_REQ_NONE: 617 desc = vd2dw_edma_desc(vd); 618 if (!desc->chunks_alloc) { 619 dw_hdma_set_callback_result(vd, 620 DMA_TRANS_NOERROR); 621 list_del(&vd->node); 622 vchan_cookie_complete(vd); 623 } 624 625 /* Continue transferring if there are remaining chunks or issued requests. 626 */ 627 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; 628 break; 629 630 case EDMA_REQ_STOP: 631 list_del(&vd->node); 632 vchan_cookie_complete(vd); 633 chan->request = EDMA_REQ_NONE; 634 chan->status = EDMA_ST_IDLE; 635 break; 636 637 case EDMA_REQ_PAUSE: 638 chan->request = EDMA_REQ_NONE; 639 chan->status = EDMA_ST_PAUSE; 640 break; 641 642 default: 643 break; 644 } 645 } 646 spin_unlock_irqrestore(&chan->vc.lock, flags); 647 } 648 649 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 650 { 651 struct virt_dma_desc *vd; 652 unsigned long flags; 653 654 spin_lock_irqsave(&chan->vc.lock, flags); 655 vd = vchan_next_desc(&chan->vc); 656 if (vd) { 657 dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); 658 list_del(&vd->node); 659 vchan_cookie_complete(vd); 660 } 661 spin_unlock_irqrestore(&chan->vc.lock, flags); 662 chan->request = EDMA_REQ_NONE; 663 chan->status = EDMA_ST_IDLE; 664 } 665 666 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 667 { 668 struct dw_edma_irq *dw_irq = data; 669 670 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, 671 dw_edma_done_interrupt, 672 dw_edma_abort_interrupt); 673 } 674 675 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 676 { 677 struct dw_edma_irq *dw_irq = data; 678 679 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, 680 dw_edma_done_interrupt, 681 dw_edma_abort_interrupt); 682 } 683 684 static irqreturn_t dw_edma_interrupt_common(int irq, void *data) 685 { 686 irqreturn_t ret = IRQ_NONE; 687 688 ret |= dw_edma_interrupt_write(irq, data); 689 ret |= dw_edma_interrupt_read(irq, data); 690 691 return ret; 692 } 693 694 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 695 { 696 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 697 698 if (chan->status != EDMA_ST_IDLE) 699 return -EBUSY; 700 701 return 0; 702 } 703 704 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 705 { 706 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 707 int ret; 708 709 while (time_before(jiffies, timeout)) { 710 ret = dw_edma_device_terminate_all(dchan); 711 if (!ret) 712 break; 713 714 if (time_after_eq(jiffies, timeout)) 715 return; 716 717 cpu_relax(); 718 } 719 } 720 721 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) 722 { 723 struct dw_edma_chip *chip = dw->chip; 724 struct device *dev = chip->dev; 725 struct dw_edma_chan *chan; 726 struct dw_edma_irq *irq; 727 struct dma_device *dma; 728 u32 i, ch_cnt; 729 u32 pos; 730 731 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 732 dma = &dw->dma; 733 734 INIT_LIST_HEAD(&dma->channels); 735 736 for (i = 0; i < ch_cnt; i++) { 737 chan = &dw->chan[i]; 738 739 chan->dw = dw; 740 741 if (i < dw->wr_ch_cnt) { 742 chan->id = i; 743 chan->dir = EDMA_DIR_WRITE; 744 } else { 745 chan->id = i - dw->wr_ch_cnt; 746 chan->dir = EDMA_DIR_READ; 747 } 748 749 chan->configured = false; 750 chan->request = EDMA_REQ_NONE; 751 chan->status = EDMA_ST_IDLE; 752 753 if (chan->dir == EDMA_DIR_WRITE) 754 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); 755 else 756 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); 757 chan->ll_max -= 1; 758 759 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 760 str_write_read(chan->dir == EDMA_DIR_WRITE), 761 chan->id, chan->ll_max); 762 763 if (dw->nr_irqs == 1) 764 pos = 0; 765 else if (chan->dir == EDMA_DIR_WRITE) 766 pos = chan->id % wr_alloc; 767 else 768 pos = wr_alloc + chan->id % rd_alloc; 769 770 irq = &dw->irq[pos]; 771 772 if (chan->dir == EDMA_DIR_WRITE) 773 irq->wr_mask |= BIT(chan->id); 774 else 775 irq->rd_mask |= BIT(chan->id); 776 777 irq->dw = dw; 778 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 779 780 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 781 str_write_read(chan->dir == EDMA_DIR_WRITE), 782 chan->id, 783 chan->msi.address_hi, chan->msi.address_lo, 784 chan->msi.data); 785 786 chan->vc.desc_free = vchan_free_desc; 787 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? 788 &dw->chip->dt_region_wr[chan->id] : 789 &dw->chip->dt_region_rd[chan->id]; 790 791 vchan_init(&chan->vc, dma); 792 793 dw_edma_core_ch_config(chan); 794 } 795 796 /* Set DMA channel capabilities */ 797 dma_cap_zero(dma->cap_mask); 798 dma_cap_set(DMA_SLAVE, dma->cap_mask); 799 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 800 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 801 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 802 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 803 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 804 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 805 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 806 807 /* Set DMA channel callbacks */ 808 dma->dev = chip->dev; 809 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 810 dma->device_free_chan_resources = dw_edma_free_chan_resources; 811 dma->device_caps = dw_edma_device_caps; 812 dma->device_config = dw_edma_device_config; 813 dma->device_pause = dw_edma_device_pause; 814 dma->device_resume = dw_edma_device_resume; 815 dma->device_terminate_all = dw_edma_device_terminate_all; 816 dma->device_issue_pending = dw_edma_device_issue_pending; 817 dma->device_tx_status = dw_edma_device_tx_status; 818 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 819 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 820 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 821 822 dma_set_max_seg_size(dma->dev, U32_MAX); 823 824 /* Register DMA device */ 825 return dma_async_device_register(dma); 826 } 827 828 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 829 { 830 if (*nr_irqs && *alloc < cnt) { 831 (*alloc)++; 832 (*nr_irqs)--; 833 } 834 } 835 836 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 837 { 838 while (*mask * alloc < cnt) 839 (*mask)++; 840 } 841 842 static int dw_edma_irq_request(struct dw_edma *dw, 843 u32 *wr_alloc, u32 *rd_alloc) 844 { 845 struct dw_edma_chip *chip = dw->chip; 846 struct device *dev = dw->chip->dev; 847 u32 wr_mask = 1; 848 u32 rd_mask = 1; 849 int i, err = 0; 850 u32 ch_cnt; 851 int irq; 852 853 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 854 855 if (chip->nr_irqs < 1 || !chip->ops->irq_vector) 856 return -EINVAL; 857 858 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); 859 if (!dw->irq) 860 return -ENOMEM; 861 862 if (chip->nr_irqs == 1) { 863 /* Common IRQ shared among all channels */ 864 irq = chip->ops->irq_vector(dev, 0); 865 err = request_irq(irq, dw_edma_interrupt_common, 866 IRQF_SHARED, dw->name, &dw->irq[0]); 867 if (err) { 868 dw->nr_irqs = 0; 869 return err; 870 } 871 872 if (irq_get_msi_desc(irq)) 873 get_cached_msi_msg(irq, &dw->irq[0].msi); 874 875 dw->nr_irqs = 1; 876 } else { 877 /* Distribute IRQs equally among all channels */ 878 int tmp = chip->nr_irqs; 879 880 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 881 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 882 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 883 } 884 885 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 886 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 887 888 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 889 irq = chip->ops->irq_vector(dev, i); 890 err = request_irq(irq, 891 i < *wr_alloc ? 892 dw_edma_interrupt_write : 893 dw_edma_interrupt_read, 894 IRQF_SHARED, dw->name, 895 &dw->irq[i]); 896 if (err) 897 goto err_irq_free; 898 899 if (irq_get_msi_desc(irq)) 900 get_cached_msi_msg(irq, &dw->irq[i].msi); 901 } 902 903 dw->nr_irqs = i; 904 } 905 906 return 0; 907 908 err_irq_free: 909 for (i--; i >= 0; i--) { 910 irq = chip->ops->irq_vector(dev, i); 911 free_irq(irq, &dw->irq[i]); 912 } 913 914 return err; 915 } 916 917 int dw_edma_probe(struct dw_edma_chip *chip) 918 { 919 struct device *dev; 920 struct dw_edma *dw; 921 u32 wr_alloc = 0; 922 u32 rd_alloc = 0; 923 int i, err; 924 925 if (!chip) 926 return -EINVAL; 927 928 dev = chip->dev; 929 if (!dev || !chip->ops) 930 return -EINVAL; 931 932 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); 933 if (!dw) 934 return -ENOMEM; 935 936 dw->chip = chip; 937 938 if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) 939 dw_hdma_v0_core_register(dw); 940 else 941 dw_edma_v0_core_register(dw); 942 943 raw_spin_lock_init(&dw->lock); 944 945 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, 946 dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); 947 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 948 949 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, 950 dw_edma_core_ch_count(dw, EDMA_DIR_READ)); 951 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 952 953 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 954 return -EINVAL; 955 956 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 957 dw->wr_ch_cnt, dw->rd_ch_cnt); 958 959 /* Allocate channels */ 960 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 961 sizeof(*dw->chan), GFP_KERNEL); 962 if (!dw->chan) 963 return -ENOMEM; 964 965 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", 966 dev_name(chip->dev)); 967 968 /* Disable eDMA, only to establish the ideal initial conditions */ 969 dw_edma_core_off(dw); 970 971 /* Request IRQs */ 972 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); 973 if (err) 974 return err; 975 976 /* Setup write/read channels */ 977 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); 978 if (err) 979 goto err_irq_free; 980 981 /* Turn debugfs on */ 982 dw_edma_core_debugfs_on(dw); 983 984 chip->dw = dw; 985 986 return 0; 987 988 err_irq_free: 989 for (i = (dw->nr_irqs - 1); i >= 0; i--) 990 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 991 992 return err; 993 } 994 EXPORT_SYMBOL_GPL(dw_edma_probe); 995 996 int dw_edma_remove(struct dw_edma_chip *chip) 997 { 998 struct dw_edma_chan *chan, *_chan; 999 struct device *dev = chip->dev; 1000 struct dw_edma *dw = chip->dw; 1001 int i; 1002 1003 /* Skip removal if no private data found */ 1004 if (!dw) 1005 return -ENODEV; 1006 1007 /* Disable eDMA */ 1008 dw_edma_core_off(dw); 1009 1010 /* Free irqs */ 1011 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1012 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1013 1014 /* Deregister eDMA device */ 1015 dma_async_device_unregister(&dw->dma); 1016 list_for_each_entry_safe(chan, _chan, &dw->dma.channels, 1017 vc.chan.device_node) { 1018 tasklet_kill(&chan->vc.task); 1019 list_del(&chan->vc.chan.device_node); 1020 } 1021 1022 return 0; 1023 } 1024 EXPORT_SYMBOL_GPL(dw_edma_remove); 1025 1026 MODULE_LICENSE("GPL v2"); 1027 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1028 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1029