1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA core driver 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/module.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/dmaengine.h> 13 #include <linux/err.h> 14 #include <linux/interrupt.h> 15 #include <linux/irq.h> 16 #include <linux/dma/edma.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/string_choices.h> 19 20 #include "dw-edma-core.h" 21 #include "dw-edma-v0-core.h" 22 #include "dw-hdma-v0-core.h" 23 #include "../dmaengine.h" 24 #include "../virt-dma.h" 25 26 static inline 27 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) 28 { 29 return container_of(vd, struct dw_edma_desc, vd); 30 } 31 32 static inline 33 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) 34 { 35 struct dw_edma_chip *chip = chan->dw->chip; 36 37 if (chip->ops->pci_address) 38 return chip->ops->pci_address(chip->dev, cpu_addr); 39 40 return cpu_addr; 41 } 42 43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) 44 { 45 struct dw_edma_burst *burst; 46 47 burst = kzalloc_obj(*burst, GFP_NOWAIT); 48 if (unlikely(!burst)) 49 return NULL; 50 51 INIT_LIST_HEAD(&burst->list); 52 if (chunk->burst) { 53 /* Create and add new element into the linked list */ 54 chunk->bursts_alloc++; 55 list_add_tail(&burst->list, &chunk->burst->list); 56 } else { 57 /* List head */ 58 chunk->bursts_alloc = 0; 59 chunk->burst = burst; 60 } 61 62 return burst; 63 } 64 65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) 66 { 67 struct dw_edma_chip *chip = desc->chan->dw->chip; 68 struct dw_edma_chan *chan = desc->chan; 69 struct dw_edma_chunk *chunk; 70 71 chunk = kzalloc_obj(*chunk, GFP_NOWAIT); 72 if (unlikely(!chunk)) 73 return NULL; 74 75 INIT_LIST_HEAD(&chunk->list); 76 chunk->chan = chan; 77 /* Toggling change bit (CB) in each chunk, this is a mechanism to 78 * inform the eDMA HW block that this is a new linked list ready 79 * to be consumed. 80 * - Odd chunks originate CB equal to 0 81 * - Even chunks originate CB equal to 1 82 */ 83 chunk->cb = !(desc->chunks_alloc % 2); 84 if (chan->dir == EDMA_DIR_WRITE) { 85 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; 86 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; 87 } else { 88 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; 89 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; 90 } 91 92 if (desc->chunk) { 93 /* Create and add new element into the linked list */ 94 if (!dw_edma_alloc_burst(chunk)) { 95 kfree(chunk); 96 return NULL; 97 } 98 desc->chunks_alloc++; 99 list_add_tail(&chunk->list, &desc->chunk->list); 100 } else { 101 /* List head */ 102 chunk->burst = NULL; 103 desc->chunks_alloc = 0; 104 desc->chunk = chunk; 105 } 106 107 return chunk; 108 } 109 110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) 111 { 112 struct dw_edma_desc *desc; 113 114 desc = kzalloc_obj(*desc, GFP_NOWAIT); 115 if (unlikely(!desc)) 116 return NULL; 117 118 desc->chan = chan; 119 if (!dw_edma_alloc_chunk(desc)) { 120 kfree(desc); 121 return NULL; 122 } 123 124 return desc; 125 } 126 127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk) 128 { 129 struct dw_edma_burst *child, *_next; 130 131 /* Remove all the list elements */ 132 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { 133 list_del(&child->list); 134 kfree(child); 135 chunk->bursts_alloc--; 136 } 137 138 /* Remove the list head */ 139 kfree(child); 140 chunk->burst = NULL; 141 } 142 143 static void dw_edma_free_chunk(struct dw_edma_desc *desc) 144 { 145 struct dw_edma_chunk *child, *_next; 146 147 if (!desc->chunk) 148 return; 149 150 /* Remove all the list elements */ 151 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { 152 dw_edma_free_burst(child); 153 list_del(&child->list); 154 kfree(child); 155 desc->chunks_alloc--; 156 } 157 158 /* Remove the list head */ 159 kfree(child); 160 desc->chunk = NULL; 161 } 162 163 static void dw_edma_free_desc(struct dw_edma_desc *desc) 164 { 165 dw_edma_free_chunk(desc); 166 kfree(desc); 167 } 168 169 static void vchan_free_desc(struct virt_dma_desc *vdesc) 170 { 171 dw_edma_free_desc(vd2dw_edma_desc(vdesc)); 172 } 173 174 static int dw_edma_start_transfer(struct dw_edma_chan *chan) 175 { 176 struct dw_edma *dw = chan->dw; 177 struct dw_edma_chunk *child; 178 struct dw_edma_desc *desc; 179 struct virt_dma_desc *vd; 180 181 vd = vchan_next_desc(&chan->vc); 182 if (!vd) 183 return 0; 184 185 desc = vd2dw_edma_desc(vd); 186 if (!desc) 187 return 0; 188 189 child = list_first_entry_or_null(&desc->chunk->list, 190 struct dw_edma_chunk, list); 191 if (!child) 192 return 0; 193 194 dw_edma_core_start(dw, child, !desc->xfer_sz); 195 desc->xfer_sz += child->ll_region.sz; 196 dw_edma_free_burst(child); 197 list_del(&child->list); 198 kfree(child); 199 desc->chunks_alloc--; 200 201 return 1; 202 } 203 204 static void dw_edma_device_caps(struct dma_chan *dchan, 205 struct dma_slave_caps *caps) 206 { 207 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 208 209 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 210 if (chan->dir == EDMA_DIR_READ) 211 caps->directions = BIT(DMA_DEV_TO_MEM); 212 else 213 caps->directions = BIT(DMA_MEM_TO_DEV); 214 } else { 215 if (chan->dir == EDMA_DIR_WRITE) 216 caps->directions = BIT(DMA_DEV_TO_MEM); 217 else 218 caps->directions = BIT(DMA_MEM_TO_DEV); 219 } 220 } 221 222 static int dw_edma_device_config(struct dma_chan *dchan, 223 struct dma_slave_config *config) 224 { 225 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 226 bool cfg_non_ll; 227 int non_ll = 0; 228 229 chan->non_ll = false; 230 if (chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE) { 231 if (config->peripheral_config && 232 config->peripheral_size != sizeof(int)) { 233 dev_err(dchan->device->dev, 234 "config param peripheral size mismatch\n"); 235 return -EINVAL; 236 } 237 238 /* 239 * When there is no valid LLP base address available then the 240 * default DMA ops will use the non-LL mode. 241 * 242 * Cases where LL mode is enabled and client wants to use the 243 * non-LL mode then also client can do so via providing the 244 * peripheral_config param. 245 */ 246 cfg_non_ll = chan->dw->chip->cfg_non_ll; 247 if (config->peripheral_config) { 248 non_ll = *(int *)config->peripheral_config; 249 250 if (cfg_non_ll && !non_ll) { 251 dev_err(dchan->device->dev, "invalid configuration\n"); 252 return -EINVAL; 253 } 254 } 255 256 if (cfg_non_ll || non_ll) 257 chan->non_ll = true; 258 } else if (config->peripheral_config) { 259 dev_err(dchan->device->dev, 260 "peripheral config param applicable only for HDMA\n"); 261 return -EINVAL; 262 } 263 264 memcpy(&chan->config, config, sizeof(*config)); 265 chan->configured = true; 266 267 return 0; 268 } 269 270 static int dw_edma_device_pause(struct dma_chan *dchan) 271 { 272 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 273 int err = 0; 274 275 if (!chan->configured) 276 err = -EPERM; 277 else if (chan->status != EDMA_ST_BUSY) 278 err = -EPERM; 279 else if (chan->request != EDMA_REQ_NONE) 280 err = -EPERM; 281 else 282 chan->request = EDMA_REQ_PAUSE; 283 284 return err; 285 } 286 287 static int dw_edma_device_resume(struct dma_chan *dchan) 288 { 289 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 290 int err = 0; 291 292 if (!chan->configured) { 293 err = -EPERM; 294 } else if (chan->status != EDMA_ST_PAUSE) { 295 err = -EPERM; 296 } else if (chan->request != EDMA_REQ_NONE) { 297 err = -EPERM; 298 } else { 299 chan->status = EDMA_ST_BUSY; 300 dw_edma_start_transfer(chan); 301 } 302 303 return err; 304 } 305 306 static int dw_edma_device_terminate_all(struct dma_chan *dchan) 307 { 308 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 309 int err = 0; 310 311 if (!chan->configured) { 312 /* Do nothing */ 313 } else if (chan->status == EDMA_ST_PAUSE) { 314 chan->status = EDMA_ST_IDLE; 315 chan->configured = false; 316 } else if (chan->status == EDMA_ST_IDLE) { 317 chan->configured = false; 318 } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { 319 /* 320 * The channel is in a false BUSY state, probably didn't 321 * receive or lost an interrupt 322 */ 323 chan->status = EDMA_ST_IDLE; 324 chan->configured = false; 325 } else if (chan->request > EDMA_REQ_PAUSE) { 326 err = -EPERM; 327 } else { 328 chan->request = EDMA_REQ_STOP; 329 } 330 331 return err; 332 } 333 334 static void dw_edma_device_issue_pending(struct dma_chan *dchan) 335 { 336 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 337 unsigned long flags; 338 339 if (!chan->configured) 340 return; 341 342 spin_lock_irqsave(&chan->vc.lock, flags); 343 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && 344 chan->status == EDMA_ST_IDLE) { 345 chan->status = EDMA_ST_BUSY; 346 dw_edma_start_transfer(chan); 347 } 348 spin_unlock_irqrestore(&chan->vc.lock, flags); 349 } 350 351 static enum dma_status 352 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 353 struct dma_tx_state *txstate) 354 { 355 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 356 struct dw_edma_desc *desc; 357 struct virt_dma_desc *vd; 358 unsigned long flags; 359 enum dma_status ret; 360 u32 residue = 0; 361 362 ret = dma_cookie_status(dchan, cookie, txstate); 363 if (ret == DMA_COMPLETE) 364 return ret; 365 366 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) 367 ret = DMA_PAUSED; 368 369 if (!txstate) 370 goto ret_residue; 371 372 spin_lock_irqsave(&chan->vc.lock, flags); 373 vd = vchan_find_desc(&chan->vc, cookie); 374 if (vd) { 375 desc = vd2dw_edma_desc(vd); 376 if (desc) 377 residue = desc->alloc_sz - desc->xfer_sz; 378 } 379 spin_unlock_irqrestore(&chan->vc.lock, flags); 380 381 ret_residue: 382 dma_set_residue(txstate, residue); 383 384 return ret; 385 } 386 387 static struct dma_async_tx_descriptor * 388 dw_edma_device_transfer(struct dw_edma_transfer *xfer) 389 { 390 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); 391 enum dma_transfer_direction dir = xfer->direction; 392 struct scatterlist *sg = NULL; 393 struct dw_edma_chunk *chunk; 394 struct dw_edma_burst *burst; 395 struct dw_edma_desc *desc; 396 u64 src_addr, dst_addr; 397 size_t fsz = 0; 398 u32 bursts_max; 399 u32 cnt = 0; 400 int i; 401 402 if (!chan->configured) 403 return NULL; 404 405 /* 406 * Local Root Port/End-point Remote End-point 407 * +-----------------------+ PCIe bus +----------------------+ 408 * | | +-+ | | 409 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | 410 * | | | | | | 411 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | 412 * | | +-+ | | 413 * +-----------------------+ +----------------------+ 414 * 415 * 1. Normal logic: 416 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the 417 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used 418 * for the device read operations (DEV_TO_MEM) and the Tx channel 419 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). 420 * 421 * 2. Inverted logic: 422 * If eDMA is embedded into a Remote PCIe EP and is controlled by the 423 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx 424 * channel (EDMA_DIR_WRITE) will be used for the device read operations 425 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write 426 * operations (MEM_TO_DEV). 427 * 428 * It is the client driver responsibility to choose a proper channel 429 * for the DMA transfers. 430 */ 431 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 432 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || 433 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) 434 return NULL; 435 } else { 436 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || 437 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) 438 return NULL; 439 } 440 441 if (xfer->type == EDMA_XFER_CYCLIC) { 442 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) 443 return NULL; 444 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 445 if (xfer->xfer.sg.len < 1) 446 return NULL; 447 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 448 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) 449 return NULL; 450 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) 451 return NULL; 452 } else { 453 return NULL; 454 } 455 456 /* 457 * For non-LL mode, only a single burst can be handled 458 * in a single chunk unlike LL mode where multiple bursts 459 * can be configured in a single chunk. 460 */ 461 bursts_max = chan->non_ll ? 1 : chan->ll_max; 462 463 desc = dw_edma_alloc_desc(chan); 464 if (unlikely(!desc)) 465 goto err_alloc; 466 467 chunk = dw_edma_alloc_chunk(desc); 468 if (unlikely(!chunk)) 469 goto err_alloc; 470 471 if (xfer->type == EDMA_XFER_INTERLEAVED) { 472 src_addr = xfer->xfer.il->src_start; 473 dst_addr = xfer->xfer.il->dst_start; 474 } else { 475 src_addr = chan->config.src_addr; 476 dst_addr = chan->config.dst_addr; 477 } 478 479 if (dir == DMA_DEV_TO_MEM) 480 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr); 481 else 482 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr); 483 484 if (xfer->type == EDMA_XFER_CYCLIC) { 485 cnt = xfer->xfer.cyclic.cnt; 486 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 487 cnt = xfer->xfer.sg.len; 488 sg = xfer->xfer.sg.sgl; 489 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 490 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; 491 fsz = xfer->xfer.il->frame_size; 492 } 493 494 for (i = 0; i < cnt; i++) { 495 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) 496 break; 497 498 if (chunk->bursts_alloc == bursts_max) { 499 chunk = dw_edma_alloc_chunk(desc); 500 if (unlikely(!chunk)) 501 goto err_alloc; 502 } 503 504 burst = dw_edma_alloc_burst(chunk); 505 if (unlikely(!burst)) 506 goto err_alloc; 507 508 if (xfer->type == EDMA_XFER_CYCLIC) 509 burst->sz = xfer->xfer.cyclic.len; 510 else if (xfer->type == EDMA_XFER_SCATTER_GATHER) 511 burst->sz = sg_dma_len(sg); 512 else if (xfer->type == EDMA_XFER_INTERLEAVED) 513 burst->sz = xfer->xfer.il->sgl[i % fsz].size; 514 515 chunk->ll_region.sz += burst->sz; 516 desc->alloc_sz += burst->sz; 517 518 if (dir == DMA_DEV_TO_MEM) { 519 burst->sar = src_addr; 520 if (xfer->type == EDMA_XFER_CYCLIC) { 521 burst->dar = xfer->xfer.cyclic.paddr; 522 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 523 src_addr += sg_dma_len(sg); 524 burst->dar = sg_dma_address(sg); 525 /* Unlike the typical assumption by other 526 * drivers/IPs the peripheral memory isn't 527 * a FIFO memory, in this case, it's a 528 * linear memory and that why the source 529 * and destination addresses are increased 530 * by the same portion (data length) 531 */ 532 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 533 burst->dar = dst_addr; 534 } 535 } else { 536 burst->dar = dst_addr; 537 if (xfer->type == EDMA_XFER_CYCLIC) { 538 burst->sar = xfer->xfer.cyclic.paddr; 539 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 540 dst_addr += sg_dma_len(sg); 541 burst->sar = sg_dma_address(sg); 542 /* Unlike the typical assumption by other 543 * drivers/IPs the peripheral memory isn't 544 * a FIFO memory, in this case, it's a 545 * linear memory and that why the source 546 * and destination addresses are increased 547 * by the same portion (data length) 548 */ 549 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 550 burst->sar = src_addr; 551 } 552 } 553 554 if (xfer->type == EDMA_XFER_SCATTER_GATHER) { 555 sg = sg_next(sg); 556 } else if (xfer->type == EDMA_XFER_INTERLEAVED) { 557 struct dma_interleaved_template *il = xfer->xfer.il; 558 struct data_chunk *dc = &il->sgl[i % fsz]; 559 560 src_addr += burst->sz; 561 if (il->src_sgl) 562 src_addr += dmaengine_get_src_icg(il, dc); 563 564 dst_addr += burst->sz; 565 if (il->dst_sgl) 566 dst_addr += dmaengine_get_dst_icg(il, dc); 567 } 568 } 569 570 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); 571 572 err_alloc: 573 if (desc) 574 dw_edma_free_desc(desc); 575 576 return NULL; 577 } 578 579 static struct dma_async_tx_descriptor * 580 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 581 unsigned int len, 582 enum dma_transfer_direction direction, 583 unsigned long flags, void *context) 584 { 585 struct dw_edma_transfer xfer; 586 587 xfer.dchan = dchan; 588 xfer.direction = direction; 589 xfer.xfer.sg.sgl = sgl; 590 xfer.xfer.sg.len = len; 591 xfer.flags = flags; 592 xfer.type = EDMA_XFER_SCATTER_GATHER; 593 594 return dw_edma_device_transfer(&xfer); 595 } 596 597 static struct dma_async_tx_descriptor * 598 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, 599 size_t len, size_t count, 600 enum dma_transfer_direction direction, 601 unsigned long flags) 602 { 603 struct dw_edma_transfer xfer; 604 605 xfer.dchan = dchan; 606 xfer.direction = direction; 607 xfer.xfer.cyclic.paddr = paddr; 608 xfer.xfer.cyclic.len = len; 609 xfer.xfer.cyclic.cnt = count; 610 xfer.flags = flags; 611 xfer.type = EDMA_XFER_CYCLIC; 612 613 return dw_edma_device_transfer(&xfer); 614 } 615 616 static struct dma_async_tx_descriptor * 617 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, 618 struct dma_interleaved_template *ilt, 619 unsigned long flags) 620 { 621 struct dw_edma_transfer xfer; 622 623 xfer.dchan = dchan; 624 xfer.direction = ilt->dir; 625 xfer.xfer.il = ilt; 626 xfer.flags = flags; 627 xfer.type = EDMA_XFER_INTERLEAVED; 628 629 return dw_edma_device_transfer(&xfer); 630 } 631 632 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, 633 enum dmaengine_tx_result result) 634 { 635 u32 residue = 0; 636 struct dw_edma_desc *desc; 637 struct dmaengine_result *res; 638 639 if (!vd->tx.callback_result) 640 return; 641 642 desc = vd2dw_edma_desc(vd); 643 if (desc) 644 residue = desc->alloc_sz - desc->xfer_sz; 645 646 res = &vd->tx_result; 647 res->result = result; 648 res->residue = residue; 649 } 650 651 static void dw_edma_done_interrupt(struct dw_edma_chan *chan) 652 { 653 struct dw_edma_desc *desc; 654 struct virt_dma_desc *vd; 655 unsigned long flags; 656 657 spin_lock_irqsave(&chan->vc.lock, flags); 658 vd = vchan_next_desc(&chan->vc); 659 if (vd) { 660 switch (chan->request) { 661 case EDMA_REQ_NONE: 662 desc = vd2dw_edma_desc(vd); 663 if (!desc->chunks_alloc) { 664 dw_hdma_set_callback_result(vd, 665 DMA_TRANS_NOERROR); 666 list_del(&vd->node); 667 vchan_cookie_complete(vd); 668 } 669 670 /* Continue transferring if there are remaining chunks or issued requests. 671 */ 672 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; 673 break; 674 675 case EDMA_REQ_STOP: 676 list_del(&vd->node); 677 vchan_cookie_complete(vd); 678 chan->request = EDMA_REQ_NONE; 679 chan->status = EDMA_ST_IDLE; 680 break; 681 682 case EDMA_REQ_PAUSE: 683 chan->request = EDMA_REQ_NONE; 684 chan->status = EDMA_ST_PAUSE; 685 break; 686 687 default: 688 break; 689 } 690 } 691 spin_unlock_irqrestore(&chan->vc.lock, flags); 692 } 693 694 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) 695 { 696 struct virt_dma_desc *vd; 697 unsigned long flags; 698 699 spin_lock_irqsave(&chan->vc.lock, flags); 700 vd = vchan_next_desc(&chan->vc); 701 if (vd) { 702 dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); 703 list_del(&vd->node); 704 vchan_cookie_complete(vd); 705 } 706 spin_unlock_irqrestore(&chan->vc.lock, flags); 707 chan->request = EDMA_REQ_NONE; 708 chan->status = EDMA_ST_IDLE; 709 } 710 711 static void dw_edma_emul_irq_ack(struct irq_data *d) 712 { 713 struct dw_edma *dw = irq_data_get_irq_chip_data(d); 714 715 dw_edma_core_ack_emulated_irq(dw); 716 } 717 718 /* 719 * irq_chip implementation for interrupt-emulation doorbells. 720 * 721 * The emulated source has no mask/unmask mechanism. With handle_level_irq(), 722 * the flow is therefore: 723 * 1) .irq_ack() deasserts the source 724 * 2) registered handlers (if any) are dispatched 725 * Since deassertion is already done in .irq_ack(), handlers do not need to take 726 * care of it, hence IRQCHIP_ONESHOT_SAFE. 727 */ 728 static struct irq_chip dw_edma_emul_irqchip = { 729 .name = "dw-edma-emul", 730 .irq_ack = dw_edma_emul_irq_ack, 731 .flags = IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE, 732 }; 733 734 static int dw_edma_emul_irq_alloc(struct dw_edma *dw) 735 { 736 struct dw_edma_chip *chip = dw->chip; 737 int virq; 738 739 chip->db_irq = 0; 740 chip->db_offset = ~0; 741 742 /* 743 * Only meaningful when the core provides the deassert sequence 744 * for interrupt emulation. 745 */ 746 if (!dw->core->ack_emulated_irq) 747 return 0; 748 749 /* 750 * Allocate a single, requestable Linux virtual IRQ number. 751 * Use >= 1 so that 0 can remain a "not available" sentinel. 752 */ 753 virq = irq_alloc_desc(NUMA_NO_NODE); 754 if (virq < 0) 755 return virq; 756 757 irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq); 758 irq_set_chip_data(virq, dw); 759 irq_set_noprobe(virq); 760 761 chip->db_irq = virq; 762 chip->db_offset = dw_edma_core_db_offset(dw); 763 764 return 0; 765 } 766 767 static void dw_edma_emul_irq_free(struct dw_edma *dw) 768 { 769 struct dw_edma_chip *chip = dw->chip; 770 771 if (!chip) 772 return; 773 if (chip->db_irq <= 0) 774 return; 775 776 irq_free_descs(chip->db_irq, 1); 777 chip->db_irq = 0; 778 chip->db_offset = ~0; 779 } 780 781 static inline irqreturn_t dw_edma_interrupt_emulated(void *data) 782 { 783 struct dw_edma_irq *dw_irq = data; 784 struct dw_edma *dw = dw_irq->dw; 785 int db_irq = dw->chip->db_irq; 786 787 if (db_irq > 0) { 788 /* 789 * Interrupt emulation may assert the IRQ line without updating the 790 * normal DONE/ABORT status bits. With a shared IRQ handler we 791 * cannot reliably detect such events by status registers alone, so 792 * always perform the core-specific deassert sequence. 793 */ 794 generic_handle_irq(db_irq); 795 return IRQ_HANDLED; 796 } 797 return IRQ_NONE; 798 } 799 800 static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data) 801 { 802 struct dw_edma_irq *dw_irq = data; 803 804 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, 805 dw_edma_done_interrupt, 806 dw_edma_abort_interrupt); 807 } 808 809 static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) 810 { 811 struct dw_edma_irq *dw_irq = data; 812 813 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, 814 dw_edma_done_interrupt, 815 dw_edma_abort_interrupt); 816 } 817 818 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) 819 { 820 irqreturn_t ret = IRQ_NONE; 821 822 ret |= dw_edma_interrupt_write_inner(irq, data); 823 ret |= dw_edma_interrupt_emulated(data); 824 825 return ret; 826 } 827 828 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) 829 { 830 irqreturn_t ret = IRQ_NONE; 831 832 ret |= dw_edma_interrupt_read_inner(irq, data); 833 ret |= dw_edma_interrupt_emulated(data); 834 835 return ret; 836 } 837 838 static inline irqreturn_t dw_edma_interrupt_common(int irq, void *data) 839 { 840 irqreturn_t ret = IRQ_NONE; 841 842 ret |= dw_edma_interrupt_write_inner(irq, data); 843 ret |= dw_edma_interrupt_read_inner(irq, data); 844 ret |= dw_edma_interrupt_emulated(data); 845 846 return ret; 847 } 848 849 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) 850 { 851 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); 852 853 if (chan->status != EDMA_ST_IDLE) 854 return -EBUSY; 855 856 return 0; 857 } 858 859 static void dw_edma_free_chan_resources(struct dma_chan *dchan) 860 { 861 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 862 int ret; 863 864 while (time_before(jiffies, timeout)) { 865 ret = dw_edma_device_terminate_all(dchan); 866 if (!ret) 867 break; 868 869 if (time_after_eq(jiffies, timeout)) 870 return; 871 872 cpu_relax(); 873 } 874 } 875 876 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) 877 { 878 struct dw_edma_chip *chip = dw->chip; 879 struct device *dev = chip->dev; 880 struct dw_edma_chan *chan; 881 struct dw_edma_irq *irq; 882 struct dma_device *dma; 883 u32 i, ch_cnt; 884 u32 pos; 885 886 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 887 dma = &dw->dma; 888 889 INIT_LIST_HEAD(&dma->channels); 890 891 for (i = 0; i < ch_cnt; i++) { 892 chan = &dw->chan[i]; 893 894 chan->dw = dw; 895 896 if (i < dw->wr_ch_cnt) { 897 chan->id = i; 898 chan->dir = EDMA_DIR_WRITE; 899 } else { 900 chan->id = i - dw->wr_ch_cnt; 901 chan->dir = EDMA_DIR_READ; 902 } 903 904 chan->configured = false; 905 chan->request = EDMA_REQ_NONE; 906 chan->status = EDMA_ST_IDLE; 907 908 if (chan->dir == EDMA_DIR_WRITE) 909 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); 910 else 911 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); 912 chan->ll_max -= 1; 913 914 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", 915 str_write_read(chan->dir == EDMA_DIR_WRITE), 916 chan->id, chan->ll_max); 917 918 if (dw->nr_irqs == 1) 919 pos = 0; 920 else if (chan->dir == EDMA_DIR_WRITE) 921 pos = chan->id % wr_alloc; 922 else 923 pos = wr_alloc + chan->id % rd_alloc; 924 925 irq = &dw->irq[pos]; 926 927 if (chan->dir == EDMA_DIR_WRITE) 928 irq->wr_mask |= BIT(chan->id); 929 else 930 irq->rd_mask |= BIT(chan->id); 931 932 irq->dw = dw; 933 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); 934 935 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", 936 str_write_read(chan->dir == EDMA_DIR_WRITE), 937 chan->id, 938 chan->msi.address_hi, chan->msi.address_lo, 939 chan->msi.data); 940 941 chan->vc.desc_free = vchan_free_desc; 942 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? 943 &dw->chip->dt_region_wr[chan->id] : 944 &dw->chip->dt_region_rd[chan->id]; 945 946 vchan_init(&chan->vc, dma); 947 948 dw_edma_core_ch_config(chan); 949 } 950 951 /* Set DMA channel capabilities */ 952 dma_cap_zero(dma->cap_mask); 953 dma_cap_set(DMA_SLAVE, dma->cap_mask); 954 dma_cap_set(DMA_CYCLIC, dma->cap_mask); 955 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 956 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); 957 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 958 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 959 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 960 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 961 962 /* Set DMA channel callbacks */ 963 dma->dev = chip->dev; 964 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; 965 dma->device_free_chan_resources = dw_edma_free_chan_resources; 966 dma->device_caps = dw_edma_device_caps; 967 dma->device_config = dw_edma_device_config; 968 dma->device_pause = dw_edma_device_pause; 969 dma->device_resume = dw_edma_device_resume; 970 dma->device_terminate_all = dw_edma_device_terminate_all; 971 dma->device_issue_pending = dw_edma_device_issue_pending; 972 dma->device_tx_status = dw_edma_device_tx_status; 973 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; 974 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; 975 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; 976 977 dma_set_max_seg_size(dma->dev, U32_MAX); 978 979 /* Register DMA device */ 980 return dma_async_device_register(dma); 981 } 982 983 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) 984 { 985 if (*nr_irqs && *alloc < cnt) { 986 (*alloc)++; 987 (*nr_irqs)--; 988 } 989 } 990 991 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) 992 { 993 while (*mask * alloc < cnt) 994 (*mask)++; 995 } 996 997 static int dw_edma_irq_request(struct dw_edma *dw, 998 u32 *wr_alloc, u32 *rd_alloc) 999 { 1000 struct dw_edma_chip *chip = dw->chip; 1001 struct device *dev = dw->chip->dev; 1002 struct msi_desc *msi_desc; 1003 u32 wr_mask = 1; 1004 u32 rd_mask = 1; 1005 int i, err = 0; 1006 u32 ch_cnt; 1007 int irq; 1008 1009 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; 1010 1011 if (chip->nr_irqs < 1 || !chip->ops->irq_vector) 1012 return -EINVAL; 1013 1014 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); 1015 if (!dw->irq) 1016 return -ENOMEM; 1017 1018 if (chip->nr_irqs == 1) { 1019 /* Common IRQ shared among all channels */ 1020 irq = chip->ops->irq_vector(dev, 0); 1021 err = request_irq(irq, dw_edma_interrupt_common, 1022 IRQF_SHARED, dw->name, &dw->irq[0]); 1023 if (err) { 1024 dw->nr_irqs = 0; 1025 return err; 1026 } 1027 1028 if (irq_get_msi_desc(irq)) 1029 get_cached_msi_msg(irq, &dw->irq[0].msi); 1030 1031 dw->nr_irqs = 1; 1032 } else { 1033 /* Distribute IRQs equally among all channels */ 1034 int tmp = chip->nr_irqs; 1035 1036 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { 1037 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); 1038 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); 1039 } 1040 1041 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); 1042 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); 1043 1044 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { 1045 irq = chip->ops->irq_vector(dev, i); 1046 err = request_irq(irq, 1047 i < *wr_alloc ? 1048 dw_edma_interrupt_write : 1049 dw_edma_interrupt_read, 1050 IRQF_SHARED, dw->name, 1051 &dw->irq[i]); 1052 if (err) 1053 goto err_irq_free; 1054 msi_desc = irq_get_msi_desc(irq); 1055 if (msi_desc) { 1056 get_cached_msi_msg(irq, &dw->irq[i].msi); 1057 if (!msi_desc->pci.msi_attrib.is_msix) 1058 dw->irq[i].msi.data = dw->irq[0].msi.data + i; 1059 } 1060 } 1061 1062 dw->nr_irqs = i; 1063 } 1064 1065 return 0; 1066 1067 err_irq_free: 1068 for (i--; i >= 0; i--) { 1069 irq = chip->ops->irq_vector(dev, i); 1070 free_irq(irq, &dw->irq[i]); 1071 } 1072 1073 return err; 1074 } 1075 1076 int dw_edma_probe(struct dw_edma_chip *chip) 1077 { 1078 struct device *dev; 1079 struct dw_edma *dw; 1080 u32 wr_alloc = 0; 1081 u32 rd_alloc = 0; 1082 int i, err; 1083 1084 if (!chip) 1085 return -EINVAL; 1086 1087 dev = chip->dev; 1088 if (!dev || !chip->ops) 1089 return -EINVAL; 1090 1091 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); 1092 if (!dw) 1093 return -ENOMEM; 1094 1095 dw->chip = chip; 1096 1097 if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) 1098 dw_hdma_v0_core_register(dw); 1099 else 1100 dw_edma_v0_core_register(dw); 1101 1102 raw_spin_lock_init(&dw->lock); 1103 1104 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, 1105 dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); 1106 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); 1107 1108 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, 1109 dw_edma_core_ch_count(dw, EDMA_DIR_READ)); 1110 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); 1111 1112 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) 1113 return -EINVAL; 1114 1115 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", 1116 dw->wr_ch_cnt, dw->rd_ch_cnt); 1117 1118 /* Allocate channels */ 1119 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, 1120 sizeof(*dw->chan), GFP_KERNEL); 1121 if (!dw->chan) 1122 return -ENOMEM; 1123 1124 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", 1125 dev_name(chip->dev)); 1126 1127 /* Disable eDMA, only to establish the ideal initial conditions */ 1128 dw_edma_core_off(dw); 1129 1130 /* Request IRQs */ 1131 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); 1132 if (err) 1133 return err; 1134 1135 /* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */ 1136 err = dw_edma_emul_irq_alloc(dw); 1137 if (err) 1138 dev_warn(dev, "Failed to allocate emulation IRQ: %d\n", err); 1139 1140 /* Setup write/read channels */ 1141 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); 1142 if (err) 1143 goto err_irq_free; 1144 1145 /* Turn debugfs on */ 1146 dw_edma_core_debugfs_on(dw); 1147 1148 chip->dw = dw; 1149 1150 return 0; 1151 1152 err_irq_free: 1153 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1154 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1155 dw_edma_emul_irq_free(dw); 1156 1157 return err; 1158 } 1159 EXPORT_SYMBOL_GPL(dw_edma_probe); 1160 1161 int dw_edma_remove(struct dw_edma_chip *chip) 1162 { 1163 struct dw_edma_chan *chan, *_chan; 1164 struct device *dev = chip->dev; 1165 struct dw_edma *dw = chip->dw; 1166 int i; 1167 1168 /* Skip removal if no private data found */ 1169 if (!dw) 1170 return -ENODEV; 1171 1172 /* Disable eDMA */ 1173 dw_edma_core_off(dw); 1174 1175 /* Free irqs */ 1176 for (i = (dw->nr_irqs - 1); i >= 0; i--) 1177 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); 1178 dw_edma_emul_irq_free(dw); 1179 1180 /* Deregister eDMA device */ 1181 dma_async_device_unregister(&dw->dma); 1182 list_for_each_entry_safe(chan, _chan, &dw->dma.channels, 1183 vc.chan.device_node) { 1184 tasklet_kill(&chan->vc.task); 1185 list_del(&chan->vc.chan.device_node); 1186 } 1187 1188 return 0; 1189 } 1190 EXPORT_SYMBOL_GPL(dw_edma_remove); 1191 1192 MODULE_LICENSE("GPL v2"); 1193 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); 1194 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); 1195