1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/delay.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <linux/err.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/list.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 #include <linux/of_device.h> 22 #include <linux/of_irq.h> 23 #include <linux/workqueue.h> 24 #include <linux/completion.h> 25 #include <linux/soc/ti/k3-ringacc.h> 26 #include <linux/soc/ti/ti_sci_protocol.h> 27 #include <linux/soc/ti/ti_sci_inta_msi.h> 28 #include <linux/dma/ti-cppi5.h> 29 30 #include "../virt-dma.h" 31 #include "k3-udma.h" 32 #include "k3-psil-priv.h" 33 34 struct udma_static_tr { 35 u8 elsize; /* RPSTR0 */ 36 u16 elcnt; /* RPSTR0 */ 37 u16 bstcnt; /* RPSTR1 */ 38 }; 39 40 #define K3_UDMA_MAX_RFLOWS 1024 41 #define K3_UDMA_DEFAULT_RING_SIZE 16 42 43 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 44 #define UDMA_RFLOW_SRCTAG_NONE 0 45 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 46 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 47 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 48 49 #define UDMA_RFLOW_DSTTAG_NONE 0 50 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 51 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 52 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 53 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 54 55 struct udma_chan; 56 57 enum udma_mmr { 58 MMR_GCFG = 0, 59 MMR_RCHANRT, 60 MMR_TCHANRT, 61 MMR_LAST, 62 }; 63 64 static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" }; 65 66 struct udma_tchan { 67 void __iomem *reg_rt; 68 69 int id; 70 struct k3_ring *t_ring; /* Transmit ring */ 71 struct k3_ring *tc_ring; /* Transmit Completion ring */ 72 }; 73 74 struct udma_rflow { 75 int id; 76 struct k3_ring *fd_ring; /* Free Descriptor ring */ 77 struct k3_ring *r_ring; /* Receive ring */ 78 }; 79 80 struct udma_rchan { 81 void __iomem *reg_rt; 82 83 int id; 84 }; 85 86 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 87 #define UDMA_FLAG_PDMA_BURST BIT(1) 88 89 struct udma_match_data { 90 u32 psil_base; 91 bool enable_memcpy_support; 92 u32 flags; 93 u32 statictr_z_mask; 94 u32 rchan_oes_offset; 95 96 u8 tpl_levels; 97 u32 level_start_idx[]; 98 }; 99 100 struct udma_hwdesc { 101 size_t cppi5_desc_size; 102 void *cppi5_desc_vaddr; 103 dma_addr_t cppi5_desc_paddr; 104 105 /* TR descriptor internal pointers */ 106 void *tr_req_base; 107 struct cppi5_tr_resp_t *tr_resp_base; 108 }; 109 110 struct udma_rx_flush { 111 struct udma_hwdesc hwdescs[2]; 112 113 size_t buffer_size; 114 void *buffer_vaddr; 115 dma_addr_t buffer_paddr; 116 }; 117 118 struct udma_dev { 119 struct dma_device ddev; 120 struct device *dev; 121 void __iomem *mmrs[MMR_LAST]; 122 const struct udma_match_data *match_data; 123 124 size_t desc_align; /* alignment to use for descriptors */ 125 126 struct udma_tisci_rm tisci_rm; 127 128 struct k3_ringacc *ringacc; 129 130 struct work_struct purge_work; 131 struct list_head desc_to_purge; 132 spinlock_t lock; 133 134 struct udma_rx_flush rx_flush; 135 136 int tchan_cnt; 137 int echan_cnt; 138 int rchan_cnt; 139 int rflow_cnt; 140 unsigned long *tchan_map; 141 unsigned long *rchan_map; 142 unsigned long *rflow_gp_map; 143 unsigned long *rflow_gp_map_allocated; 144 unsigned long *rflow_in_use; 145 146 struct udma_tchan *tchans; 147 struct udma_rchan *rchans; 148 struct udma_rflow *rflows; 149 150 struct udma_chan *channels; 151 u32 psil_base; 152 u32 atype; 153 }; 154 155 struct udma_desc { 156 struct virt_dma_desc vd; 157 158 bool terminated; 159 160 enum dma_transfer_direction dir; 161 162 struct udma_static_tr static_tr; 163 u32 residue; 164 165 unsigned int sglen; 166 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 167 unsigned int tr_idx; 168 169 u32 metadata_size; 170 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 171 172 unsigned int hwdesc_count; 173 struct udma_hwdesc hwdesc[]; 174 }; 175 176 enum udma_chan_state { 177 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 178 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 179 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 180 }; 181 182 struct udma_tx_drain { 183 struct delayed_work work; 184 ktime_t tstamp; 185 u32 residue; 186 }; 187 188 struct udma_chan_config { 189 bool pkt_mode; /* TR or packet */ 190 bool needs_epib; /* EPIB is needed for the communication or not */ 191 u32 psd_size; /* size of Protocol Specific Data */ 192 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 193 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 194 bool notdpkt; /* Suppress sending TDC packet */ 195 int remote_thread_id; 196 u32 atype; 197 u32 src_thread; 198 u32 dst_thread; 199 enum psil_endpoint_type ep_type; 200 bool enable_acc32; 201 bool enable_burst; 202 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 203 204 enum dma_transfer_direction dir; 205 }; 206 207 struct udma_chan { 208 struct virt_dma_chan vc; 209 struct dma_slave_config cfg; 210 struct udma_dev *ud; 211 struct udma_desc *desc; 212 struct udma_desc *terminated_desc; 213 struct udma_static_tr static_tr; 214 char *name; 215 216 struct udma_tchan *tchan; 217 struct udma_rchan *rchan; 218 struct udma_rflow *rflow; 219 220 bool psil_paired; 221 222 int irq_num_ring; 223 int irq_num_udma; 224 225 bool cyclic; 226 bool paused; 227 228 enum udma_chan_state state; 229 struct completion teardown_completed; 230 231 struct udma_tx_drain tx_drain; 232 233 u32 bcnt; /* number of bytes completed since the start of the channel */ 234 235 /* Channel configuration parameters */ 236 struct udma_chan_config config; 237 238 /* dmapool for packet mode descriptors */ 239 bool use_dma_pool; 240 struct dma_pool *hdesc_pool; 241 242 u32 id; 243 }; 244 245 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 246 { 247 return container_of(d, struct udma_dev, ddev); 248 } 249 250 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 251 { 252 return container_of(c, struct udma_chan, vc.chan); 253 } 254 255 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 256 { 257 return container_of(t, struct udma_desc, vd.tx); 258 } 259 260 /* Generic register access functions */ 261 static inline u32 udma_read(void __iomem *base, int reg) 262 { 263 return readl(base + reg); 264 } 265 266 static inline void udma_write(void __iomem *base, int reg, u32 val) 267 { 268 writel(val, base + reg); 269 } 270 271 static inline void udma_update_bits(void __iomem *base, int reg, 272 u32 mask, u32 val) 273 { 274 u32 tmp, orig; 275 276 orig = readl(base + reg); 277 tmp = orig & ~mask; 278 tmp |= (val & mask); 279 280 if (tmp != orig) 281 writel(tmp, base + reg); 282 } 283 284 /* TCHANRT */ 285 static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg) 286 { 287 if (!tchan) 288 return 0; 289 return udma_read(tchan->reg_rt, reg); 290 } 291 292 static inline void udma_tchanrt_write(struct udma_tchan *tchan, int reg, 293 u32 val) 294 { 295 if (!tchan) 296 return; 297 udma_write(tchan->reg_rt, reg, val); 298 } 299 300 static inline void udma_tchanrt_update_bits(struct udma_tchan *tchan, int reg, 301 u32 mask, u32 val) 302 { 303 if (!tchan) 304 return; 305 udma_update_bits(tchan->reg_rt, reg, mask, val); 306 } 307 308 /* RCHANRT */ 309 static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg) 310 { 311 if (!rchan) 312 return 0; 313 return udma_read(rchan->reg_rt, reg); 314 } 315 316 static inline void udma_rchanrt_write(struct udma_rchan *rchan, int reg, 317 u32 val) 318 { 319 if (!rchan) 320 return; 321 udma_write(rchan->reg_rt, reg, val); 322 } 323 324 static inline void udma_rchanrt_update_bits(struct udma_rchan *rchan, int reg, 325 u32 mask, u32 val) 326 { 327 if (!rchan) 328 return; 329 udma_update_bits(rchan->reg_rt, reg, mask, val); 330 } 331 332 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 333 { 334 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 335 336 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 337 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 338 tisci_rm->tisci_navss_dev_id, 339 src_thread, dst_thread); 340 } 341 342 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 343 u32 dst_thread) 344 { 345 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 346 347 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 348 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 349 tisci_rm->tisci_navss_dev_id, 350 src_thread, dst_thread); 351 } 352 353 static void udma_reset_uchan(struct udma_chan *uc) 354 { 355 memset(&uc->config, 0, sizeof(uc->config)); 356 uc->config.remote_thread_id = -1; 357 uc->state = UDMA_CHAN_IS_IDLE; 358 } 359 360 static void udma_dump_chan_stdata(struct udma_chan *uc) 361 { 362 struct device *dev = uc->ud->dev; 363 u32 offset; 364 int i; 365 366 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 367 dev_dbg(dev, "TCHAN State data:\n"); 368 for (i = 0; i < 32; i++) { 369 offset = UDMA_TCHAN_RT_STDATA_REG + i * 4; 370 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 371 udma_tchanrt_read(uc->tchan, offset)); 372 } 373 } 374 375 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 376 dev_dbg(dev, "RCHAN State data:\n"); 377 for (i = 0; i < 32; i++) { 378 offset = UDMA_RCHAN_RT_STDATA_REG + i * 4; 379 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 380 udma_rchanrt_read(uc->rchan, offset)); 381 } 382 } 383 } 384 385 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 386 int idx) 387 { 388 return d->hwdesc[idx].cppi5_desc_paddr; 389 } 390 391 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 392 { 393 return d->hwdesc[idx].cppi5_desc_vaddr; 394 } 395 396 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 397 dma_addr_t paddr) 398 { 399 struct udma_desc *d = uc->terminated_desc; 400 401 if (d) { 402 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 403 d->desc_idx); 404 405 if (desc_paddr != paddr) 406 d = NULL; 407 } 408 409 if (!d) { 410 d = uc->desc; 411 if (d) { 412 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 413 d->desc_idx); 414 415 if (desc_paddr != paddr) 416 d = NULL; 417 } 418 } 419 420 return d; 421 } 422 423 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 424 { 425 if (uc->use_dma_pool) { 426 int i; 427 428 for (i = 0; i < d->hwdesc_count; i++) { 429 if (!d->hwdesc[i].cppi5_desc_vaddr) 430 continue; 431 432 dma_pool_free(uc->hdesc_pool, 433 d->hwdesc[i].cppi5_desc_vaddr, 434 d->hwdesc[i].cppi5_desc_paddr); 435 436 d->hwdesc[i].cppi5_desc_vaddr = NULL; 437 } 438 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 439 struct udma_dev *ud = uc->ud; 440 441 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, 442 d->hwdesc[0].cppi5_desc_vaddr, 443 d->hwdesc[0].cppi5_desc_paddr); 444 445 d->hwdesc[0].cppi5_desc_vaddr = NULL; 446 } 447 } 448 449 static void udma_purge_desc_work(struct work_struct *work) 450 { 451 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 452 struct virt_dma_desc *vd, *_vd; 453 unsigned long flags; 454 LIST_HEAD(head); 455 456 spin_lock_irqsave(&ud->lock, flags); 457 list_splice_tail_init(&ud->desc_to_purge, &head); 458 spin_unlock_irqrestore(&ud->lock, flags); 459 460 list_for_each_entry_safe(vd, _vd, &head, node) { 461 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 462 struct udma_desc *d = to_udma_desc(&vd->tx); 463 464 udma_free_hwdesc(uc, d); 465 list_del(&vd->node); 466 kfree(d); 467 } 468 469 /* If more to purge, schedule the work again */ 470 if (!list_empty(&ud->desc_to_purge)) 471 schedule_work(&ud->purge_work); 472 } 473 474 static void udma_desc_free(struct virt_dma_desc *vd) 475 { 476 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 477 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 478 struct udma_desc *d = to_udma_desc(&vd->tx); 479 unsigned long flags; 480 481 if (uc->terminated_desc == d) 482 uc->terminated_desc = NULL; 483 484 if (uc->use_dma_pool) { 485 udma_free_hwdesc(uc, d); 486 kfree(d); 487 return; 488 } 489 490 spin_lock_irqsave(&ud->lock, flags); 491 list_add_tail(&vd->node, &ud->desc_to_purge); 492 spin_unlock_irqrestore(&ud->lock, flags); 493 494 schedule_work(&ud->purge_work); 495 } 496 497 static bool udma_is_chan_running(struct udma_chan *uc) 498 { 499 u32 trt_ctl = 0; 500 u32 rrt_ctl = 0; 501 502 if (uc->tchan) 503 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 504 if (uc->rchan) 505 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG); 506 507 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 508 return true; 509 510 return false; 511 } 512 513 static bool udma_is_chan_paused(struct udma_chan *uc) 514 { 515 u32 val, pause_mask; 516 517 switch (uc->config.dir) { 518 case DMA_DEV_TO_MEM: 519 val = udma_rchanrt_read(uc->rchan, 520 UDMA_RCHAN_RT_PEER_RT_EN_REG); 521 pause_mask = UDMA_PEER_RT_EN_PAUSE; 522 break; 523 case DMA_MEM_TO_DEV: 524 val = udma_tchanrt_read(uc->tchan, 525 UDMA_TCHAN_RT_PEER_RT_EN_REG); 526 pause_mask = UDMA_PEER_RT_EN_PAUSE; 527 break; 528 case DMA_MEM_TO_MEM: 529 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 530 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 531 break; 532 default: 533 return false; 534 } 535 536 if (val & pause_mask) 537 return true; 538 539 return false; 540 } 541 542 static void udma_sync_for_device(struct udma_chan *uc, int idx) 543 { 544 struct udma_desc *d = uc->desc; 545 546 if (uc->cyclic && uc->config.pkt_mode) { 547 dma_sync_single_for_device(uc->ud->dev, 548 d->hwdesc[idx].cppi5_desc_paddr, 549 d->hwdesc[idx].cppi5_desc_size, 550 DMA_TO_DEVICE); 551 } else { 552 int i; 553 554 for (i = 0; i < d->hwdesc_count; i++) { 555 if (!d->hwdesc[i].cppi5_desc_vaddr) 556 continue; 557 558 dma_sync_single_for_device(uc->ud->dev, 559 d->hwdesc[i].cppi5_desc_paddr, 560 d->hwdesc[i].cppi5_desc_size, 561 DMA_TO_DEVICE); 562 } 563 } 564 } 565 566 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) 567 { 568 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; 569 } 570 571 static int udma_push_to_ring(struct udma_chan *uc, int idx) 572 { 573 struct udma_desc *d = uc->desc; 574 struct k3_ring *ring = NULL; 575 dma_addr_t paddr; 576 577 switch (uc->config.dir) { 578 case DMA_DEV_TO_MEM: 579 ring = uc->rflow->fd_ring; 580 break; 581 case DMA_MEM_TO_DEV: 582 case DMA_MEM_TO_MEM: 583 ring = uc->tchan->t_ring; 584 break; 585 default: 586 return -EINVAL; 587 } 588 589 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ 590 if (idx == -1) { 591 paddr = udma_get_rx_flush_hwdesc_paddr(uc); 592 } else { 593 paddr = udma_curr_cppi5_desc_paddr(d, idx); 594 595 wmb(); /* Ensure that writes are not moved over this point */ 596 udma_sync_for_device(uc, idx); 597 } 598 599 return k3_ringacc_ring_push(ring, &paddr); 600 } 601 602 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) 603 { 604 if (uc->config.dir != DMA_DEV_TO_MEM) 605 return false; 606 607 if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) 608 return true; 609 610 return false; 611 } 612 613 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 614 { 615 struct k3_ring *ring = NULL; 616 int ret = -ENOENT; 617 618 switch (uc->config.dir) { 619 case DMA_DEV_TO_MEM: 620 ring = uc->rflow->r_ring; 621 break; 622 case DMA_MEM_TO_DEV: 623 case DMA_MEM_TO_MEM: 624 ring = uc->tchan->tc_ring; 625 break; 626 default: 627 break; 628 } 629 630 if (ring && k3_ringacc_ring_get_occ(ring)) { 631 struct udma_desc *d = NULL; 632 633 ret = k3_ringacc_ring_pop(ring, addr); 634 if (ret) 635 return ret; 636 637 /* Teardown completion */ 638 if (cppi5_desc_is_tdcm(*addr)) 639 return ret; 640 641 /* Check for flush descriptor */ 642 if (udma_desc_is_rx_flush(uc, *addr)) 643 return -ENOENT; 644 645 d = udma_udma_desc_from_paddr(uc, *addr); 646 647 if (d) 648 dma_sync_single_for_cpu(uc->ud->dev, *addr, 649 d->hwdesc[0].cppi5_desc_size, 650 DMA_FROM_DEVICE); 651 rmb(); /* Ensure that reads are not moved before this point */ 652 } 653 654 return ret; 655 } 656 657 static void udma_reset_rings(struct udma_chan *uc) 658 { 659 struct k3_ring *ring1 = NULL; 660 struct k3_ring *ring2 = NULL; 661 662 switch (uc->config.dir) { 663 case DMA_DEV_TO_MEM: 664 if (uc->rchan) { 665 ring1 = uc->rflow->fd_ring; 666 ring2 = uc->rflow->r_ring; 667 } 668 break; 669 case DMA_MEM_TO_DEV: 670 case DMA_MEM_TO_MEM: 671 if (uc->tchan) { 672 ring1 = uc->tchan->t_ring; 673 ring2 = uc->tchan->tc_ring; 674 } 675 break; 676 default: 677 break; 678 } 679 680 if (ring1) 681 k3_ringacc_ring_reset_dma(ring1, 682 k3_ringacc_ring_get_occ(ring1)); 683 if (ring2) 684 k3_ringacc_ring_reset(ring2); 685 686 /* make sure we are not leaking memory by stalled descriptor */ 687 if (uc->terminated_desc) { 688 udma_desc_free(&uc->terminated_desc->vd); 689 uc->terminated_desc = NULL; 690 } 691 } 692 693 static void udma_reset_counters(struct udma_chan *uc) 694 { 695 u32 val; 696 697 if (uc->tchan) { 698 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 699 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val); 700 701 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG); 702 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val); 703 704 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG); 705 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val); 706 707 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 708 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val); 709 } 710 711 if (uc->rchan) { 712 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG); 713 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val); 714 715 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG); 716 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val); 717 718 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG); 719 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val); 720 721 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG); 722 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val); 723 } 724 725 uc->bcnt = 0; 726 } 727 728 static int udma_reset_chan(struct udma_chan *uc, bool hard) 729 { 730 switch (uc->config.dir) { 731 case DMA_DEV_TO_MEM: 732 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0); 733 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 734 break; 735 case DMA_MEM_TO_DEV: 736 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 737 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 738 break; 739 case DMA_MEM_TO_MEM: 740 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 741 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 742 break; 743 default: 744 return -EINVAL; 745 } 746 747 /* Reset all counters */ 748 udma_reset_counters(uc); 749 750 /* Hard reset: re-initialize the channel to reset */ 751 if (hard) { 752 struct udma_chan_config ucc_backup; 753 int ret; 754 755 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 756 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 757 758 /* restore the channel configuration */ 759 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 760 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 761 if (ret) 762 return ret; 763 764 /* 765 * Setting forced teardown after forced reset helps recovering 766 * the rchan. 767 */ 768 if (uc->config.dir == DMA_DEV_TO_MEM) 769 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 770 UDMA_CHAN_RT_CTL_EN | 771 UDMA_CHAN_RT_CTL_TDOWN | 772 UDMA_CHAN_RT_CTL_FTDOWN); 773 } 774 uc->state = UDMA_CHAN_IS_IDLE; 775 776 return 0; 777 } 778 779 static void udma_start_desc(struct udma_chan *uc) 780 { 781 struct udma_chan_config *ucc = &uc->config; 782 783 if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 784 int i; 785 786 /* Push all descriptors to ring for packet mode cyclic or RX */ 787 for (i = 0; i < uc->desc->sglen; i++) 788 udma_push_to_ring(uc, i); 789 } else { 790 udma_push_to_ring(uc, 0); 791 } 792 } 793 794 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 795 { 796 /* Only PDMAs have staticTR */ 797 if (uc->config.ep_type == PSIL_EP_NATIVE) 798 return false; 799 800 /* Check if the staticTR configuration has changed for TX */ 801 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 802 return true; 803 804 return false; 805 } 806 807 static int udma_start(struct udma_chan *uc) 808 { 809 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 810 811 if (!vd) { 812 uc->desc = NULL; 813 return -ENOENT; 814 } 815 816 list_del(&vd->node); 817 818 uc->desc = to_udma_desc(&vd->tx); 819 820 /* Channel is already running and does not need reconfiguration */ 821 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 822 udma_start_desc(uc); 823 goto out; 824 } 825 826 /* Make sure that we clear the teardown bit, if it is set */ 827 udma_reset_chan(uc, false); 828 829 /* Push descriptors before we start the channel */ 830 udma_start_desc(uc); 831 832 switch (uc->desc->dir) { 833 case DMA_DEV_TO_MEM: 834 /* Config remote TR */ 835 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 836 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 837 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 838 const struct udma_match_data *match_data = 839 uc->ud->match_data; 840 841 if (uc->config.enable_acc32) 842 val |= PDMA_STATIC_TR_XY_ACC32; 843 if (uc->config.enable_burst) 844 val |= PDMA_STATIC_TR_XY_BURST; 845 846 udma_rchanrt_write(uc->rchan, 847 UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG, val); 848 849 udma_rchanrt_write(uc->rchan, 850 UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG, 851 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 852 match_data->statictr_z_mask)); 853 854 /* save the current staticTR configuration */ 855 memcpy(&uc->static_tr, &uc->desc->static_tr, 856 sizeof(uc->static_tr)); 857 } 858 859 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 860 UDMA_CHAN_RT_CTL_EN); 861 862 /* Enable remote */ 863 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 864 UDMA_PEER_RT_EN_ENABLE); 865 866 break; 867 case DMA_MEM_TO_DEV: 868 /* Config remote TR */ 869 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 870 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 871 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 872 873 if (uc->config.enable_acc32) 874 val |= PDMA_STATIC_TR_XY_ACC32; 875 if (uc->config.enable_burst) 876 val |= PDMA_STATIC_TR_XY_BURST; 877 878 udma_tchanrt_write(uc->tchan, 879 UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG, val); 880 881 /* save the current staticTR configuration */ 882 memcpy(&uc->static_tr, &uc->desc->static_tr, 883 sizeof(uc->static_tr)); 884 } 885 886 /* Enable remote */ 887 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 888 UDMA_PEER_RT_EN_ENABLE); 889 890 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 891 UDMA_CHAN_RT_CTL_EN); 892 893 break; 894 case DMA_MEM_TO_MEM: 895 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 896 UDMA_CHAN_RT_CTL_EN); 897 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 898 UDMA_CHAN_RT_CTL_EN); 899 900 break; 901 default: 902 return -EINVAL; 903 } 904 905 uc->state = UDMA_CHAN_IS_ACTIVE; 906 out: 907 908 return 0; 909 } 910 911 static int udma_stop(struct udma_chan *uc) 912 { 913 enum udma_chan_state old_state = uc->state; 914 915 uc->state = UDMA_CHAN_IS_TERMINATING; 916 reinit_completion(&uc->teardown_completed); 917 918 switch (uc->config.dir) { 919 case DMA_DEV_TO_MEM: 920 if (!uc->cyclic && !uc->desc) 921 udma_push_to_ring(uc, -1); 922 923 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 924 UDMA_PEER_RT_EN_ENABLE | 925 UDMA_PEER_RT_EN_TEARDOWN); 926 break; 927 case DMA_MEM_TO_DEV: 928 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 929 UDMA_PEER_RT_EN_ENABLE | 930 UDMA_PEER_RT_EN_FLUSH); 931 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 932 UDMA_CHAN_RT_CTL_EN | 933 UDMA_CHAN_RT_CTL_TDOWN); 934 break; 935 case DMA_MEM_TO_MEM: 936 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 937 UDMA_CHAN_RT_CTL_EN | 938 UDMA_CHAN_RT_CTL_TDOWN); 939 break; 940 default: 941 uc->state = old_state; 942 complete_all(&uc->teardown_completed); 943 return -EINVAL; 944 } 945 946 return 0; 947 } 948 949 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 950 { 951 struct udma_desc *d = uc->desc; 952 struct cppi5_host_desc_t *h_desc; 953 954 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 955 cppi5_hdesc_reset_to_original(h_desc); 956 udma_push_to_ring(uc, d->desc_idx); 957 d->desc_idx = (d->desc_idx + 1) % d->sglen; 958 } 959 960 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 961 { 962 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 963 964 memcpy(d->metadata, h_desc->epib, d->metadata_size); 965 } 966 967 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 968 { 969 u32 peer_bcnt, bcnt; 970 971 /* Only TX towards PDMA is affected */ 972 if (uc->config.ep_type == PSIL_EP_NATIVE || 973 uc->config.dir != DMA_MEM_TO_DEV) 974 return true; 975 976 peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 977 bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 978 979 /* Transfer is incomplete, store current residue and time stamp */ 980 if (peer_bcnt < bcnt) { 981 uc->tx_drain.residue = bcnt - peer_bcnt; 982 uc->tx_drain.tstamp = ktime_get(); 983 return false; 984 } 985 986 return true; 987 } 988 989 static void udma_check_tx_completion(struct work_struct *work) 990 { 991 struct udma_chan *uc = container_of(work, typeof(*uc), 992 tx_drain.work.work); 993 bool desc_done = true; 994 u32 residue_diff; 995 ktime_t time_diff; 996 unsigned long delay; 997 998 while (1) { 999 if (uc->desc) { 1000 /* Get previous residue and time stamp */ 1001 residue_diff = uc->tx_drain.residue; 1002 time_diff = uc->tx_drain.tstamp; 1003 /* 1004 * Get current residue and time stamp or see if 1005 * transfer is complete 1006 */ 1007 desc_done = udma_is_desc_really_done(uc, uc->desc); 1008 } 1009 1010 if (!desc_done) { 1011 /* 1012 * Find the time delta and residue delta w.r.t 1013 * previous poll 1014 */ 1015 time_diff = ktime_sub(uc->tx_drain.tstamp, 1016 time_diff) + 1; 1017 residue_diff -= uc->tx_drain.residue; 1018 if (residue_diff) { 1019 /* 1020 * Try to guess when we should check 1021 * next time by calculating rate at 1022 * which data is being drained at the 1023 * peer device 1024 */ 1025 delay = (time_diff / residue_diff) * 1026 uc->tx_drain.residue; 1027 } else { 1028 /* No progress, check again in 1 second */ 1029 schedule_delayed_work(&uc->tx_drain.work, HZ); 1030 break; 1031 } 1032 1033 usleep_range(ktime_to_us(delay), 1034 ktime_to_us(delay) + 10); 1035 continue; 1036 } 1037 1038 if (uc->desc) { 1039 struct udma_desc *d = uc->desc; 1040 1041 uc->bcnt += d->residue; 1042 udma_start(uc); 1043 vchan_cookie_complete(&d->vd); 1044 break; 1045 } 1046 1047 break; 1048 } 1049 } 1050 1051 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 1052 { 1053 struct udma_chan *uc = data; 1054 struct udma_desc *d; 1055 unsigned long flags; 1056 dma_addr_t paddr = 0; 1057 1058 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1059 return IRQ_HANDLED; 1060 1061 spin_lock_irqsave(&uc->vc.lock, flags); 1062 1063 /* Teardown completion message */ 1064 if (cppi5_desc_is_tdcm(paddr)) { 1065 complete_all(&uc->teardown_completed); 1066 1067 if (uc->terminated_desc) { 1068 udma_desc_free(&uc->terminated_desc->vd); 1069 uc->terminated_desc = NULL; 1070 } 1071 1072 if (!uc->desc) 1073 udma_start(uc); 1074 1075 goto out; 1076 } 1077 1078 d = udma_udma_desc_from_paddr(uc, paddr); 1079 1080 if (d) { 1081 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1082 d->desc_idx); 1083 if (desc_paddr != paddr) { 1084 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1085 goto out; 1086 } 1087 1088 if (d == uc->desc) { 1089 /* active descriptor */ 1090 if (uc->cyclic) { 1091 udma_cyclic_packet_elapsed(uc); 1092 vchan_cyclic_callback(&d->vd); 1093 } else { 1094 if (udma_is_desc_really_done(uc, d)) { 1095 uc->bcnt += d->residue; 1096 udma_start(uc); 1097 vchan_cookie_complete(&d->vd); 1098 } else { 1099 schedule_delayed_work(&uc->tx_drain.work, 1100 0); 1101 } 1102 } 1103 } else { 1104 /* 1105 * terminated descriptor, mark the descriptor as 1106 * completed to update the channel's cookie marker 1107 */ 1108 dma_cookie_complete(&d->vd.tx); 1109 } 1110 } 1111 out: 1112 spin_unlock_irqrestore(&uc->vc.lock, flags); 1113 1114 return IRQ_HANDLED; 1115 } 1116 1117 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1118 { 1119 struct udma_chan *uc = data; 1120 struct udma_desc *d; 1121 unsigned long flags; 1122 1123 spin_lock_irqsave(&uc->vc.lock, flags); 1124 d = uc->desc; 1125 if (d) { 1126 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1127 1128 if (uc->cyclic) { 1129 vchan_cyclic_callback(&d->vd); 1130 } else { 1131 /* TODO: figure out the real amount of data */ 1132 uc->bcnt += d->residue; 1133 udma_start(uc); 1134 vchan_cookie_complete(&d->vd); 1135 } 1136 } 1137 1138 spin_unlock_irqrestore(&uc->vc.lock, flags); 1139 1140 return IRQ_HANDLED; 1141 } 1142 1143 /** 1144 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1145 * @ud: UDMA device 1146 * @from: Start the search from this flow id number 1147 * @cnt: Number of consecutive flow ids to allocate 1148 * 1149 * Allocate range of RX flow ids for future use, those flows can be requested 1150 * only using explicit flow id number. if @from is set to -1 it will try to find 1151 * first free range. if @from is positive value it will force allocation only 1152 * of the specified range of flows. 1153 * 1154 * Returns -ENOMEM if can't find free range. 1155 * -EEXIST if requested range is busy. 1156 * -EINVAL if wrong input values passed. 1157 * Returns flow id on success. 1158 */ 1159 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1160 { 1161 int start, tmp_from; 1162 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1163 1164 tmp_from = from; 1165 if (tmp_from < 0) 1166 tmp_from = ud->rchan_cnt; 1167 /* default flows can't be allocated and accessible only by id */ 1168 if (tmp_from < ud->rchan_cnt) 1169 return -EINVAL; 1170 1171 if (tmp_from + cnt > ud->rflow_cnt) 1172 return -EINVAL; 1173 1174 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1175 ud->rflow_cnt); 1176 1177 start = bitmap_find_next_zero_area(tmp, 1178 ud->rflow_cnt, 1179 tmp_from, cnt, 0); 1180 if (start >= ud->rflow_cnt) 1181 return -ENOMEM; 1182 1183 if (from >= 0 && start != from) 1184 return -EEXIST; 1185 1186 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1187 return start; 1188 } 1189 1190 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1191 { 1192 if (from < ud->rchan_cnt) 1193 return -EINVAL; 1194 if (from + cnt > ud->rflow_cnt) 1195 return -EINVAL; 1196 1197 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1198 return 0; 1199 } 1200 1201 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1202 { 1203 /* 1204 * Attempt to request rflow by ID can be made for any rflow 1205 * if not in use with assumption that caller knows what's doing. 1206 * TI-SCI FW will perform additional permission check ant way, it's 1207 * safe 1208 */ 1209 1210 if (id < 0 || id >= ud->rflow_cnt) 1211 return ERR_PTR(-ENOENT); 1212 1213 if (test_bit(id, ud->rflow_in_use)) 1214 return ERR_PTR(-ENOENT); 1215 1216 /* GP rflow has to be allocated first */ 1217 if (!test_bit(id, ud->rflow_gp_map) && 1218 !test_bit(id, ud->rflow_gp_map_allocated)) 1219 return ERR_PTR(-EINVAL); 1220 1221 dev_dbg(ud->dev, "get rflow%d\n", id); 1222 set_bit(id, ud->rflow_in_use); 1223 return &ud->rflows[id]; 1224 } 1225 1226 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1227 { 1228 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1229 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1230 return; 1231 } 1232 1233 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1234 clear_bit(rflow->id, ud->rflow_in_use); 1235 } 1236 1237 #define UDMA_RESERVE_RESOURCE(res) \ 1238 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1239 enum udma_tp_level tpl, \ 1240 int id) \ 1241 { \ 1242 if (id >= 0) { \ 1243 if (test_bit(id, ud->res##_map)) { \ 1244 dev_err(ud->dev, "res##%d is in use\n", id); \ 1245 return ERR_PTR(-ENOENT); \ 1246 } \ 1247 } else { \ 1248 int start; \ 1249 \ 1250 if (tpl >= ud->match_data->tpl_levels) \ 1251 tpl = ud->match_data->tpl_levels - 1; \ 1252 \ 1253 start = ud->match_data->level_start_idx[tpl]; \ 1254 \ 1255 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1256 start); \ 1257 if (id == ud->res##_cnt) { \ 1258 return ERR_PTR(-ENOENT); \ 1259 } \ 1260 } \ 1261 \ 1262 set_bit(id, ud->res##_map); \ 1263 return &ud->res##s[id]; \ 1264 } 1265 1266 UDMA_RESERVE_RESOURCE(tchan); 1267 UDMA_RESERVE_RESOURCE(rchan); 1268 1269 static int udma_get_tchan(struct udma_chan *uc) 1270 { 1271 struct udma_dev *ud = uc->ud; 1272 1273 if (uc->tchan) { 1274 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1275 uc->id, uc->tchan->id); 1276 return 0; 1277 } 1278 1279 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); 1280 1281 return PTR_ERR_OR_ZERO(uc->tchan); 1282 } 1283 1284 static int udma_get_rchan(struct udma_chan *uc) 1285 { 1286 struct udma_dev *ud = uc->ud; 1287 1288 if (uc->rchan) { 1289 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1290 uc->id, uc->rchan->id); 1291 return 0; 1292 } 1293 1294 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); 1295 1296 return PTR_ERR_OR_ZERO(uc->rchan); 1297 } 1298 1299 static int udma_get_chan_pair(struct udma_chan *uc) 1300 { 1301 struct udma_dev *ud = uc->ud; 1302 const struct udma_match_data *match_data = ud->match_data; 1303 int chan_id, end; 1304 1305 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1306 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1307 uc->id, uc->tchan->id); 1308 return 0; 1309 } 1310 1311 if (uc->tchan) { 1312 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1313 uc->id, uc->tchan->id); 1314 return -EBUSY; 1315 } else if (uc->rchan) { 1316 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1317 uc->id, uc->rchan->id); 1318 return -EBUSY; 1319 } 1320 1321 /* Can be optimized, but let's have it like this for now */ 1322 end = min(ud->tchan_cnt, ud->rchan_cnt); 1323 /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ 1324 chan_id = match_data->level_start_idx[match_data->tpl_levels - 1]; 1325 for (; chan_id < end; chan_id++) { 1326 if (!test_bit(chan_id, ud->tchan_map) && 1327 !test_bit(chan_id, ud->rchan_map)) 1328 break; 1329 } 1330 1331 if (chan_id == end) 1332 return -ENOENT; 1333 1334 set_bit(chan_id, ud->tchan_map); 1335 set_bit(chan_id, ud->rchan_map); 1336 uc->tchan = &ud->tchans[chan_id]; 1337 uc->rchan = &ud->rchans[chan_id]; 1338 1339 return 0; 1340 } 1341 1342 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1343 { 1344 struct udma_dev *ud = uc->ud; 1345 1346 if (!uc->rchan) { 1347 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1348 return -EINVAL; 1349 } 1350 1351 if (uc->rflow) { 1352 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1353 uc->id, uc->rflow->id); 1354 return 0; 1355 } 1356 1357 uc->rflow = __udma_get_rflow(ud, flow_id); 1358 1359 return PTR_ERR_OR_ZERO(uc->rflow); 1360 } 1361 1362 static void udma_put_rchan(struct udma_chan *uc) 1363 { 1364 struct udma_dev *ud = uc->ud; 1365 1366 if (uc->rchan) { 1367 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1368 uc->rchan->id); 1369 clear_bit(uc->rchan->id, ud->rchan_map); 1370 uc->rchan = NULL; 1371 } 1372 } 1373 1374 static void udma_put_tchan(struct udma_chan *uc) 1375 { 1376 struct udma_dev *ud = uc->ud; 1377 1378 if (uc->tchan) { 1379 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1380 uc->tchan->id); 1381 clear_bit(uc->tchan->id, ud->tchan_map); 1382 uc->tchan = NULL; 1383 } 1384 } 1385 1386 static void udma_put_rflow(struct udma_chan *uc) 1387 { 1388 struct udma_dev *ud = uc->ud; 1389 1390 if (uc->rflow) { 1391 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1392 uc->rflow->id); 1393 __udma_put_rflow(ud, uc->rflow); 1394 uc->rflow = NULL; 1395 } 1396 } 1397 1398 static void udma_free_tx_resources(struct udma_chan *uc) 1399 { 1400 if (!uc->tchan) 1401 return; 1402 1403 k3_ringacc_ring_free(uc->tchan->t_ring); 1404 k3_ringacc_ring_free(uc->tchan->tc_ring); 1405 uc->tchan->t_ring = NULL; 1406 uc->tchan->tc_ring = NULL; 1407 1408 udma_put_tchan(uc); 1409 } 1410 1411 static int udma_alloc_tx_resources(struct udma_chan *uc) 1412 { 1413 struct k3_ring_cfg ring_cfg; 1414 struct udma_dev *ud = uc->ud; 1415 int ret; 1416 1417 ret = udma_get_tchan(uc); 1418 if (ret) 1419 return ret; 1420 1421 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->tchan->id, -1, 1422 &uc->tchan->t_ring, 1423 &uc->tchan->tc_ring); 1424 if (ret) { 1425 ret = -EBUSY; 1426 goto err_ring; 1427 } 1428 1429 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1430 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1431 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1432 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1433 1434 ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg); 1435 ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg); 1436 1437 if (ret) 1438 goto err_ringcfg; 1439 1440 return 0; 1441 1442 err_ringcfg: 1443 k3_ringacc_ring_free(uc->tchan->tc_ring); 1444 uc->tchan->tc_ring = NULL; 1445 k3_ringacc_ring_free(uc->tchan->t_ring); 1446 uc->tchan->t_ring = NULL; 1447 err_ring: 1448 udma_put_tchan(uc); 1449 1450 return ret; 1451 } 1452 1453 static void udma_free_rx_resources(struct udma_chan *uc) 1454 { 1455 if (!uc->rchan) 1456 return; 1457 1458 if (uc->rflow) { 1459 struct udma_rflow *rflow = uc->rflow; 1460 1461 k3_ringacc_ring_free(rflow->fd_ring); 1462 k3_ringacc_ring_free(rflow->r_ring); 1463 rflow->fd_ring = NULL; 1464 rflow->r_ring = NULL; 1465 1466 udma_put_rflow(uc); 1467 } 1468 1469 udma_put_rchan(uc); 1470 } 1471 1472 static int udma_alloc_rx_resources(struct udma_chan *uc) 1473 { 1474 struct udma_dev *ud = uc->ud; 1475 struct k3_ring_cfg ring_cfg; 1476 struct udma_rflow *rflow; 1477 int fd_ring_id; 1478 int ret; 1479 1480 ret = udma_get_rchan(uc); 1481 if (ret) 1482 return ret; 1483 1484 /* For MEM_TO_MEM we don't need rflow or rings */ 1485 if (uc->config.dir == DMA_MEM_TO_MEM) 1486 return 0; 1487 1488 ret = udma_get_rflow(uc, uc->rchan->id); 1489 if (ret) { 1490 ret = -EBUSY; 1491 goto err_rflow; 1492 } 1493 1494 rflow = uc->rflow; 1495 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; 1496 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, 1497 &rflow->fd_ring, &rflow->r_ring); 1498 if (ret) { 1499 ret = -EBUSY; 1500 goto err_ring; 1501 } 1502 1503 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1504 1505 if (uc->config.pkt_mode) 1506 ring_cfg.size = SG_MAX_SEGMENTS; 1507 else 1508 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1509 1510 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1511 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1512 1513 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1514 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1515 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1516 1517 if (ret) 1518 goto err_ringcfg; 1519 1520 return 0; 1521 1522 err_ringcfg: 1523 k3_ringacc_ring_free(rflow->r_ring); 1524 rflow->r_ring = NULL; 1525 k3_ringacc_ring_free(rflow->fd_ring); 1526 rflow->fd_ring = NULL; 1527 err_ring: 1528 udma_put_rflow(uc); 1529 err_rflow: 1530 udma_put_rchan(uc); 1531 1532 return ret; 1533 } 1534 1535 #define TISCI_TCHAN_VALID_PARAMS ( \ 1536 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1537 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1538 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1539 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1540 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1541 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1542 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1543 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1544 1545 #define TISCI_RCHAN_VALID_PARAMS ( \ 1546 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1547 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1548 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1549 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1550 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1551 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1552 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1553 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1554 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1555 1556 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1557 { 1558 struct udma_dev *ud = uc->ud; 1559 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1560 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1561 struct udma_tchan *tchan = uc->tchan; 1562 struct udma_rchan *rchan = uc->rchan; 1563 int ret = 0; 1564 1565 /* Non synchronized - mem to mem type of transfer */ 1566 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1567 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1568 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1569 1570 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1571 req_tx.nav_id = tisci_rm->tisci_dev_id; 1572 req_tx.index = tchan->id; 1573 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1574 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1575 req_tx.txcq_qnum = tc_ring; 1576 req_tx.tx_atype = ud->atype; 1577 1578 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1579 if (ret) { 1580 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1581 return ret; 1582 } 1583 1584 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1585 req_rx.nav_id = tisci_rm->tisci_dev_id; 1586 req_rx.index = rchan->id; 1587 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1588 req_rx.rxcq_qnum = tc_ring; 1589 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1590 req_rx.rx_atype = ud->atype; 1591 1592 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1593 if (ret) 1594 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1595 1596 return ret; 1597 } 1598 1599 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1600 { 1601 struct udma_dev *ud = uc->ud; 1602 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1603 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1604 struct udma_tchan *tchan = uc->tchan; 1605 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1606 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1607 u32 mode, fetch_size; 1608 int ret = 0; 1609 1610 if (uc->config.pkt_mode) { 1611 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1612 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1613 uc->config.psd_size, 0); 1614 } else { 1615 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1616 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1617 } 1618 1619 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1620 req_tx.nav_id = tisci_rm->tisci_dev_id; 1621 req_tx.index = tchan->id; 1622 req_tx.tx_chan_type = mode; 1623 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1624 req_tx.tx_fetch_size = fetch_size >> 2; 1625 req_tx.txcq_qnum = tc_ring; 1626 req_tx.tx_atype = uc->config.atype; 1627 1628 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1629 if (ret) 1630 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1631 1632 return ret; 1633 } 1634 1635 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 1636 { 1637 struct udma_dev *ud = uc->ud; 1638 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1639 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1640 struct udma_rchan *rchan = uc->rchan; 1641 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 1642 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1643 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1644 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 1645 u32 mode, fetch_size; 1646 int ret = 0; 1647 1648 if (uc->config.pkt_mode) { 1649 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1650 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1651 uc->config.psd_size, 0); 1652 } else { 1653 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1654 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1655 } 1656 1657 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1658 req_rx.nav_id = tisci_rm->tisci_dev_id; 1659 req_rx.index = rchan->id; 1660 req_rx.rx_fetch_size = fetch_size >> 2; 1661 req_rx.rxcq_qnum = rx_ring; 1662 req_rx.rx_chan_type = mode; 1663 req_rx.rx_atype = uc->config.atype; 1664 1665 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1666 if (ret) { 1667 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 1668 return ret; 1669 } 1670 1671 flow_req.valid_params = 1672 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 1673 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 1674 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 1675 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 1676 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1677 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 1678 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 1679 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 1680 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 1681 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1682 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1683 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1684 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1685 1686 flow_req.nav_id = tisci_rm->tisci_dev_id; 1687 flow_req.flow_index = rchan->id; 1688 1689 if (uc->config.needs_epib) 1690 flow_req.rx_einfo_present = 1; 1691 else 1692 flow_req.rx_einfo_present = 0; 1693 if (uc->config.psd_size) 1694 flow_req.rx_psinfo_present = 1; 1695 else 1696 flow_req.rx_psinfo_present = 0; 1697 flow_req.rx_error_handling = 1; 1698 flow_req.rx_dest_qnum = rx_ring; 1699 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 1700 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 1701 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 1702 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 1703 flow_req.rx_fdq0_sz0_qnum = fd_ring; 1704 flow_req.rx_fdq1_qnum = fd_ring; 1705 flow_req.rx_fdq2_qnum = fd_ring; 1706 flow_req.rx_fdq3_qnum = fd_ring; 1707 1708 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 1709 1710 if (ret) 1711 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 1712 1713 return 0; 1714 } 1715 1716 static int udma_alloc_chan_resources(struct dma_chan *chan) 1717 { 1718 struct udma_chan *uc = to_udma_chan(chan); 1719 struct udma_dev *ud = to_udma_dev(chan->device); 1720 const struct udma_match_data *match_data = ud->match_data; 1721 struct k3_ring *irq_ring; 1722 u32 irq_udma_idx; 1723 int ret; 1724 1725 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 1726 uc->use_dma_pool = true; 1727 /* in case of MEM_TO_MEM we have maximum of two TRs */ 1728 if (uc->config.dir == DMA_MEM_TO_MEM) { 1729 uc->config.hdesc_size = cppi5_trdesc_calc_size( 1730 sizeof(struct cppi5_tr_type15_t), 2); 1731 uc->config.pkt_mode = false; 1732 } 1733 } 1734 1735 if (uc->use_dma_pool) { 1736 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 1737 uc->config.hdesc_size, 1738 ud->desc_align, 1739 0); 1740 if (!uc->hdesc_pool) { 1741 dev_err(ud->ddev.dev, 1742 "Descriptor pool allocation failed\n"); 1743 uc->use_dma_pool = false; 1744 ret = -ENOMEM; 1745 goto err_cleanup; 1746 } 1747 } 1748 1749 /* 1750 * Make sure that the completion is in a known state: 1751 * No teardown, the channel is idle 1752 */ 1753 reinit_completion(&uc->teardown_completed); 1754 complete_all(&uc->teardown_completed); 1755 uc->state = UDMA_CHAN_IS_IDLE; 1756 1757 switch (uc->config.dir) { 1758 case DMA_MEM_TO_MEM: 1759 /* Non synchronized - mem to mem type of transfer */ 1760 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 1761 uc->id); 1762 1763 ret = udma_get_chan_pair(uc); 1764 if (ret) 1765 goto err_cleanup; 1766 1767 ret = udma_alloc_tx_resources(uc); 1768 if (ret) { 1769 udma_put_rchan(uc); 1770 goto err_cleanup; 1771 } 1772 1773 ret = udma_alloc_rx_resources(uc); 1774 if (ret) { 1775 udma_free_tx_resources(uc); 1776 goto err_cleanup; 1777 } 1778 1779 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1780 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1781 K3_PSIL_DST_THREAD_ID_OFFSET; 1782 1783 irq_ring = uc->tchan->tc_ring; 1784 irq_udma_idx = uc->tchan->id; 1785 1786 ret = udma_tisci_m2m_channel_config(uc); 1787 break; 1788 case DMA_MEM_TO_DEV: 1789 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 1790 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 1791 uc->id); 1792 1793 ret = udma_alloc_tx_resources(uc); 1794 if (ret) 1795 goto err_cleanup; 1796 1797 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1798 uc->config.dst_thread = uc->config.remote_thread_id; 1799 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 1800 1801 irq_ring = uc->tchan->tc_ring; 1802 irq_udma_idx = uc->tchan->id; 1803 1804 ret = udma_tisci_tx_channel_config(uc); 1805 break; 1806 case DMA_DEV_TO_MEM: 1807 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 1808 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 1809 uc->id); 1810 1811 ret = udma_alloc_rx_resources(uc); 1812 if (ret) 1813 goto err_cleanup; 1814 1815 uc->config.src_thread = uc->config.remote_thread_id; 1816 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1817 K3_PSIL_DST_THREAD_ID_OFFSET; 1818 1819 irq_ring = uc->rflow->r_ring; 1820 irq_udma_idx = match_data->rchan_oes_offset + uc->rchan->id; 1821 1822 ret = udma_tisci_rx_channel_config(uc); 1823 break; 1824 default: 1825 /* Can not happen */ 1826 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 1827 __func__, uc->id, uc->config.dir); 1828 ret = -EINVAL; 1829 goto err_cleanup; 1830 1831 } 1832 1833 /* check if the channel configuration was successful */ 1834 if (ret) 1835 goto err_res_free; 1836 1837 if (udma_is_chan_running(uc)) { 1838 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 1839 udma_reset_chan(uc, false); 1840 if (udma_is_chan_running(uc)) { 1841 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 1842 ret = -EBUSY; 1843 goto err_res_free; 1844 } 1845 } 1846 1847 /* PSI-L pairing */ 1848 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 1849 if (ret) { 1850 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 1851 uc->config.src_thread, uc->config.dst_thread); 1852 goto err_res_free; 1853 } 1854 1855 uc->psil_paired = true; 1856 1857 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 1858 if (uc->irq_num_ring <= 0) { 1859 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 1860 k3_ringacc_get_ring_id(irq_ring)); 1861 ret = -EINVAL; 1862 goto err_psi_free; 1863 } 1864 1865 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 1866 IRQF_TRIGGER_HIGH, uc->name, uc); 1867 if (ret) { 1868 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 1869 goto err_irq_free; 1870 } 1871 1872 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 1873 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 1874 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, 1875 irq_udma_idx); 1876 if (uc->irq_num_udma <= 0) { 1877 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 1878 irq_udma_idx); 1879 free_irq(uc->irq_num_ring, uc); 1880 ret = -EINVAL; 1881 goto err_irq_free; 1882 } 1883 1884 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 1885 uc->name, uc); 1886 if (ret) { 1887 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 1888 uc->id); 1889 free_irq(uc->irq_num_ring, uc); 1890 goto err_irq_free; 1891 } 1892 } else { 1893 uc->irq_num_udma = 0; 1894 } 1895 1896 udma_reset_rings(uc); 1897 1898 return 0; 1899 1900 err_irq_free: 1901 uc->irq_num_ring = 0; 1902 uc->irq_num_udma = 0; 1903 err_psi_free: 1904 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 1905 uc->psil_paired = false; 1906 err_res_free: 1907 udma_free_tx_resources(uc); 1908 udma_free_rx_resources(uc); 1909 err_cleanup: 1910 udma_reset_uchan(uc); 1911 1912 if (uc->use_dma_pool) { 1913 dma_pool_destroy(uc->hdesc_pool); 1914 uc->use_dma_pool = false; 1915 } 1916 1917 return ret; 1918 } 1919 1920 static int udma_slave_config(struct dma_chan *chan, 1921 struct dma_slave_config *cfg) 1922 { 1923 struct udma_chan *uc = to_udma_chan(chan); 1924 1925 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 1926 1927 return 0; 1928 } 1929 1930 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 1931 size_t tr_size, int tr_count, 1932 enum dma_transfer_direction dir) 1933 { 1934 struct udma_hwdesc *hwdesc; 1935 struct cppi5_desc_hdr_t *tr_desc; 1936 struct udma_desc *d; 1937 u32 reload_count = 0; 1938 u32 ring_id; 1939 1940 switch (tr_size) { 1941 case 16: 1942 case 32: 1943 case 64: 1944 case 128: 1945 break; 1946 default: 1947 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 1948 return NULL; 1949 } 1950 1951 /* We have only one descriptor containing multiple TRs */ 1952 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 1953 if (!d) 1954 return NULL; 1955 1956 d->sglen = tr_count; 1957 1958 d->hwdesc_count = 1; 1959 hwdesc = &d->hwdesc[0]; 1960 1961 /* Allocate memory for DMA ring descriptor */ 1962 if (uc->use_dma_pool) { 1963 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 1964 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 1965 GFP_NOWAIT, 1966 &hwdesc->cppi5_desc_paddr); 1967 } else { 1968 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1969 tr_count); 1970 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 1971 uc->ud->desc_align); 1972 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 1973 hwdesc->cppi5_desc_size, 1974 &hwdesc->cppi5_desc_paddr, 1975 GFP_NOWAIT); 1976 } 1977 1978 if (!hwdesc->cppi5_desc_vaddr) { 1979 kfree(d); 1980 return NULL; 1981 } 1982 1983 /* Start of the TR req records */ 1984 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 1985 /* Start address of the TR response array */ 1986 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 1987 1988 tr_desc = hwdesc->cppi5_desc_vaddr; 1989 1990 if (uc->cyclic) 1991 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 1992 1993 if (dir == DMA_DEV_TO_MEM) 1994 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1995 else 1996 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 1997 1998 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 1999 cppi5_desc_set_pktids(tr_desc, uc->id, 2000 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2001 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 2002 2003 return d; 2004 } 2005 2006 /** 2007 * udma_get_tr_counters - calculate TR counters for a given length 2008 * @len: Length of the trasnfer 2009 * @align_to: Preferred alignment 2010 * @tr0_cnt0: First TR icnt0 2011 * @tr0_cnt1: First TR icnt1 2012 * @tr1_cnt0: Second (if used) TR icnt0 2013 * 2014 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated 2015 * For len >= SZ_64K two TRs are used in a simple way: 2016 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) 2017 * Second TR: the remaining length (tr1_cnt0) 2018 * 2019 * Returns the number of TRs the length needs (1 or 2) 2020 * -EINVAL if the length can not be supported 2021 */ 2022 static int udma_get_tr_counters(size_t len, unsigned long align_to, 2023 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) 2024 { 2025 if (len < SZ_64K) { 2026 *tr0_cnt0 = len; 2027 *tr0_cnt1 = 1; 2028 2029 return 1; 2030 } 2031 2032 if (align_to > 3) 2033 align_to = 3; 2034 2035 realign: 2036 *tr0_cnt0 = SZ_64K - BIT(align_to); 2037 if (len / *tr0_cnt0 >= SZ_64K) { 2038 if (align_to) { 2039 align_to--; 2040 goto realign; 2041 } 2042 return -EINVAL; 2043 } 2044 2045 *tr0_cnt1 = len / *tr0_cnt0; 2046 *tr1_cnt0 = len % *tr0_cnt0; 2047 2048 return 2; 2049 } 2050 2051 static struct udma_desc * 2052 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 2053 unsigned int sglen, enum dma_transfer_direction dir, 2054 unsigned long tx_flags, void *context) 2055 { 2056 struct scatterlist *sgent; 2057 struct udma_desc *d; 2058 struct cppi5_tr_type1_t *tr_req = NULL; 2059 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2060 unsigned int i; 2061 size_t tr_size; 2062 int num_tr = 0; 2063 int tr_idx = 0; 2064 2065 if (!is_slave_direction(dir)) { 2066 dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); 2067 return NULL; 2068 } 2069 2070 /* estimate the number of TRs we will need */ 2071 for_each_sg(sgl, sgent, sglen, i) { 2072 if (sg_dma_len(sgent) < SZ_64K) 2073 num_tr++; 2074 else 2075 num_tr += 2; 2076 } 2077 2078 /* Now allocate and setup the descriptor. */ 2079 tr_size = sizeof(struct cppi5_tr_type1_t); 2080 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 2081 if (!d) 2082 return NULL; 2083 2084 d->sglen = sglen; 2085 2086 tr_req = d->hwdesc[0].tr_req_base; 2087 for_each_sg(sgl, sgent, sglen, i) { 2088 dma_addr_t sg_addr = sg_dma_address(sgent); 2089 2090 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), 2091 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); 2092 if (num_tr < 0) { 2093 dev_err(uc->ud->dev, "size %u is not supported\n", 2094 sg_dma_len(sgent)); 2095 udma_free_hwdesc(uc, d); 2096 kfree(d); 2097 return NULL; 2098 } 2099 2100 cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, 2101 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2102 cppi5_tr_csf_set(&tr_req[i].flags, CPPI5_TR_CSF_SUPR_EVT); 2103 2104 tr_req[tr_idx].addr = sg_addr; 2105 tr_req[tr_idx].icnt0 = tr0_cnt0; 2106 tr_req[tr_idx].icnt1 = tr0_cnt1; 2107 tr_req[tr_idx].dim1 = tr0_cnt0; 2108 tr_idx++; 2109 2110 if (num_tr == 2) { 2111 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2112 false, false, 2113 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2114 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2115 CPPI5_TR_CSF_SUPR_EVT); 2116 2117 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; 2118 tr_req[tr_idx].icnt0 = tr1_cnt0; 2119 tr_req[tr_idx].icnt1 = 1; 2120 tr_req[tr_idx].dim1 = tr1_cnt0; 2121 tr_idx++; 2122 } 2123 2124 d->residue += sg_dma_len(sgent); 2125 } 2126 2127 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, 2128 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2129 2130 return d; 2131 } 2132 2133 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 2134 enum dma_slave_buswidth dev_width, 2135 u16 elcnt) 2136 { 2137 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 2138 return 0; 2139 2140 /* Bus width translates to the element size (ES) */ 2141 switch (dev_width) { 2142 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2143 d->static_tr.elsize = 0; 2144 break; 2145 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2146 d->static_tr.elsize = 1; 2147 break; 2148 case DMA_SLAVE_BUSWIDTH_3_BYTES: 2149 d->static_tr.elsize = 2; 2150 break; 2151 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2152 d->static_tr.elsize = 3; 2153 break; 2154 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2155 d->static_tr.elsize = 4; 2156 break; 2157 default: /* not reached */ 2158 return -EINVAL; 2159 } 2160 2161 d->static_tr.elcnt = elcnt; 2162 2163 /* 2164 * PDMA must to close the packet when the channel is in packet mode. 2165 * For TR mode when the channel is not cyclic we also need PDMA to close 2166 * the packet otherwise the transfer will stall because PDMA holds on 2167 * the data it has received from the peripheral. 2168 */ 2169 if (uc->config.pkt_mode || !uc->cyclic) { 2170 unsigned int div = dev_width * elcnt; 2171 2172 if (uc->cyclic) 2173 d->static_tr.bstcnt = d->residue / d->sglen / div; 2174 else 2175 d->static_tr.bstcnt = d->residue / div; 2176 2177 if (uc->config.dir == DMA_DEV_TO_MEM && 2178 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 2179 return -EINVAL; 2180 } else { 2181 d->static_tr.bstcnt = 0; 2182 } 2183 2184 return 0; 2185 } 2186 2187 static struct udma_desc * 2188 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 2189 unsigned int sglen, enum dma_transfer_direction dir, 2190 unsigned long tx_flags, void *context) 2191 { 2192 struct scatterlist *sgent; 2193 struct cppi5_host_desc_t *h_desc = NULL; 2194 struct udma_desc *d; 2195 u32 ring_id; 2196 unsigned int i; 2197 2198 d = kzalloc(sizeof(*d) + sglen * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2199 if (!d) 2200 return NULL; 2201 2202 d->sglen = sglen; 2203 d->hwdesc_count = sglen; 2204 2205 if (dir == DMA_DEV_TO_MEM) 2206 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2207 else 2208 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2209 2210 for_each_sg(sgl, sgent, sglen, i) { 2211 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2212 dma_addr_t sg_addr = sg_dma_address(sgent); 2213 struct cppi5_host_desc_t *desc; 2214 size_t sg_len = sg_dma_len(sgent); 2215 2216 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2217 GFP_NOWAIT, 2218 &hwdesc->cppi5_desc_paddr); 2219 if (!hwdesc->cppi5_desc_vaddr) { 2220 dev_err(uc->ud->dev, 2221 "descriptor%d allocation failed\n", i); 2222 2223 udma_free_hwdesc(uc, d); 2224 kfree(d); 2225 return NULL; 2226 } 2227 2228 d->residue += sg_len; 2229 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2230 desc = hwdesc->cppi5_desc_vaddr; 2231 2232 if (i == 0) { 2233 cppi5_hdesc_init(desc, 0, 0); 2234 /* Flow and Packed ID */ 2235 cppi5_desc_set_pktids(&desc->hdr, uc->id, 2236 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2237 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 2238 } else { 2239 cppi5_hdesc_reset_hbdesc(desc); 2240 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 2241 } 2242 2243 /* attach the sg buffer to the descriptor */ 2244 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 2245 2246 /* Attach link as host buffer descriptor */ 2247 if (h_desc) 2248 cppi5_hdesc_link_hbdesc(h_desc, 2249 hwdesc->cppi5_desc_paddr); 2250 2251 if (dir == DMA_MEM_TO_DEV) 2252 h_desc = desc; 2253 } 2254 2255 if (d->residue >= SZ_4M) { 2256 dev_err(uc->ud->dev, 2257 "%s: Transfer size %u is over the supported 4M range\n", 2258 __func__, d->residue); 2259 udma_free_hwdesc(uc, d); 2260 kfree(d); 2261 return NULL; 2262 } 2263 2264 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2265 cppi5_hdesc_set_pktlen(h_desc, d->residue); 2266 2267 return d; 2268 } 2269 2270 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 2271 void *data, size_t len) 2272 { 2273 struct udma_desc *d = to_udma_desc(desc); 2274 struct udma_chan *uc = to_udma_chan(desc->chan); 2275 struct cppi5_host_desc_t *h_desc; 2276 u32 psd_size = len; 2277 u32 flags = 0; 2278 2279 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2280 return -ENOTSUPP; 2281 2282 if (!data || len > uc->config.metadata_size) 2283 return -EINVAL; 2284 2285 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2286 return -EINVAL; 2287 2288 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2289 if (d->dir == DMA_MEM_TO_DEV) 2290 memcpy(h_desc->epib, data, len); 2291 2292 if (uc->config.needs_epib) 2293 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2294 2295 d->metadata = data; 2296 d->metadata_size = len; 2297 if (uc->config.needs_epib) 2298 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2299 2300 cppi5_hdesc_update_flags(h_desc, flags); 2301 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2302 2303 return 0; 2304 } 2305 2306 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 2307 size_t *payload_len, size_t *max_len) 2308 { 2309 struct udma_desc *d = to_udma_desc(desc); 2310 struct udma_chan *uc = to_udma_chan(desc->chan); 2311 struct cppi5_host_desc_t *h_desc; 2312 2313 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2314 return ERR_PTR(-ENOTSUPP); 2315 2316 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2317 2318 *max_len = uc->config.metadata_size; 2319 2320 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 2321 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 2322 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 2323 2324 return h_desc->epib; 2325 } 2326 2327 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 2328 size_t payload_len) 2329 { 2330 struct udma_desc *d = to_udma_desc(desc); 2331 struct udma_chan *uc = to_udma_chan(desc->chan); 2332 struct cppi5_host_desc_t *h_desc; 2333 u32 psd_size = payload_len; 2334 u32 flags = 0; 2335 2336 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2337 return -ENOTSUPP; 2338 2339 if (payload_len > uc->config.metadata_size) 2340 return -EINVAL; 2341 2342 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2343 return -EINVAL; 2344 2345 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2346 2347 if (uc->config.needs_epib) { 2348 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2349 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2350 } 2351 2352 cppi5_hdesc_update_flags(h_desc, flags); 2353 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2354 2355 return 0; 2356 } 2357 2358 static struct dma_descriptor_metadata_ops metadata_ops = { 2359 .attach = udma_attach_metadata, 2360 .get_ptr = udma_get_metadata_ptr, 2361 .set_len = udma_set_metadata_len, 2362 }; 2363 2364 static struct dma_async_tx_descriptor * 2365 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2366 unsigned int sglen, enum dma_transfer_direction dir, 2367 unsigned long tx_flags, void *context) 2368 { 2369 struct udma_chan *uc = to_udma_chan(chan); 2370 enum dma_slave_buswidth dev_width; 2371 struct udma_desc *d; 2372 u32 burst; 2373 2374 if (dir != uc->config.dir) { 2375 dev_err(chan->device->dev, 2376 "%s: chan%d is for %s, not supporting %s\n", 2377 __func__, uc->id, 2378 dmaengine_get_direction_text(uc->config.dir), 2379 dmaengine_get_direction_text(dir)); 2380 return NULL; 2381 } 2382 2383 if (dir == DMA_DEV_TO_MEM) { 2384 dev_width = uc->cfg.src_addr_width; 2385 burst = uc->cfg.src_maxburst; 2386 } else if (dir == DMA_MEM_TO_DEV) { 2387 dev_width = uc->cfg.dst_addr_width; 2388 burst = uc->cfg.dst_maxburst; 2389 } else { 2390 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 2391 return NULL; 2392 } 2393 2394 if (!burst) 2395 burst = 1; 2396 2397 if (uc->config.pkt_mode) 2398 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 2399 context); 2400 else 2401 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 2402 context); 2403 2404 if (!d) 2405 return NULL; 2406 2407 d->dir = dir; 2408 d->desc_idx = 0; 2409 d->tr_idx = 0; 2410 2411 /* static TR for remote PDMA */ 2412 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2413 dev_err(uc->ud->dev, 2414 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2415 __func__, d->static_tr.bstcnt); 2416 2417 udma_free_hwdesc(uc, d); 2418 kfree(d); 2419 return NULL; 2420 } 2421 2422 if (uc->config.metadata_size) 2423 d->vd.tx.metadata_ops = &metadata_ops; 2424 2425 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2426 } 2427 2428 static struct udma_desc * 2429 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 2430 size_t buf_len, size_t period_len, 2431 enum dma_transfer_direction dir, unsigned long flags) 2432 { 2433 struct udma_desc *d; 2434 size_t tr_size, period_addr; 2435 struct cppi5_tr_type1_t *tr_req; 2436 unsigned int periods = buf_len / period_len; 2437 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2438 unsigned int i; 2439 int num_tr; 2440 2441 if (!is_slave_direction(dir)) { 2442 dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); 2443 return NULL; 2444 } 2445 2446 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, 2447 &tr0_cnt1, &tr1_cnt0); 2448 if (num_tr < 0) { 2449 dev_err(uc->ud->dev, "size %zu is not supported\n", 2450 period_len); 2451 return NULL; 2452 } 2453 2454 /* Now allocate and setup the descriptor. */ 2455 tr_size = sizeof(struct cppi5_tr_type1_t); 2456 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); 2457 if (!d) 2458 return NULL; 2459 2460 tr_req = d->hwdesc[0].tr_req_base; 2461 period_addr = buf_addr; 2462 for (i = 0; i < periods; i++) { 2463 int tr_idx = i * num_tr; 2464 2465 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2466 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2467 2468 tr_req[tr_idx].addr = period_addr; 2469 tr_req[tr_idx].icnt0 = tr0_cnt0; 2470 tr_req[tr_idx].icnt1 = tr0_cnt1; 2471 tr_req[tr_idx].dim1 = tr0_cnt0; 2472 2473 if (num_tr == 2) { 2474 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2475 CPPI5_TR_CSF_SUPR_EVT); 2476 tr_idx++; 2477 2478 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2479 false, false, 2480 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2481 2482 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; 2483 tr_req[tr_idx].icnt0 = tr1_cnt0; 2484 tr_req[tr_idx].icnt1 = 1; 2485 tr_req[tr_idx].dim1 = tr1_cnt0; 2486 } 2487 2488 if (!(flags & DMA_PREP_INTERRUPT)) 2489 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2490 CPPI5_TR_CSF_SUPR_EVT); 2491 2492 period_addr += period_len; 2493 } 2494 2495 return d; 2496 } 2497 2498 static struct udma_desc * 2499 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 2500 size_t buf_len, size_t period_len, 2501 enum dma_transfer_direction dir, unsigned long flags) 2502 { 2503 struct udma_desc *d; 2504 u32 ring_id; 2505 int i; 2506 int periods = buf_len / period_len; 2507 2508 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 2509 return NULL; 2510 2511 if (period_len >= SZ_4M) 2512 return NULL; 2513 2514 d = kzalloc(sizeof(*d) + periods * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2515 if (!d) 2516 return NULL; 2517 2518 d->hwdesc_count = periods; 2519 2520 /* TODO: re-check this... */ 2521 if (dir == DMA_DEV_TO_MEM) 2522 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2523 else 2524 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2525 2526 for (i = 0; i < periods; i++) { 2527 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2528 dma_addr_t period_addr = buf_addr + (period_len * i); 2529 struct cppi5_host_desc_t *h_desc; 2530 2531 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2532 GFP_NOWAIT, 2533 &hwdesc->cppi5_desc_paddr); 2534 if (!hwdesc->cppi5_desc_vaddr) { 2535 dev_err(uc->ud->dev, 2536 "descriptor%d allocation failed\n", i); 2537 2538 udma_free_hwdesc(uc, d); 2539 kfree(d); 2540 return NULL; 2541 } 2542 2543 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2544 h_desc = hwdesc->cppi5_desc_vaddr; 2545 2546 cppi5_hdesc_init(h_desc, 0, 0); 2547 cppi5_hdesc_set_pktlen(h_desc, period_len); 2548 2549 /* Flow and Packed ID */ 2550 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 2551 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2552 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 2553 2554 /* attach each period to a new descriptor */ 2555 cppi5_hdesc_attach_buf(h_desc, 2556 period_addr, period_len, 2557 period_addr, period_len); 2558 } 2559 2560 return d; 2561 } 2562 2563 static struct dma_async_tx_descriptor * 2564 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2565 size_t period_len, enum dma_transfer_direction dir, 2566 unsigned long flags) 2567 { 2568 struct udma_chan *uc = to_udma_chan(chan); 2569 enum dma_slave_buswidth dev_width; 2570 struct udma_desc *d; 2571 u32 burst; 2572 2573 if (dir != uc->config.dir) { 2574 dev_err(chan->device->dev, 2575 "%s: chan%d is for %s, not supporting %s\n", 2576 __func__, uc->id, 2577 dmaengine_get_direction_text(uc->config.dir), 2578 dmaengine_get_direction_text(dir)); 2579 return NULL; 2580 } 2581 2582 uc->cyclic = true; 2583 2584 if (dir == DMA_DEV_TO_MEM) { 2585 dev_width = uc->cfg.src_addr_width; 2586 burst = uc->cfg.src_maxburst; 2587 } else if (dir == DMA_MEM_TO_DEV) { 2588 dev_width = uc->cfg.dst_addr_width; 2589 burst = uc->cfg.dst_maxburst; 2590 } else { 2591 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2592 return NULL; 2593 } 2594 2595 if (!burst) 2596 burst = 1; 2597 2598 if (uc->config.pkt_mode) 2599 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 2600 dir, flags); 2601 else 2602 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 2603 dir, flags); 2604 2605 if (!d) 2606 return NULL; 2607 2608 d->sglen = buf_len / period_len; 2609 2610 d->dir = dir; 2611 d->residue = buf_len; 2612 2613 /* static TR for remote PDMA */ 2614 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2615 dev_err(uc->ud->dev, 2616 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2617 __func__, d->static_tr.bstcnt); 2618 2619 udma_free_hwdesc(uc, d); 2620 kfree(d); 2621 return NULL; 2622 } 2623 2624 if (uc->config.metadata_size) 2625 d->vd.tx.metadata_ops = &metadata_ops; 2626 2627 return vchan_tx_prep(&uc->vc, &d->vd, flags); 2628 } 2629 2630 static struct dma_async_tx_descriptor * 2631 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 2632 size_t len, unsigned long tx_flags) 2633 { 2634 struct udma_chan *uc = to_udma_chan(chan); 2635 struct udma_desc *d; 2636 struct cppi5_tr_type15_t *tr_req; 2637 int num_tr; 2638 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 2639 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2640 2641 if (uc->config.dir != DMA_MEM_TO_MEM) { 2642 dev_err(chan->device->dev, 2643 "%s: chan%d is for %s, not supporting %s\n", 2644 __func__, uc->id, 2645 dmaengine_get_direction_text(uc->config.dir), 2646 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 2647 return NULL; 2648 } 2649 2650 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, 2651 &tr0_cnt1, &tr1_cnt0); 2652 if (num_tr < 0) { 2653 dev_err(uc->ud->dev, "size %zu is not supported\n", 2654 len); 2655 return NULL; 2656 } 2657 2658 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 2659 if (!d) 2660 return NULL; 2661 2662 d->dir = DMA_MEM_TO_MEM; 2663 d->desc_idx = 0; 2664 d->tr_idx = 0; 2665 d->residue = len; 2666 2667 tr_req = d->hwdesc[0].tr_req_base; 2668 2669 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 2670 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2671 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); 2672 2673 tr_req[0].addr = src; 2674 tr_req[0].icnt0 = tr0_cnt0; 2675 tr_req[0].icnt1 = tr0_cnt1; 2676 tr_req[0].icnt2 = 1; 2677 tr_req[0].icnt3 = 1; 2678 tr_req[0].dim1 = tr0_cnt0; 2679 2680 tr_req[0].daddr = dest; 2681 tr_req[0].dicnt0 = tr0_cnt0; 2682 tr_req[0].dicnt1 = tr0_cnt1; 2683 tr_req[0].dicnt2 = 1; 2684 tr_req[0].dicnt3 = 1; 2685 tr_req[0].ddim1 = tr0_cnt0; 2686 2687 if (num_tr == 2) { 2688 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 2689 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2690 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); 2691 2692 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 2693 tr_req[1].icnt0 = tr1_cnt0; 2694 tr_req[1].icnt1 = 1; 2695 tr_req[1].icnt2 = 1; 2696 tr_req[1].icnt3 = 1; 2697 2698 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 2699 tr_req[1].dicnt0 = tr1_cnt0; 2700 tr_req[1].dicnt1 = 1; 2701 tr_req[1].dicnt2 = 1; 2702 tr_req[1].dicnt3 = 1; 2703 } 2704 2705 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, 2706 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2707 2708 if (uc->config.metadata_size) 2709 d->vd.tx.metadata_ops = &metadata_ops; 2710 2711 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2712 } 2713 2714 static void udma_issue_pending(struct dma_chan *chan) 2715 { 2716 struct udma_chan *uc = to_udma_chan(chan); 2717 unsigned long flags; 2718 2719 spin_lock_irqsave(&uc->vc.lock, flags); 2720 2721 /* If we have something pending and no active descriptor, then */ 2722 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 2723 /* 2724 * start a descriptor if the channel is NOT [marked as 2725 * terminating _and_ it is still running (teardown has not 2726 * completed yet)]. 2727 */ 2728 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 2729 udma_is_chan_running(uc))) 2730 udma_start(uc); 2731 } 2732 2733 spin_unlock_irqrestore(&uc->vc.lock, flags); 2734 } 2735 2736 static enum dma_status udma_tx_status(struct dma_chan *chan, 2737 dma_cookie_t cookie, 2738 struct dma_tx_state *txstate) 2739 { 2740 struct udma_chan *uc = to_udma_chan(chan); 2741 enum dma_status ret; 2742 unsigned long flags; 2743 2744 spin_lock_irqsave(&uc->vc.lock, flags); 2745 2746 ret = dma_cookie_status(chan, cookie, txstate); 2747 2748 if (!udma_is_chan_running(uc)) 2749 ret = DMA_COMPLETE; 2750 2751 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 2752 ret = DMA_PAUSED; 2753 2754 if (ret == DMA_COMPLETE || !txstate) 2755 goto out; 2756 2757 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 2758 u32 peer_bcnt = 0; 2759 u32 bcnt = 0; 2760 u32 residue = uc->desc->residue; 2761 u32 delay = 0; 2762 2763 if (uc->desc->dir == DMA_MEM_TO_DEV) { 2764 bcnt = udma_tchanrt_read(uc->tchan, 2765 UDMA_TCHAN_RT_SBCNT_REG); 2766 2767 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2768 peer_bcnt = udma_tchanrt_read(uc->tchan, 2769 UDMA_TCHAN_RT_PEER_BCNT_REG); 2770 2771 if (bcnt > peer_bcnt) 2772 delay = bcnt - peer_bcnt; 2773 } 2774 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 2775 bcnt = udma_rchanrt_read(uc->rchan, 2776 UDMA_RCHAN_RT_BCNT_REG); 2777 2778 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2779 peer_bcnt = udma_rchanrt_read(uc->rchan, 2780 UDMA_RCHAN_RT_PEER_BCNT_REG); 2781 2782 if (peer_bcnt > bcnt) 2783 delay = peer_bcnt - bcnt; 2784 } 2785 } else { 2786 bcnt = udma_tchanrt_read(uc->tchan, 2787 UDMA_TCHAN_RT_BCNT_REG); 2788 } 2789 2790 bcnt -= uc->bcnt; 2791 if (bcnt && !(bcnt % uc->desc->residue)) 2792 residue = 0; 2793 else 2794 residue -= bcnt % uc->desc->residue; 2795 2796 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 2797 ret = DMA_COMPLETE; 2798 delay = 0; 2799 } 2800 2801 dma_set_residue(txstate, residue); 2802 dma_set_in_flight_bytes(txstate, delay); 2803 2804 } else { 2805 ret = DMA_COMPLETE; 2806 } 2807 2808 out: 2809 spin_unlock_irqrestore(&uc->vc.lock, flags); 2810 return ret; 2811 } 2812 2813 static int udma_pause(struct dma_chan *chan) 2814 { 2815 struct udma_chan *uc = to_udma_chan(chan); 2816 2817 /* pause the channel */ 2818 switch (uc->config.dir) { 2819 case DMA_DEV_TO_MEM: 2820 udma_rchanrt_update_bits(uc->rchan, 2821 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2822 UDMA_PEER_RT_EN_PAUSE, 2823 UDMA_PEER_RT_EN_PAUSE); 2824 break; 2825 case DMA_MEM_TO_DEV: 2826 udma_tchanrt_update_bits(uc->tchan, 2827 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2828 UDMA_PEER_RT_EN_PAUSE, 2829 UDMA_PEER_RT_EN_PAUSE); 2830 break; 2831 case DMA_MEM_TO_MEM: 2832 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2833 UDMA_CHAN_RT_CTL_PAUSE, 2834 UDMA_CHAN_RT_CTL_PAUSE); 2835 break; 2836 default: 2837 return -EINVAL; 2838 } 2839 2840 return 0; 2841 } 2842 2843 static int udma_resume(struct dma_chan *chan) 2844 { 2845 struct udma_chan *uc = to_udma_chan(chan); 2846 2847 /* resume the channel */ 2848 switch (uc->config.dir) { 2849 case DMA_DEV_TO_MEM: 2850 udma_rchanrt_update_bits(uc->rchan, 2851 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2852 UDMA_PEER_RT_EN_PAUSE, 0); 2853 2854 break; 2855 case DMA_MEM_TO_DEV: 2856 udma_tchanrt_update_bits(uc->tchan, 2857 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2858 UDMA_PEER_RT_EN_PAUSE, 0); 2859 break; 2860 case DMA_MEM_TO_MEM: 2861 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2862 UDMA_CHAN_RT_CTL_PAUSE, 0); 2863 break; 2864 default: 2865 return -EINVAL; 2866 } 2867 2868 return 0; 2869 } 2870 2871 static int udma_terminate_all(struct dma_chan *chan) 2872 { 2873 struct udma_chan *uc = to_udma_chan(chan); 2874 unsigned long flags; 2875 LIST_HEAD(head); 2876 2877 spin_lock_irqsave(&uc->vc.lock, flags); 2878 2879 if (udma_is_chan_running(uc)) 2880 udma_stop(uc); 2881 2882 if (uc->desc) { 2883 uc->terminated_desc = uc->desc; 2884 uc->desc = NULL; 2885 uc->terminated_desc->terminated = true; 2886 cancel_delayed_work(&uc->tx_drain.work); 2887 } 2888 2889 uc->paused = false; 2890 2891 vchan_get_all_descriptors(&uc->vc, &head); 2892 spin_unlock_irqrestore(&uc->vc.lock, flags); 2893 vchan_dma_desc_free_list(&uc->vc, &head); 2894 2895 return 0; 2896 } 2897 2898 static void udma_synchronize(struct dma_chan *chan) 2899 { 2900 struct udma_chan *uc = to_udma_chan(chan); 2901 unsigned long timeout = msecs_to_jiffies(1000); 2902 2903 vchan_synchronize(&uc->vc); 2904 2905 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 2906 timeout = wait_for_completion_timeout(&uc->teardown_completed, 2907 timeout); 2908 if (!timeout) { 2909 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 2910 uc->id); 2911 udma_dump_chan_stdata(uc); 2912 udma_reset_chan(uc, true); 2913 } 2914 } 2915 2916 udma_reset_chan(uc, false); 2917 if (udma_is_chan_running(uc)) 2918 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 2919 2920 cancel_delayed_work_sync(&uc->tx_drain.work); 2921 udma_reset_rings(uc); 2922 } 2923 2924 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 2925 struct virt_dma_desc *vd, 2926 struct dmaengine_result *result) 2927 { 2928 struct udma_chan *uc = to_udma_chan(&vc->chan); 2929 struct udma_desc *d; 2930 2931 if (!vd) 2932 return; 2933 2934 d = to_udma_desc(&vd->tx); 2935 2936 if (d->metadata_size) 2937 udma_fetch_epib(uc, d); 2938 2939 /* Provide residue information for the client */ 2940 if (result) { 2941 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 2942 2943 if (cppi5_desc_get_type(desc_vaddr) == 2944 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 2945 result->residue = d->residue - 2946 cppi5_hdesc_get_pktlen(desc_vaddr); 2947 if (result->residue) 2948 result->result = DMA_TRANS_ABORTED; 2949 else 2950 result->result = DMA_TRANS_NOERROR; 2951 } else { 2952 result->residue = 0; 2953 result->result = DMA_TRANS_NOERROR; 2954 } 2955 } 2956 } 2957 2958 /* 2959 * This tasklet handles the completion of a DMA descriptor by 2960 * calling its callback and freeing it. 2961 */ 2962 static void udma_vchan_complete(unsigned long arg) 2963 { 2964 struct virt_dma_chan *vc = (struct virt_dma_chan *)arg; 2965 struct virt_dma_desc *vd, *_vd; 2966 struct dmaengine_desc_callback cb; 2967 LIST_HEAD(head); 2968 2969 spin_lock_irq(&vc->lock); 2970 list_splice_tail_init(&vc->desc_completed, &head); 2971 vd = vc->cyclic; 2972 if (vd) { 2973 vc->cyclic = NULL; 2974 dmaengine_desc_get_callback(&vd->tx, &cb); 2975 } else { 2976 memset(&cb, 0, sizeof(cb)); 2977 } 2978 spin_unlock_irq(&vc->lock); 2979 2980 udma_desc_pre_callback(vc, vd, NULL); 2981 dmaengine_desc_callback_invoke(&cb, NULL); 2982 2983 list_for_each_entry_safe(vd, _vd, &head, node) { 2984 struct dmaengine_result result; 2985 2986 dmaengine_desc_get_callback(&vd->tx, &cb); 2987 2988 list_del(&vd->node); 2989 2990 udma_desc_pre_callback(vc, vd, &result); 2991 dmaengine_desc_callback_invoke(&cb, &result); 2992 2993 vchan_vdesc_fini(vd); 2994 } 2995 } 2996 2997 static void udma_free_chan_resources(struct dma_chan *chan) 2998 { 2999 struct udma_chan *uc = to_udma_chan(chan); 3000 struct udma_dev *ud = to_udma_dev(chan->device); 3001 3002 udma_terminate_all(chan); 3003 if (uc->terminated_desc) { 3004 udma_reset_chan(uc, false); 3005 udma_reset_rings(uc); 3006 } 3007 3008 cancel_delayed_work_sync(&uc->tx_drain.work); 3009 3010 if (uc->irq_num_ring > 0) { 3011 free_irq(uc->irq_num_ring, uc); 3012 3013 uc->irq_num_ring = 0; 3014 } 3015 if (uc->irq_num_udma > 0) { 3016 free_irq(uc->irq_num_udma, uc); 3017 3018 uc->irq_num_udma = 0; 3019 } 3020 3021 /* Release PSI-L pairing */ 3022 if (uc->psil_paired) { 3023 navss_psil_unpair(ud, uc->config.src_thread, 3024 uc->config.dst_thread); 3025 uc->psil_paired = false; 3026 } 3027 3028 vchan_free_chan_resources(&uc->vc); 3029 tasklet_kill(&uc->vc.task); 3030 3031 udma_free_tx_resources(uc); 3032 udma_free_rx_resources(uc); 3033 udma_reset_uchan(uc); 3034 3035 if (uc->use_dma_pool) { 3036 dma_pool_destroy(uc->hdesc_pool); 3037 uc->use_dma_pool = false; 3038 } 3039 } 3040 3041 static struct platform_driver udma_driver; 3042 3043 struct udma_filter_param { 3044 int remote_thread_id; 3045 u32 atype; 3046 }; 3047 3048 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 3049 { 3050 struct udma_chan_config *ucc; 3051 struct psil_endpoint_config *ep_config; 3052 struct udma_filter_param *filter_param; 3053 struct udma_chan *uc; 3054 struct udma_dev *ud; 3055 3056 if (chan->device->dev->driver != &udma_driver.driver) 3057 return false; 3058 3059 uc = to_udma_chan(chan); 3060 ucc = &uc->config; 3061 ud = uc->ud; 3062 filter_param = param; 3063 3064 if (filter_param->atype > 2) { 3065 dev_err(ud->dev, "Invalid channel atype: %u\n", 3066 filter_param->atype); 3067 return false; 3068 } 3069 3070 ucc->remote_thread_id = filter_param->remote_thread_id; 3071 ucc->atype = filter_param->atype; 3072 3073 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) 3074 ucc->dir = DMA_MEM_TO_DEV; 3075 else 3076 ucc->dir = DMA_DEV_TO_MEM; 3077 3078 ep_config = psil_get_ep_config(ucc->remote_thread_id); 3079 if (IS_ERR(ep_config)) { 3080 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 3081 ucc->remote_thread_id); 3082 ucc->dir = DMA_MEM_TO_MEM; 3083 ucc->remote_thread_id = -1; 3084 ucc->atype = 0; 3085 return false; 3086 } 3087 3088 ucc->pkt_mode = ep_config->pkt_mode; 3089 ucc->channel_tpl = ep_config->channel_tpl; 3090 ucc->notdpkt = ep_config->notdpkt; 3091 ucc->ep_type = ep_config->ep_type; 3092 3093 if (ucc->ep_type != PSIL_EP_NATIVE) { 3094 const struct udma_match_data *match_data = ud->match_data; 3095 3096 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 3097 ucc->enable_acc32 = ep_config->pdma_acc32; 3098 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 3099 ucc->enable_burst = ep_config->pdma_burst; 3100 } 3101 3102 ucc->needs_epib = ep_config->needs_epib; 3103 ucc->psd_size = ep_config->psd_size; 3104 ucc->metadata_size = 3105 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 3106 ucc->psd_size; 3107 3108 if (ucc->pkt_mode) 3109 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3110 ucc->metadata_size, ud->desc_align); 3111 3112 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 3113 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 3114 3115 return true; 3116 } 3117 3118 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 3119 struct of_dma *ofdma) 3120 { 3121 struct udma_dev *ud = ofdma->of_dma_data; 3122 dma_cap_mask_t mask = ud->ddev.cap_mask; 3123 struct udma_filter_param filter_param; 3124 struct dma_chan *chan; 3125 3126 if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 3127 return NULL; 3128 3129 filter_param.remote_thread_id = dma_spec->args[0]; 3130 if (dma_spec->args_count == 2) 3131 filter_param.atype = dma_spec->args[1]; 3132 else 3133 filter_param.atype = 0; 3134 3135 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 3136 ofdma->of_node); 3137 if (!chan) { 3138 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 3139 return ERR_PTR(-EINVAL); 3140 } 3141 3142 return chan; 3143 } 3144 3145 static struct udma_match_data am654_main_data = { 3146 .psil_base = 0x1000, 3147 .enable_memcpy_support = true, 3148 .statictr_z_mask = GENMASK(11, 0), 3149 .rchan_oes_offset = 0x2000, 3150 .tpl_levels = 2, 3151 .level_start_idx = { 3152 [0] = 8, /* Normal channels */ 3153 [1] = 0, /* High Throughput channels */ 3154 }, 3155 }; 3156 3157 static struct udma_match_data am654_mcu_data = { 3158 .psil_base = 0x6000, 3159 .enable_memcpy_support = false, 3160 .statictr_z_mask = GENMASK(11, 0), 3161 .rchan_oes_offset = 0x2000, 3162 .tpl_levels = 2, 3163 .level_start_idx = { 3164 [0] = 2, /* Normal channels */ 3165 [1] = 0, /* High Throughput channels */ 3166 }, 3167 }; 3168 3169 static struct udma_match_data j721e_main_data = { 3170 .psil_base = 0x1000, 3171 .enable_memcpy_support = true, 3172 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3173 .statictr_z_mask = GENMASK(23, 0), 3174 .rchan_oes_offset = 0x400, 3175 .tpl_levels = 3, 3176 .level_start_idx = { 3177 [0] = 16, /* Normal channels */ 3178 [1] = 4, /* High Throughput channels */ 3179 [2] = 0, /* Ultra High Throughput channels */ 3180 }, 3181 }; 3182 3183 static struct udma_match_data j721e_mcu_data = { 3184 .psil_base = 0x6000, 3185 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 3186 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3187 .statictr_z_mask = GENMASK(23, 0), 3188 .rchan_oes_offset = 0x400, 3189 .tpl_levels = 2, 3190 .level_start_idx = { 3191 [0] = 2, /* Normal channels */ 3192 [1] = 0, /* High Throughput channels */ 3193 }, 3194 }; 3195 3196 static const struct of_device_id udma_of_match[] = { 3197 { 3198 .compatible = "ti,am654-navss-main-udmap", 3199 .data = &am654_main_data, 3200 }, 3201 { 3202 .compatible = "ti,am654-navss-mcu-udmap", 3203 .data = &am654_mcu_data, 3204 }, { 3205 .compatible = "ti,j721e-navss-main-udmap", 3206 .data = &j721e_main_data, 3207 }, { 3208 .compatible = "ti,j721e-navss-mcu-udmap", 3209 .data = &j721e_mcu_data, 3210 }, 3211 { /* Sentinel */ }, 3212 }; 3213 3214 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 3215 { 3216 struct resource *res; 3217 int i; 3218 3219 for (i = 0; i < MMR_LAST; i++) { 3220 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 3221 mmr_names[i]); 3222 ud->mmrs[i] = devm_ioremap_resource(&pdev->dev, res); 3223 if (IS_ERR(ud->mmrs[i])) 3224 return PTR_ERR(ud->mmrs[i]); 3225 } 3226 3227 return 0; 3228 } 3229 3230 static int udma_setup_resources(struct udma_dev *ud) 3231 { 3232 struct device *dev = ud->dev; 3233 int ch_count, ret, i, j; 3234 u32 cap2, cap3; 3235 struct ti_sci_resource_desc *rm_desc; 3236 struct ti_sci_resource *rm_res, irq_res; 3237 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 3238 static const char * const range_names[] = { "ti,sci-rm-range-tchan", 3239 "ti,sci-rm-range-rchan", 3240 "ti,sci-rm-range-rflow" }; 3241 3242 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); 3243 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 3244 3245 ud->rflow_cnt = cap3 & 0x3fff; 3246 ud->tchan_cnt = cap2 & 0x1ff; 3247 ud->echan_cnt = (cap2 >> 9) & 0x1ff; 3248 ud->rchan_cnt = (cap2 >> 18) & 0x1ff; 3249 ch_count = ud->tchan_cnt + ud->rchan_cnt; 3250 3251 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 3252 sizeof(unsigned long), GFP_KERNEL); 3253 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 3254 GFP_KERNEL); 3255 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 3256 sizeof(unsigned long), GFP_KERNEL); 3257 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 3258 GFP_KERNEL); 3259 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 3260 sizeof(unsigned long), 3261 GFP_KERNEL); 3262 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 3263 BITS_TO_LONGS(ud->rflow_cnt), 3264 sizeof(unsigned long), 3265 GFP_KERNEL); 3266 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 3267 sizeof(unsigned long), 3268 GFP_KERNEL); 3269 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 3270 GFP_KERNEL); 3271 3272 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 3273 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 3274 !ud->rflows || !ud->rflow_in_use) 3275 return -ENOMEM; 3276 3277 /* 3278 * RX flows with the same Ids as RX channels are reserved to be used 3279 * as default flows if remote HW can't generate flow_ids. Those 3280 * RX flows can be requested only explicitly by id. 3281 */ 3282 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 3283 3284 /* by default no GP rflows are assigned to Linux */ 3285 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 3286 3287 /* Get resource ranges from tisci */ 3288 for (i = 0; i < RM_RANGE_LAST; i++) 3289 tisci_rm->rm_ranges[i] = 3290 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 3291 tisci_rm->tisci_dev_id, 3292 (char *)range_names[i]); 3293 3294 /* tchan ranges */ 3295 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3296 if (IS_ERR(rm_res)) { 3297 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 3298 } else { 3299 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 3300 for (i = 0; i < rm_res->sets; i++) { 3301 rm_desc = &rm_res->desc[i]; 3302 bitmap_clear(ud->tchan_map, rm_desc->start, 3303 rm_desc->num); 3304 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n", 3305 rm_desc->start, rm_desc->num); 3306 } 3307 } 3308 irq_res.sets = rm_res->sets; 3309 3310 /* rchan and matching default flow ranges */ 3311 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3312 if (IS_ERR(rm_res)) { 3313 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 3314 } else { 3315 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 3316 for (i = 0; i < rm_res->sets; i++) { 3317 rm_desc = &rm_res->desc[i]; 3318 bitmap_clear(ud->rchan_map, rm_desc->start, 3319 rm_desc->num); 3320 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n", 3321 rm_desc->start, rm_desc->num); 3322 } 3323 } 3324 3325 irq_res.sets += rm_res->sets; 3326 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 3327 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3328 for (i = 0; i < rm_res->sets; i++) { 3329 irq_res.desc[i].start = rm_res->desc[i].start; 3330 irq_res.desc[i].num = rm_res->desc[i].num; 3331 } 3332 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3333 for (j = 0; j < rm_res->sets; j++, i++) { 3334 irq_res.desc[i].start = rm_res->desc[j].start + 3335 ud->match_data->rchan_oes_offset; 3336 irq_res.desc[i].num = rm_res->desc[j].num; 3337 } 3338 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 3339 kfree(irq_res.desc); 3340 if (ret) { 3341 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 3342 return ret; 3343 } 3344 3345 /* GP rflow ranges */ 3346 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 3347 if (IS_ERR(rm_res)) { 3348 /* all gp flows are assigned exclusively to Linux */ 3349 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 3350 ud->rflow_cnt - ud->rchan_cnt); 3351 } else { 3352 for (i = 0; i < rm_res->sets; i++) { 3353 rm_desc = &rm_res->desc[i]; 3354 bitmap_clear(ud->rflow_gp_map, rm_desc->start, 3355 rm_desc->num); 3356 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n", 3357 rm_desc->start, rm_desc->num); 3358 } 3359 } 3360 3361 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 3362 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 3363 if (!ch_count) 3364 return -ENODEV; 3365 3366 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 3367 GFP_KERNEL); 3368 if (!ud->channels) 3369 return -ENOMEM; 3370 3371 dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 3372 ch_count, 3373 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), 3374 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), 3375 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 3376 ud->rflow_cnt)); 3377 3378 return ch_count; 3379 } 3380 3381 static int udma_setup_rx_flush(struct udma_dev *ud) 3382 { 3383 struct udma_rx_flush *rx_flush = &ud->rx_flush; 3384 struct cppi5_desc_hdr_t *tr_desc; 3385 struct cppi5_tr_type1_t *tr_req; 3386 struct cppi5_host_desc_t *desc; 3387 struct device *dev = ud->dev; 3388 struct udma_hwdesc *hwdesc; 3389 size_t tr_size; 3390 3391 /* Allocate 1K buffer for discarded data on RX channel teardown */ 3392 rx_flush->buffer_size = SZ_1K; 3393 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, 3394 GFP_KERNEL); 3395 if (!rx_flush->buffer_vaddr) 3396 return -ENOMEM; 3397 3398 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, 3399 rx_flush->buffer_size, 3400 DMA_TO_DEVICE); 3401 if (dma_mapping_error(dev, rx_flush->buffer_paddr)) 3402 return -ENOMEM; 3403 3404 /* Set up descriptor to be used for TR mode */ 3405 hwdesc = &rx_flush->hwdescs[0]; 3406 tr_size = sizeof(struct cppi5_tr_type1_t); 3407 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); 3408 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 3409 ud->desc_align); 3410 3411 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3412 GFP_KERNEL); 3413 if (!hwdesc->cppi5_desc_vaddr) 3414 return -ENOMEM; 3415 3416 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3417 hwdesc->cppi5_desc_size, 3418 DMA_TO_DEVICE); 3419 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3420 return -ENOMEM; 3421 3422 /* Start of the TR req records */ 3423 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 3424 /* Start address of the TR response array */ 3425 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; 3426 3427 tr_desc = hwdesc->cppi5_desc_vaddr; 3428 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); 3429 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3430 cppi5_desc_set_retpolicy(tr_desc, 0, 0); 3431 3432 tr_req = hwdesc->tr_req_base; 3433 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, 3434 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3435 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); 3436 3437 tr_req->addr = rx_flush->buffer_paddr; 3438 tr_req->icnt0 = rx_flush->buffer_size; 3439 tr_req->icnt1 = 1; 3440 3441 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 3442 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 3443 3444 /* Set up descriptor to be used for packet mode */ 3445 hwdesc = &rx_flush->hwdescs[1]; 3446 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3447 CPPI5_INFO0_HDESC_EPIB_SIZE + 3448 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, 3449 ud->desc_align); 3450 3451 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3452 GFP_KERNEL); 3453 if (!hwdesc->cppi5_desc_vaddr) 3454 return -ENOMEM; 3455 3456 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3457 hwdesc->cppi5_desc_size, 3458 DMA_TO_DEVICE); 3459 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3460 return -ENOMEM; 3461 3462 desc = hwdesc->cppi5_desc_vaddr; 3463 cppi5_hdesc_init(desc, 0, 0); 3464 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3465 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); 3466 3467 cppi5_hdesc_attach_buf(desc, 3468 rx_flush->buffer_paddr, rx_flush->buffer_size, 3469 rx_flush->buffer_paddr, rx_flush->buffer_size); 3470 3471 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 3472 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 3473 return 0; 3474 } 3475 3476 #ifdef CONFIG_DEBUG_FS 3477 static void udma_dbg_summary_show_chan(struct seq_file *s, 3478 struct dma_chan *chan) 3479 { 3480 struct udma_chan *uc = to_udma_chan(chan); 3481 struct udma_chan_config *ucc = &uc->config; 3482 3483 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 3484 chan->dbg_client_name ?: "in-use"); 3485 seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); 3486 3487 switch (uc->config.dir) { 3488 case DMA_MEM_TO_MEM: 3489 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 3490 ucc->src_thread, ucc->dst_thread); 3491 break; 3492 case DMA_DEV_TO_MEM: 3493 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 3494 ucc->src_thread, ucc->dst_thread); 3495 break; 3496 case DMA_MEM_TO_DEV: 3497 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 3498 ucc->src_thread, ucc->dst_thread); 3499 break; 3500 default: 3501 seq_printf(s, ")\n"); 3502 return; 3503 } 3504 3505 if (ucc->ep_type == PSIL_EP_NATIVE) { 3506 seq_printf(s, "PSI-L Native"); 3507 if (ucc->metadata_size) { 3508 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 3509 if (ucc->psd_size) 3510 seq_printf(s, " PSDsize:%u", ucc->psd_size); 3511 seq_printf(s, " ]"); 3512 } 3513 } else { 3514 seq_printf(s, "PDMA"); 3515 if (ucc->enable_acc32 || ucc->enable_burst) 3516 seq_printf(s, "[%s%s ]", 3517 ucc->enable_acc32 ? " ACC32" : "", 3518 ucc->enable_burst ? " BURST" : ""); 3519 } 3520 3521 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 3522 } 3523 3524 static void udma_dbg_summary_show(struct seq_file *s, 3525 struct dma_device *dma_dev) 3526 { 3527 struct dma_chan *chan; 3528 3529 list_for_each_entry(chan, &dma_dev->channels, device_node) { 3530 if (chan->client_count) 3531 udma_dbg_summary_show_chan(s, chan); 3532 } 3533 } 3534 #endif /* CONFIG_DEBUG_FS */ 3535 3536 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 3537 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 3538 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 3539 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 3540 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 3541 3542 static int udma_probe(struct platform_device *pdev) 3543 { 3544 struct device_node *navss_node = pdev->dev.parent->of_node; 3545 struct device *dev = &pdev->dev; 3546 struct udma_dev *ud; 3547 const struct of_device_id *match; 3548 int i, ret; 3549 int ch_count; 3550 3551 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 3552 if (ret) 3553 dev_err(dev, "failed to set dma mask stuff\n"); 3554 3555 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 3556 if (!ud) 3557 return -ENOMEM; 3558 3559 ret = udma_get_mmrs(pdev, ud); 3560 if (ret) 3561 return ret; 3562 3563 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 3564 if (IS_ERR(ud->tisci_rm.tisci)) 3565 return PTR_ERR(ud->tisci_rm.tisci); 3566 3567 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 3568 &ud->tisci_rm.tisci_dev_id); 3569 if (ret) { 3570 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 3571 return ret; 3572 } 3573 pdev->id = ud->tisci_rm.tisci_dev_id; 3574 3575 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 3576 &ud->tisci_rm.tisci_navss_dev_id); 3577 if (ret) { 3578 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 3579 return ret; 3580 } 3581 3582 ret = of_property_read_u32(dev->of_node, "ti,udma-atype", &ud->atype); 3583 if (!ret && ud->atype > 2) { 3584 dev_err(dev, "Invalid atype: %u\n", ud->atype); 3585 return -EINVAL; 3586 } 3587 3588 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 3589 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 3590 3591 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 3592 if (IS_ERR(ud->ringacc)) 3593 return PTR_ERR(ud->ringacc); 3594 3595 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 3596 DOMAIN_BUS_TI_SCI_INTA_MSI); 3597 if (!dev->msi_domain) { 3598 dev_err(dev, "Failed to get MSI domain\n"); 3599 return -EPROBE_DEFER; 3600 } 3601 3602 match = of_match_node(udma_of_match, dev->of_node); 3603 if (!match) { 3604 dev_err(dev, "No compatible match found\n"); 3605 return -ENODEV; 3606 } 3607 ud->match_data = match->data; 3608 3609 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 3610 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 3611 3612 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; 3613 ud->ddev.device_config = udma_slave_config; 3614 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 3615 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 3616 ud->ddev.device_issue_pending = udma_issue_pending; 3617 ud->ddev.device_tx_status = udma_tx_status; 3618 ud->ddev.device_pause = udma_pause; 3619 ud->ddev.device_resume = udma_resume; 3620 ud->ddev.device_terminate_all = udma_terminate_all; 3621 ud->ddev.device_synchronize = udma_synchronize; 3622 #ifdef CONFIG_DEBUG_FS 3623 ud->ddev.dbg_summary_show = udma_dbg_summary_show; 3624 #endif 3625 3626 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 3627 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 3628 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 3629 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 3630 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 3631 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; 3632 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 3633 DESC_METADATA_ENGINE; 3634 if (ud->match_data->enable_memcpy_support) { 3635 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 3636 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 3637 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 3638 } 3639 3640 ud->ddev.dev = dev; 3641 ud->dev = dev; 3642 ud->psil_base = ud->match_data->psil_base; 3643 3644 INIT_LIST_HEAD(&ud->ddev.channels); 3645 INIT_LIST_HEAD(&ud->desc_to_purge); 3646 3647 ch_count = udma_setup_resources(ud); 3648 if (ch_count <= 0) 3649 return ch_count; 3650 3651 spin_lock_init(&ud->lock); 3652 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 3653 3654 ud->desc_align = 64; 3655 if (ud->desc_align < dma_get_cache_alignment()) 3656 ud->desc_align = dma_get_cache_alignment(); 3657 3658 ret = udma_setup_rx_flush(ud); 3659 if (ret) 3660 return ret; 3661 3662 for (i = 0; i < ud->tchan_cnt; i++) { 3663 struct udma_tchan *tchan = &ud->tchans[i]; 3664 3665 tchan->id = i; 3666 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 3667 } 3668 3669 for (i = 0; i < ud->rchan_cnt; i++) { 3670 struct udma_rchan *rchan = &ud->rchans[i]; 3671 3672 rchan->id = i; 3673 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 3674 } 3675 3676 for (i = 0; i < ud->rflow_cnt; i++) { 3677 struct udma_rflow *rflow = &ud->rflows[i]; 3678 3679 rflow->id = i; 3680 } 3681 3682 for (i = 0; i < ch_count; i++) { 3683 struct udma_chan *uc = &ud->channels[i]; 3684 3685 uc->ud = ud; 3686 uc->vc.desc_free = udma_desc_free; 3687 uc->id = i; 3688 uc->tchan = NULL; 3689 uc->rchan = NULL; 3690 uc->config.remote_thread_id = -1; 3691 uc->config.dir = DMA_MEM_TO_MEM; 3692 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 3693 dev_name(dev), i); 3694 3695 vchan_init(&uc->vc, &ud->ddev); 3696 /* Use custom vchan completion handling */ 3697 tasklet_init(&uc->vc.task, udma_vchan_complete, 3698 (unsigned long)&uc->vc); 3699 init_completion(&uc->teardown_completed); 3700 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); 3701 } 3702 3703 ret = dma_async_device_register(&ud->ddev); 3704 if (ret) { 3705 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 3706 return ret; 3707 } 3708 3709 platform_set_drvdata(pdev, ud); 3710 3711 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 3712 if (ret) { 3713 dev_err(dev, "failed to register of_dma controller\n"); 3714 dma_async_device_unregister(&ud->ddev); 3715 } 3716 3717 return ret; 3718 } 3719 3720 static struct platform_driver udma_driver = { 3721 .driver = { 3722 .name = "ti-udma", 3723 .of_match_table = udma_of_match, 3724 .suppress_bind_attrs = true, 3725 }, 3726 .probe = udma_probe, 3727 }; 3728 builtin_platform_driver(udma_driver); 3729 3730 /* Private interfaces to UDMA */ 3731 #include "k3-udma-private.c" 3732