1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/delay.h> 10 #include <linux/dmaengine.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmapool.h> 13 #include <linux/err.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/list.h> 17 #include <linux/platform_device.h> 18 #include <linux/slab.h> 19 #include <linux/spinlock.h> 20 #include <linux/sys_soc.h> 21 #include <linux/of.h> 22 #include <linux/of_dma.h> 23 #include <linux/of_device.h> 24 #include <linux/of_irq.h> 25 #include <linux/workqueue.h> 26 #include <linux/completion.h> 27 #include <linux/soc/ti/k3-ringacc.h> 28 #include <linux/soc/ti/ti_sci_protocol.h> 29 #include <linux/soc/ti/ti_sci_inta_msi.h> 30 #include <linux/dma/k3-event-router.h> 31 #include <linux/dma/ti-cppi5.h> 32 33 #include "../virt-dma.h" 34 #include "k3-udma.h" 35 #include "k3-psil-priv.h" 36 37 struct udma_static_tr { 38 u8 elsize; /* RPSTR0 */ 39 u16 elcnt; /* RPSTR0 */ 40 u16 bstcnt; /* RPSTR1 */ 41 }; 42 43 #define K3_UDMA_MAX_RFLOWS 1024 44 #define K3_UDMA_DEFAULT_RING_SIZE 16 45 46 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 47 #define UDMA_RFLOW_SRCTAG_NONE 0 48 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 49 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 50 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 51 52 #define UDMA_RFLOW_DSTTAG_NONE 0 53 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 54 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 55 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 56 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 57 58 struct udma_chan; 59 60 enum k3_dma_type { 61 DMA_TYPE_UDMA = 0, 62 DMA_TYPE_BCDMA, 63 DMA_TYPE_PKTDMA, 64 }; 65 66 enum udma_mmr { 67 MMR_GCFG = 0, 68 MMR_BCHANRT, 69 MMR_RCHANRT, 70 MMR_TCHANRT, 71 MMR_LAST, 72 }; 73 74 static const char * const mmr_names[] = { 75 [MMR_GCFG] = "gcfg", 76 [MMR_BCHANRT] = "bchanrt", 77 [MMR_RCHANRT] = "rchanrt", 78 [MMR_TCHANRT] = "tchanrt", 79 }; 80 81 struct udma_tchan { 82 void __iomem *reg_rt; 83 84 int id; 85 struct k3_ring *t_ring; /* Transmit ring */ 86 struct k3_ring *tc_ring; /* Transmit Completion ring */ 87 int tflow_id; /* applicable only for PKTDMA */ 88 89 }; 90 91 #define udma_bchan udma_tchan 92 93 struct udma_rflow { 94 int id; 95 struct k3_ring *fd_ring; /* Free Descriptor ring */ 96 struct k3_ring *r_ring; /* Receive ring */ 97 }; 98 99 struct udma_rchan { 100 void __iomem *reg_rt; 101 102 int id; 103 }; 104 105 struct udma_oes_offsets { 106 /* K3 UDMA Output Event Offset */ 107 u32 udma_rchan; 108 109 /* BCDMA Output Event Offsets */ 110 u32 bcdma_bchan_data; 111 u32 bcdma_bchan_ring; 112 u32 bcdma_tchan_data; 113 u32 bcdma_tchan_ring; 114 u32 bcdma_rchan_data; 115 u32 bcdma_rchan_ring; 116 117 /* PKTDMA Output Event Offsets */ 118 u32 pktdma_tchan_flow; 119 u32 pktdma_rchan_flow; 120 }; 121 122 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 123 #define UDMA_FLAG_PDMA_BURST BIT(1) 124 #define UDMA_FLAG_TDTYPE BIT(2) 125 #define UDMA_FLAG_BURST_SIZE BIT(3) 126 #define UDMA_FLAGS_J7_CLASS (UDMA_FLAG_PDMA_ACC32 | \ 127 UDMA_FLAG_PDMA_BURST | \ 128 UDMA_FLAG_TDTYPE | \ 129 UDMA_FLAG_BURST_SIZE) 130 131 struct udma_match_data { 132 enum k3_dma_type type; 133 u32 psil_base; 134 bool enable_memcpy_support; 135 u32 flags; 136 u32 statictr_z_mask; 137 u8 burst_size[3]; 138 struct udma_soc_data *soc_data; 139 }; 140 141 struct udma_soc_data { 142 struct udma_oes_offsets oes; 143 u32 bcdma_trigger_event_offset; 144 }; 145 146 struct udma_hwdesc { 147 size_t cppi5_desc_size; 148 void *cppi5_desc_vaddr; 149 dma_addr_t cppi5_desc_paddr; 150 151 /* TR descriptor internal pointers */ 152 void *tr_req_base; 153 struct cppi5_tr_resp_t *tr_resp_base; 154 }; 155 156 struct udma_rx_flush { 157 struct udma_hwdesc hwdescs[2]; 158 159 size_t buffer_size; 160 void *buffer_vaddr; 161 dma_addr_t buffer_paddr; 162 }; 163 164 struct udma_tpl { 165 u8 levels; 166 u32 start_idx[3]; 167 }; 168 169 struct udma_dev { 170 struct dma_device ddev; 171 struct device *dev; 172 void __iomem *mmrs[MMR_LAST]; 173 const struct udma_match_data *match_data; 174 const struct udma_soc_data *soc_data; 175 176 struct udma_tpl bchan_tpl; 177 struct udma_tpl tchan_tpl; 178 struct udma_tpl rchan_tpl; 179 180 size_t desc_align; /* alignment to use for descriptors */ 181 182 struct udma_tisci_rm tisci_rm; 183 184 struct k3_ringacc *ringacc; 185 186 struct work_struct purge_work; 187 struct list_head desc_to_purge; 188 spinlock_t lock; 189 190 struct udma_rx_flush rx_flush; 191 192 int bchan_cnt; 193 int tchan_cnt; 194 int echan_cnt; 195 int rchan_cnt; 196 int rflow_cnt; 197 int tflow_cnt; 198 unsigned long *bchan_map; 199 unsigned long *tchan_map; 200 unsigned long *rchan_map; 201 unsigned long *rflow_gp_map; 202 unsigned long *rflow_gp_map_allocated; 203 unsigned long *rflow_in_use; 204 unsigned long *tflow_map; 205 206 struct udma_bchan *bchans; 207 struct udma_tchan *tchans; 208 struct udma_rchan *rchans; 209 struct udma_rflow *rflows; 210 211 struct udma_chan *channels; 212 u32 psil_base; 213 u32 atype; 214 u32 asel; 215 }; 216 217 struct udma_desc { 218 struct virt_dma_desc vd; 219 220 bool terminated; 221 222 enum dma_transfer_direction dir; 223 224 struct udma_static_tr static_tr; 225 u32 residue; 226 227 unsigned int sglen; 228 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 229 unsigned int tr_idx; 230 231 u32 metadata_size; 232 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 233 234 unsigned int hwdesc_count; 235 struct udma_hwdesc hwdesc[]; 236 }; 237 238 enum udma_chan_state { 239 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 240 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 241 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 242 }; 243 244 struct udma_tx_drain { 245 struct delayed_work work; 246 ktime_t tstamp; 247 u32 residue; 248 }; 249 250 struct udma_chan_config { 251 bool pkt_mode; /* TR or packet */ 252 bool needs_epib; /* EPIB is needed for the communication or not */ 253 u32 psd_size; /* size of Protocol Specific Data */ 254 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 255 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 256 bool notdpkt; /* Suppress sending TDC packet */ 257 int remote_thread_id; 258 u32 atype; 259 u32 asel; 260 u32 src_thread; 261 u32 dst_thread; 262 enum psil_endpoint_type ep_type; 263 bool enable_acc32; 264 bool enable_burst; 265 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 266 267 u32 tr_trigger_type; 268 unsigned long tx_flags; 269 270 /* PKDMA mapped channel */ 271 int mapped_channel_id; 272 /* PKTDMA default tflow or rflow for mapped channel */ 273 int default_flow_id; 274 275 enum dma_transfer_direction dir; 276 }; 277 278 struct udma_chan { 279 struct virt_dma_chan vc; 280 struct dma_slave_config cfg; 281 struct udma_dev *ud; 282 struct device *dma_dev; 283 struct udma_desc *desc; 284 struct udma_desc *terminated_desc; 285 struct udma_static_tr static_tr; 286 char *name; 287 288 struct udma_bchan *bchan; 289 struct udma_tchan *tchan; 290 struct udma_rchan *rchan; 291 struct udma_rflow *rflow; 292 293 bool psil_paired; 294 295 int irq_num_ring; 296 int irq_num_udma; 297 298 bool cyclic; 299 bool paused; 300 301 enum udma_chan_state state; 302 struct completion teardown_completed; 303 304 struct udma_tx_drain tx_drain; 305 306 /* Channel configuration parameters */ 307 struct udma_chan_config config; 308 /* Channel configuration parameters (backup) */ 309 struct udma_chan_config backup_config; 310 311 /* dmapool for packet mode descriptors */ 312 bool use_dma_pool; 313 struct dma_pool *hdesc_pool; 314 315 u32 id; 316 }; 317 318 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 319 { 320 return container_of(d, struct udma_dev, ddev); 321 } 322 323 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 324 { 325 return container_of(c, struct udma_chan, vc.chan); 326 } 327 328 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 329 { 330 return container_of(t, struct udma_desc, vd.tx); 331 } 332 333 /* Generic register access functions */ 334 static inline u32 udma_read(void __iomem *base, int reg) 335 { 336 return readl(base + reg); 337 } 338 339 static inline void udma_write(void __iomem *base, int reg, u32 val) 340 { 341 writel(val, base + reg); 342 } 343 344 static inline void udma_update_bits(void __iomem *base, int reg, 345 u32 mask, u32 val) 346 { 347 u32 tmp, orig; 348 349 orig = readl(base + reg); 350 tmp = orig & ~mask; 351 tmp |= (val & mask); 352 353 if (tmp != orig) 354 writel(tmp, base + reg); 355 } 356 357 /* TCHANRT */ 358 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg) 359 { 360 if (!uc->tchan) 361 return 0; 362 return udma_read(uc->tchan->reg_rt, reg); 363 } 364 365 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val) 366 { 367 if (!uc->tchan) 368 return; 369 udma_write(uc->tchan->reg_rt, reg, val); 370 } 371 372 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg, 373 u32 mask, u32 val) 374 { 375 if (!uc->tchan) 376 return; 377 udma_update_bits(uc->tchan->reg_rt, reg, mask, val); 378 } 379 380 /* RCHANRT */ 381 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg) 382 { 383 if (!uc->rchan) 384 return 0; 385 return udma_read(uc->rchan->reg_rt, reg); 386 } 387 388 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val) 389 { 390 if (!uc->rchan) 391 return; 392 udma_write(uc->rchan->reg_rt, reg, val); 393 } 394 395 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg, 396 u32 mask, u32 val) 397 { 398 if (!uc->rchan) 399 return; 400 udma_update_bits(uc->rchan->reg_rt, reg, mask, val); 401 } 402 403 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 404 { 405 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 406 407 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 408 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 409 tisci_rm->tisci_navss_dev_id, 410 src_thread, dst_thread); 411 } 412 413 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 414 u32 dst_thread) 415 { 416 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 417 418 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 419 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 420 tisci_rm->tisci_navss_dev_id, 421 src_thread, dst_thread); 422 } 423 424 static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) 425 { 426 struct device *chan_dev = &chan->dev->device; 427 428 if (asel == 0) { 429 /* No special handling for the channel */ 430 chan->dev->chan_dma_dev = false; 431 432 chan_dev->dma_coherent = false; 433 chan_dev->dma_parms = NULL; 434 } else if (asel == 14 || asel == 15) { 435 chan->dev->chan_dma_dev = true; 436 437 chan_dev->dma_coherent = true; 438 dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48)); 439 chan_dev->dma_parms = chan_dev->parent->dma_parms; 440 } else { 441 dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel); 442 443 chan_dev->dma_coherent = false; 444 chan_dev->dma_parms = NULL; 445 } 446 } 447 448 static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id) 449 { 450 int i; 451 452 for (i = 0; i < tpl_map->levels; i++) { 453 if (chan_id >= tpl_map->start_idx[i]) 454 return i; 455 } 456 457 return 0; 458 } 459 460 static void udma_reset_uchan(struct udma_chan *uc) 461 { 462 memset(&uc->config, 0, sizeof(uc->config)); 463 uc->config.remote_thread_id = -1; 464 uc->config.mapped_channel_id = -1; 465 uc->config.default_flow_id = -1; 466 uc->state = UDMA_CHAN_IS_IDLE; 467 } 468 469 static void udma_dump_chan_stdata(struct udma_chan *uc) 470 { 471 struct device *dev = uc->ud->dev; 472 u32 offset; 473 int i; 474 475 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 476 dev_dbg(dev, "TCHAN State data:\n"); 477 for (i = 0; i < 32; i++) { 478 offset = UDMA_CHAN_RT_STDATA_REG + i * 4; 479 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 480 udma_tchanrt_read(uc, offset)); 481 } 482 } 483 484 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 485 dev_dbg(dev, "RCHAN State data:\n"); 486 for (i = 0; i < 32; i++) { 487 offset = UDMA_CHAN_RT_STDATA_REG + i * 4; 488 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 489 udma_rchanrt_read(uc, offset)); 490 } 491 } 492 } 493 494 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 495 int idx) 496 { 497 return d->hwdesc[idx].cppi5_desc_paddr; 498 } 499 500 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 501 { 502 return d->hwdesc[idx].cppi5_desc_vaddr; 503 } 504 505 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 506 dma_addr_t paddr) 507 { 508 struct udma_desc *d = uc->terminated_desc; 509 510 if (d) { 511 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 512 d->desc_idx); 513 514 if (desc_paddr != paddr) 515 d = NULL; 516 } 517 518 if (!d) { 519 d = uc->desc; 520 if (d) { 521 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 522 d->desc_idx); 523 524 if (desc_paddr != paddr) 525 d = NULL; 526 } 527 } 528 529 return d; 530 } 531 532 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 533 { 534 if (uc->use_dma_pool) { 535 int i; 536 537 for (i = 0; i < d->hwdesc_count; i++) { 538 if (!d->hwdesc[i].cppi5_desc_vaddr) 539 continue; 540 541 dma_pool_free(uc->hdesc_pool, 542 d->hwdesc[i].cppi5_desc_vaddr, 543 d->hwdesc[i].cppi5_desc_paddr); 544 545 d->hwdesc[i].cppi5_desc_vaddr = NULL; 546 } 547 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 548 dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size, 549 d->hwdesc[0].cppi5_desc_vaddr, 550 d->hwdesc[0].cppi5_desc_paddr); 551 552 d->hwdesc[0].cppi5_desc_vaddr = NULL; 553 } 554 } 555 556 static void udma_purge_desc_work(struct work_struct *work) 557 { 558 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 559 struct virt_dma_desc *vd, *_vd; 560 unsigned long flags; 561 LIST_HEAD(head); 562 563 spin_lock_irqsave(&ud->lock, flags); 564 list_splice_tail_init(&ud->desc_to_purge, &head); 565 spin_unlock_irqrestore(&ud->lock, flags); 566 567 list_for_each_entry_safe(vd, _vd, &head, node) { 568 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 569 struct udma_desc *d = to_udma_desc(&vd->tx); 570 571 udma_free_hwdesc(uc, d); 572 list_del(&vd->node); 573 kfree(d); 574 } 575 576 /* If more to purge, schedule the work again */ 577 if (!list_empty(&ud->desc_to_purge)) 578 schedule_work(&ud->purge_work); 579 } 580 581 static void udma_desc_free(struct virt_dma_desc *vd) 582 { 583 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 584 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 585 struct udma_desc *d = to_udma_desc(&vd->tx); 586 unsigned long flags; 587 588 if (uc->terminated_desc == d) 589 uc->terminated_desc = NULL; 590 591 if (uc->use_dma_pool) { 592 udma_free_hwdesc(uc, d); 593 kfree(d); 594 return; 595 } 596 597 spin_lock_irqsave(&ud->lock, flags); 598 list_add_tail(&vd->node, &ud->desc_to_purge); 599 spin_unlock_irqrestore(&ud->lock, flags); 600 601 schedule_work(&ud->purge_work); 602 } 603 604 static bool udma_is_chan_running(struct udma_chan *uc) 605 { 606 u32 trt_ctl = 0; 607 u32 rrt_ctl = 0; 608 609 if (uc->tchan) 610 trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 611 if (uc->rchan) 612 rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 613 614 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 615 return true; 616 617 return false; 618 } 619 620 static bool udma_is_chan_paused(struct udma_chan *uc) 621 { 622 u32 val, pause_mask; 623 624 switch (uc->config.dir) { 625 case DMA_DEV_TO_MEM: 626 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); 627 pause_mask = UDMA_PEER_RT_EN_PAUSE; 628 break; 629 case DMA_MEM_TO_DEV: 630 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); 631 pause_mask = UDMA_PEER_RT_EN_PAUSE; 632 break; 633 case DMA_MEM_TO_MEM: 634 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 635 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 636 break; 637 default: 638 return false; 639 } 640 641 if (val & pause_mask) 642 return true; 643 644 return false; 645 } 646 647 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) 648 { 649 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; 650 } 651 652 static int udma_push_to_ring(struct udma_chan *uc, int idx) 653 { 654 struct udma_desc *d = uc->desc; 655 struct k3_ring *ring = NULL; 656 dma_addr_t paddr; 657 658 switch (uc->config.dir) { 659 case DMA_DEV_TO_MEM: 660 ring = uc->rflow->fd_ring; 661 break; 662 case DMA_MEM_TO_DEV: 663 case DMA_MEM_TO_MEM: 664 ring = uc->tchan->t_ring; 665 break; 666 default: 667 return -EINVAL; 668 } 669 670 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ 671 if (idx == -1) { 672 paddr = udma_get_rx_flush_hwdesc_paddr(uc); 673 } else { 674 paddr = udma_curr_cppi5_desc_paddr(d, idx); 675 676 wmb(); /* Ensure that writes are not moved over this point */ 677 } 678 679 return k3_ringacc_ring_push(ring, &paddr); 680 } 681 682 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) 683 { 684 if (uc->config.dir != DMA_DEV_TO_MEM) 685 return false; 686 687 if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) 688 return true; 689 690 return false; 691 } 692 693 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 694 { 695 struct k3_ring *ring = NULL; 696 int ret; 697 698 switch (uc->config.dir) { 699 case DMA_DEV_TO_MEM: 700 ring = uc->rflow->r_ring; 701 break; 702 case DMA_MEM_TO_DEV: 703 case DMA_MEM_TO_MEM: 704 ring = uc->tchan->tc_ring; 705 break; 706 default: 707 return -ENOENT; 708 } 709 710 ret = k3_ringacc_ring_pop(ring, addr); 711 if (ret) 712 return ret; 713 714 rmb(); /* Ensure that reads are not moved before this point */ 715 716 /* Teardown completion */ 717 if (cppi5_desc_is_tdcm(*addr)) 718 return 0; 719 720 /* Check for flush descriptor */ 721 if (udma_desc_is_rx_flush(uc, *addr)) 722 return -ENOENT; 723 724 return 0; 725 } 726 727 static void udma_reset_rings(struct udma_chan *uc) 728 { 729 struct k3_ring *ring1 = NULL; 730 struct k3_ring *ring2 = NULL; 731 732 switch (uc->config.dir) { 733 case DMA_DEV_TO_MEM: 734 if (uc->rchan) { 735 ring1 = uc->rflow->fd_ring; 736 ring2 = uc->rflow->r_ring; 737 } 738 break; 739 case DMA_MEM_TO_DEV: 740 case DMA_MEM_TO_MEM: 741 if (uc->tchan) { 742 ring1 = uc->tchan->t_ring; 743 ring2 = uc->tchan->tc_ring; 744 } 745 break; 746 default: 747 break; 748 } 749 750 if (ring1) 751 k3_ringacc_ring_reset_dma(ring1, 752 k3_ringacc_ring_get_occ(ring1)); 753 if (ring2) 754 k3_ringacc_ring_reset(ring2); 755 756 /* make sure we are not leaking memory by stalled descriptor */ 757 if (uc->terminated_desc) { 758 udma_desc_free(&uc->terminated_desc->vd); 759 uc->terminated_desc = NULL; 760 } 761 } 762 763 static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val) 764 { 765 if (uc->desc->dir == DMA_DEV_TO_MEM) { 766 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 767 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 768 if (uc->config.ep_type != PSIL_EP_NATIVE) 769 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 770 } else { 771 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 772 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 773 if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE) 774 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 775 } 776 } 777 778 static void udma_reset_counters(struct udma_chan *uc) 779 { 780 u32 val; 781 782 if (uc->tchan) { 783 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 784 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 785 786 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 787 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 788 789 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); 790 udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); 791 792 if (!uc->bchan) { 793 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 794 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 795 } 796 } 797 798 if (uc->rchan) { 799 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 800 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 801 802 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 803 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 804 805 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); 806 udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); 807 808 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 809 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 810 } 811 } 812 813 static int udma_reset_chan(struct udma_chan *uc, bool hard) 814 { 815 switch (uc->config.dir) { 816 case DMA_DEV_TO_MEM: 817 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 818 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 819 break; 820 case DMA_MEM_TO_DEV: 821 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 822 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 823 break; 824 case DMA_MEM_TO_MEM: 825 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 826 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 827 break; 828 default: 829 return -EINVAL; 830 } 831 832 /* Reset all counters */ 833 udma_reset_counters(uc); 834 835 /* Hard reset: re-initialize the channel to reset */ 836 if (hard) { 837 struct udma_chan_config ucc_backup; 838 int ret; 839 840 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 841 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 842 843 /* restore the channel configuration */ 844 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 845 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 846 if (ret) 847 return ret; 848 849 /* 850 * Setting forced teardown after forced reset helps recovering 851 * the rchan. 852 */ 853 if (uc->config.dir == DMA_DEV_TO_MEM) 854 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 855 UDMA_CHAN_RT_CTL_EN | 856 UDMA_CHAN_RT_CTL_TDOWN | 857 UDMA_CHAN_RT_CTL_FTDOWN); 858 } 859 uc->state = UDMA_CHAN_IS_IDLE; 860 861 return 0; 862 } 863 864 static void udma_start_desc(struct udma_chan *uc) 865 { 866 struct udma_chan_config *ucc = &uc->config; 867 868 if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode && 869 (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 870 int i; 871 872 /* 873 * UDMA only: Push all descriptors to ring for packet mode 874 * cyclic or RX 875 * PKTDMA supports pre-linked descriptor and cyclic is not 876 * supported 877 */ 878 for (i = 0; i < uc->desc->sglen; i++) 879 udma_push_to_ring(uc, i); 880 } else { 881 udma_push_to_ring(uc, 0); 882 } 883 } 884 885 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 886 { 887 /* Only PDMAs have staticTR */ 888 if (uc->config.ep_type == PSIL_EP_NATIVE) 889 return false; 890 891 /* Check if the staticTR configuration has changed for TX */ 892 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 893 return true; 894 895 return false; 896 } 897 898 static int udma_start(struct udma_chan *uc) 899 { 900 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 901 902 if (!vd) { 903 uc->desc = NULL; 904 return -ENOENT; 905 } 906 907 list_del(&vd->node); 908 909 uc->desc = to_udma_desc(&vd->tx); 910 911 /* Channel is already running and does not need reconfiguration */ 912 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 913 udma_start_desc(uc); 914 goto out; 915 } 916 917 /* Make sure that we clear the teardown bit, if it is set */ 918 udma_reset_chan(uc, false); 919 920 /* Push descriptors before we start the channel */ 921 udma_start_desc(uc); 922 923 switch (uc->desc->dir) { 924 case DMA_DEV_TO_MEM: 925 /* Config remote TR */ 926 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 927 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 928 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 929 const struct udma_match_data *match_data = 930 uc->ud->match_data; 931 932 if (uc->config.enable_acc32) 933 val |= PDMA_STATIC_TR_XY_ACC32; 934 if (uc->config.enable_burst) 935 val |= PDMA_STATIC_TR_XY_BURST; 936 937 udma_rchanrt_write(uc, 938 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, 939 val); 940 941 udma_rchanrt_write(uc, 942 UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG, 943 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 944 match_data->statictr_z_mask)); 945 946 /* save the current staticTR configuration */ 947 memcpy(&uc->static_tr, &uc->desc->static_tr, 948 sizeof(uc->static_tr)); 949 } 950 951 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 952 UDMA_CHAN_RT_CTL_EN); 953 954 /* Enable remote */ 955 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 956 UDMA_PEER_RT_EN_ENABLE); 957 958 break; 959 case DMA_MEM_TO_DEV: 960 /* Config remote TR */ 961 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 962 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 963 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 964 965 if (uc->config.enable_acc32) 966 val |= PDMA_STATIC_TR_XY_ACC32; 967 if (uc->config.enable_burst) 968 val |= PDMA_STATIC_TR_XY_BURST; 969 970 udma_tchanrt_write(uc, 971 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, 972 val); 973 974 /* save the current staticTR configuration */ 975 memcpy(&uc->static_tr, &uc->desc->static_tr, 976 sizeof(uc->static_tr)); 977 } 978 979 /* Enable remote */ 980 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 981 UDMA_PEER_RT_EN_ENABLE); 982 983 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 984 UDMA_CHAN_RT_CTL_EN); 985 986 break; 987 case DMA_MEM_TO_MEM: 988 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 989 UDMA_CHAN_RT_CTL_EN); 990 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 991 UDMA_CHAN_RT_CTL_EN); 992 993 break; 994 default: 995 return -EINVAL; 996 } 997 998 uc->state = UDMA_CHAN_IS_ACTIVE; 999 out: 1000 1001 return 0; 1002 } 1003 1004 static int udma_stop(struct udma_chan *uc) 1005 { 1006 enum udma_chan_state old_state = uc->state; 1007 1008 uc->state = UDMA_CHAN_IS_TERMINATING; 1009 reinit_completion(&uc->teardown_completed); 1010 1011 switch (uc->config.dir) { 1012 case DMA_DEV_TO_MEM: 1013 if (!uc->cyclic && !uc->desc) 1014 udma_push_to_ring(uc, -1); 1015 1016 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 1017 UDMA_PEER_RT_EN_ENABLE | 1018 UDMA_PEER_RT_EN_TEARDOWN); 1019 break; 1020 case DMA_MEM_TO_DEV: 1021 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 1022 UDMA_PEER_RT_EN_ENABLE | 1023 UDMA_PEER_RT_EN_FLUSH); 1024 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 1025 UDMA_CHAN_RT_CTL_EN | 1026 UDMA_CHAN_RT_CTL_TDOWN); 1027 break; 1028 case DMA_MEM_TO_MEM: 1029 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 1030 UDMA_CHAN_RT_CTL_EN | 1031 UDMA_CHAN_RT_CTL_TDOWN); 1032 break; 1033 default: 1034 uc->state = old_state; 1035 complete_all(&uc->teardown_completed); 1036 return -EINVAL; 1037 } 1038 1039 return 0; 1040 } 1041 1042 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 1043 { 1044 struct udma_desc *d = uc->desc; 1045 struct cppi5_host_desc_t *h_desc; 1046 1047 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 1048 cppi5_hdesc_reset_to_original(h_desc); 1049 udma_push_to_ring(uc, d->desc_idx); 1050 d->desc_idx = (d->desc_idx + 1) % d->sglen; 1051 } 1052 1053 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 1054 { 1055 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 1056 1057 memcpy(d->metadata, h_desc->epib, d->metadata_size); 1058 } 1059 1060 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 1061 { 1062 u32 peer_bcnt, bcnt; 1063 1064 /* 1065 * Only TX towards PDMA is affected. 1066 * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer 1067 * completion calculation, consumer must ensure that there is no stale 1068 * data in DMA fabric in this case. 1069 */ 1070 if (uc->config.ep_type == PSIL_EP_NATIVE || 1071 uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT)) 1072 return true; 1073 1074 peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 1075 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 1076 1077 /* Transfer is incomplete, store current residue and time stamp */ 1078 if (peer_bcnt < bcnt) { 1079 uc->tx_drain.residue = bcnt - peer_bcnt; 1080 uc->tx_drain.tstamp = ktime_get(); 1081 return false; 1082 } 1083 1084 return true; 1085 } 1086 1087 static void udma_check_tx_completion(struct work_struct *work) 1088 { 1089 struct udma_chan *uc = container_of(work, typeof(*uc), 1090 tx_drain.work.work); 1091 bool desc_done = true; 1092 u32 residue_diff; 1093 ktime_t time_diff; 1094 unsigned long delay; 1095 1096 while (1) { 1097 if (uc->desc) { 1098 /* Get previous residue and time stamp */ 1099 residue_diff = uc->tx_drain.residue; 1100 time_diff = uc->tx_drain.tstamp; 1101 /* 1102 * Get current residue and time stamp or see if 1103 * transfer is complete 1104 */ 1105 desc_done = udma_is_desc_really_done(uc, uc->desc); 1106 } 1107 1108 if (!desc_done) { 1109 /* 1110 * Find the time delta and residue delta w.r.t 1111 * previous poll 1112 */ 1113 time_diff = ktime_sub(uc->tx_drain.tstamp, 1114 time_diff) + 1; 1115 residue_diff -= uc->tx_drain.residue; 1116 if (residue_diff) { 1117 /* 1118 * Try to guess when we should check 1119 * next time by calculating rate at 1120 * which data is being drained at the 1121 * peer device 1122 */ 1123 delay = (time_diff / residue_diff) * 1124 uc->tx_drain.residue; 1125 } else { 1126 /* No progress, check again in 1 second */ 1127 schedule_delayed_work(&uc->tx_drain.work, HZ); 1128 break; 1129 } 1130 1131 usleep_range(ktime_to_us(delay), 1132 ktime_to_us(delay) + 10); 1133 continue; 1134 } 1135 1136 if (uc->desc) { 1137 struct udma_desc *d = uc->desc; 1138 1139 udma_decrement_byte_counters(uc, d->residue); 1140 udma_start(uc); 1141 vchan_cookie_complete(&d->vd); 1142 break; 1143 } 1144 1145 break; 1146 } 1147 } 1148 1149 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 1150 { 1151 struct udma_chan *uc = data; 1152 struct udma_desc *d; 1153 dma_addr_t paddr = 0; 1154 1155 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1156 return IRQ_HANDLED; 1157 1158 spin_lock(&uc->vc.lock); 1159 1160 /* Teardown completion message */ 1161 if (cppi5_desc_is_tdcm(paddr)) { 1162 complete_all(&uc->teardown_completed); 1163 1164 if (uc->terminated_desc) { 1165 udma_desc_free(&uc->terminated_desc->vd); 1166 uc->terminated_desc = NULL; 1167 } 1168 1169 if (!uc->desc) 1170 udma_start(uc); 1171 1172 goto out; 1173 } 1174 1175 d = udma_udma_desc_from_paddr(uc, paddr); 1176 1177 if (d) { 1178 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1179 d->desc_idx); 1180 if (desc_paddr != paddr) { 1181 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1182 goto out; 1183 } 1184 1185 if (d == uc->desc) { 1186 /* active descriptor */ 1187 if (uc->cyclic) { 1188 udma_cyclic_packet_elapsed(uc); 1189 vchan_cyclic_callback(&d->vd); 1190 } else { 1191 if (udma_is_desc_really_done(uc, d)) { 1192 udma_decrement_byte_counters(uc, d->residue); 1193 udma_start(uc); 1194 vchan_cookie_complete(&d->vd); 1195 } else { 1196 schedule_delayed_work(&uc->tx_drain.work, 1197 0); 1198 } 1199 } 1200 } else { 1201 /* 1202 * terminated descriptor, mark the descriptor as 1203 * completed to update the channel's cookie marker 1204 */ 1205 dma_cookie_complete(&d->vd.tx); 1206 } 1207 } 1208 out: 1209 spin_unlock(&uc->vc.lock); 1210 1211 return IRQ_HANDLED; 1212 } 1213 1214 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1215 { 1216 struct udma_chan *uc = data; 1217 struct udma_desc *d; 1218 1219 spin_lock(&uc->vc.lock); 1220 d = uc->desc; 1221 if (d) { 1222 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1223 1224 if (uc->cyclic) { 1225 vchan_cyclic_callback(&d->vd); 1226 } else { 1227 /* TODO: figure out the real amount of data */ 1228 udma_decrement_byte_counters(uc, d->residue); 1229 udma_start(uc); 1230 vchan_cookie_complete(&d->vd); 1231 } 1232 } 1233 1234 spin_unlock(&uc->vc.lock); 1235 1236 return IRQ_HANDLED; 1237 } 1238 1239 /** 1240 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1241 * @ud: UDMA device 1242 * @from: Start the search from this flow id number 1243 * @cnt: Number of consecutive flow ids to allocate 1244 * 1245 * Allocate range of RX flow ids for future use, those flows can be requested 1246 * only using explicit flow id number. if @from is set to -1 it will try to find 1247 * first free range. if @from is positive value it will force allocation only 1248 * of the specified range of flows. 1249 * 1250 * Returns -ENOMEM if can't find free range. 1251 * -EEXIST if requested range is busy. 1252 * -EINVAL if wrong input values passed. 1253 * Returns flow id on success. 1254 */ 1255 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1256 { 1257 int start, tmp_from; 1258 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1259 1260 tmp_from = from; 1261 if (tmp_from < 0) 1262 tmp_from = ud->rchan_cnt; 1263 /* default flows can't be allocated and accessible only by id */ 1264 if (tmp_from < ud->rchan_cnt) 1265 return -EINVAL; 1266 1267 if (tmp_from + cnt > ud->rflow_cnt) 1268 return -EINVAL; 1269 1270 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1271 ud->rflow_cnt); 1272 1273 start = bitmap_find_next_zero_area(tmp, 1274 ud->rflow_cnt, 1275 tmp_from, cnt, 0); 1276 if (start >= ud->rflow_cnt) 1277 return -ENOMEM; 1278 1279 if (from >= 0 && start != from) 1280 return -EEXIST; 1281 1282 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1283 return start; 1284 } 1285 1286 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1287 { 1288 if (from < ud->rchan_cnt) 1289 return -EINVAL; 1290 if (from + cnt > ud->rflow_cnt) 1291 return -EINVAL; 1292 1293 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1294 return 0; 1295 } 1296 1297 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1298 { 1299 /* 1300 * Attempt to request rflow by ID can be made for any rflow 1301 * if not in use with assumption that caller knows what's doing. 1302 * TI-SCI FW will perform additional permission check ant way, it's 1303 * safe 1304 */ 1305 1306 if (id < 0 || id >= ud->rflow_cnt) 1307 return ERR_PTR(-ENOENT); 1308 1309 if (test_bit(id, ud->rflow_in_use)) 1310 return ERR_PTR(-ENOENT); 1311 1312 if (ud->rflow_gp_map) { 1313 /* GP rflow has to be allocated first */ 1314 if (!test_bit(id, ud->rflow_gp_map) && 1315 !test_bit(id, ud->rflow_gp_map_allocated)) 1316 return ERR_PTR(-EINVAL); 1317 } 1318 1319 dev_dbg(ud->dev, "get rflow%d\n", id); 1320 set_bit(id, ud->rflow_in_use); 1321 return &ud->rflows[id]; 1322 } 1323 1324 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1325 { 1326 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1327 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1328 return; 1329 } 1330 1331 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1332 clear_bit(rflow->id, ud->rflow_in_use); 1333 } 1334 1335 #define UDMA_RESERVE_RESOURCE(res) \ 1336 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1337 enum udma_tp_level tpl, \ 1338 int id) \ 1339 { \ 1340 if (id >= 0) { \ 1341 if (test_bit(id, ud->res##_map)) { \ 1342 dev_err(ud->dev, "res##%d is in use\n", id); \ 1343 return ERR_PTR(-ENOENT); \ 1344 } \ 1345 } else { \ 1346 int start; \ 1347 \ 1348 if (tpl >= ud->res##_tpl.levels) \ 1349 tpl = ud->res##_tpl.levels - 1; \ 1350 \ 1351 start = ud->res##_tpl.start_idx[tpl]; \ 1352 \ 1353 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1354 start); \ 1355 if (id == ud->res##_cnt) { \ 1356 return ERR_PTR(-ENOENT); \ 1357 } \ 1358 } \ 1359 \ 1360 set_bit(id, ud->res##_map); \ 1361 return &ud->res##s[id]; \ 1362 } 1363 1364 UDMA_RESERVE_RESOURCE(bchan); 1365 UDMA_RESERVE_RESOURCE(tchan); 1366 UDMA_RESERVE_RESOURCE(rchan); 1367 1368 static int bcdma_get_bchan(struct udma_chan *uc) 1369 { 1370 struct udma_dev *ud = uc->ud; 1371 enum udma_tp_level tpl; 1372 int ret; 1373 1374 if (uc->bchan) { 1375 dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n", 1376 uc->id, uc->bchan->id); 1377 return 0; 1378 } 1379 1380 /* 1381 * Use normal channels for peripherals, and highest TPL channel for 1382 * mem2mem 1383 */ 1384 if (uc->config.tr_trigger_type) 1385 tpl = 0; 1386 else 1387 tpl = ud->bchan_tpl.levels - 1; 1388 1389 uc->bchan = __udma_reserve_bchan(ud, tpl, -1); 1390 if (IS_ERR(uc->bchan)) { 1391 ret = PTR_ERR(uc->bchan); 1392 uc->bchan = NULL; 1393 return ret; 1394 } 1395 1396 uc->tchan = uc->bchan; 1397 1398 return 0; 1399 } 1400 1401 static int udma_get_tchan(struct udma_chan *uc) 1402 { 1403 struct udma_dev *ud = uc->ud; 1404 int ret; 1405 1406 if (uc->tchan) { 1407 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1408 uc->id, uc->tchan->id); 1409 return 0; 1410 } 1411 1412 /* 1413 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels. 1414 * For PKTDMA mapped channels it is configured to a channel which must 1415 * be used to service the peripheral. 1416 */ 1417 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, 1418 uc->config.mapped_channel_id); 1419 if (IS_ERR(uc->tchan)) { 1420 ret = PTR_ERR(uc->tchan); 1421 uc->tchan = NULL; 1422 return ret; 1423 } 1424 1425 if (ud->tflow_cnt) { 1426 int tflow_id; 1427 1428 /* Only PKTDMA have support for tx flows */ 1429 if (uc->config.default_flow_id >= 0) 1430 tflow_id = uc->config.default_flow_id; 1431 else 1432 tflow_id = uc->tchan->id; 1433 1434 if (test_bit(tflow_id, ud->tflow_map)) { 1435 dev_err(ud->dev, "tflow%d is in use\n", tflow_id); 1436 clear_bit(uc->tchan->id, ud->tchan_map); 1437 uc->tchan = NULL; 1438 return -ENOENT; 1439 } 1440 1441 uc->tchan->tflow_id = tflow_id; 1442 set_bit(tflow_id, ud->tflow_map); 1443 } else { 1444 uc->tchan->tflow_id = -1; 1445 } 1446 1447 return 0; 1448 } 1449 1450 static int udma_get_rchan(struct udma_chan *uc) 1451 { 1452 struct udma_dev *ud = uc->ud; 1453 int ret; 1454 1455 if (uc->rchan) { 1456 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1457 uc->id, uc->rchan->id); 1458 return 0; 1459 } 1460 1461 /* 1462 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels. 1463 * For PKTDMA mapped channels it is configured to a channel which must 1464 * be used to service the peripheral. 1465 */ 1466 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, 1467 uc->config.mapped_channel_id); 1468 if (IS_ERR(uc->rchan)) { 1469 ret = PTR_ERR(uc->rchan); 1470 uc->rchan = NULL; 1471 return ret; 1472 } 1473 1474 return 0; 1475 } 1476 1477 static int udma_get_chan_pair(struct udma_chan *uc) 1478 { 1479 struct udma_dev *ud = uc->ud; 1480 int chan_id, end; 1481 1482 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1483 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1484 uc->id, uc->tchan->id); 1485 return 0; 1486 } 1487 1488 if (uc->tchan) { 1489 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1490 uc->id, uc->tchan->id); 1491 return -EBUSY; 1492 } else if (uc->rchan) { 1493 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1494 uc->id, uc->rchan->id); 1495 return -EBUSY; 1496 } 1497 1498 /* Can be optimized, but let's have it like this for now */ 1499 end = min(ud->tchan_cnt, ud->rchan_cnt); 1500 /* 1501 * Try to use the highest TPL channel pair for MEM_TO_MEM channels 1502 * Note: in UDMAP the channel TPL is symmetric between tchan and rchan 1503 */ 1504 chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1]; 1505 for (; chan_id < end; chan_id++) { 1506 if (!test_bit(chan_id, ud->tchan_map) && 1507 !test_bit(chan_id, ud->rchan_map)) 1508 break; 1509 } 1510 1511 if (chan_id == end) 1512 return -ENOENT; 1513 1514 set_bit(chan_id, ud->tchan_map); 1515 set_bit(chan_id, ud->rchan_map); 1516 uc->tchan = &ud->tchans[chan_id]; 1517 uc->rchan = &ud->rchans[chan_id]; 1518 1519 /* UDMA does not use tx flows */ 1520 uc->tchan->tflow_id = -1; 1521 1522 return 0; 1523 } 1524 1525 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1526 { 1527 struct udma_dev *ud = uc->ud; 1528 int ret; 1529 1530 if (!uc->rchan) { 1531 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1532 return -EINVAL; 1533 } 1534 1535 if (uc->rflow) { 1536 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1537 uc->id, uc->rflow->id); 1538 return 0; 1539 } 1540 1541 uc->rflow = __udma_get_rflow(ud, flow_id); 1542 if (IS_ERR(uc->rflow)) { 1543 ret = PTR_ERR(uc->rflow); 1544 uc->rflow = NULL; 1545 return ret; 1546 } 1547 1548 return 0; 1549 } 1550 1551 static void bcdma_put_bchan(struct udma_chan *uc) 1552 { 1553 struct udma_dev *ud = uc->ud; 1554 1555 if (uc->bchan) { 1556 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id, 1557 uc->bchan->id); 1558 clear_bit(uc->bchan->id, ud->bchan_map); 1559 uc->bchan = NULL; 1560 uc->tchan = NULL; 1561 } 1562 } 1563 1564 static void udma_put_rchan(struct udma_chan *uc) 1565 { 1566 struct udma_dev *ud = uc->ud; 1567 1568 if (uc->rchan) { 1569 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1570 uc->rchan->id); 1571 clear_bit(uc->rchan->id, ud->rchan_map); 1572 uc->rchan = NULL; 1573 } 1574 } 1575 1576 static void udma_put_tchan(struct udma_chan *uc) 1577 { 1578 struct udma_dev *ud = uc->ud; 1579 1580 if (uc->tchan) { 1581 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1582 uc->tchan->id); 1583 clear_bit(uc->tchan->id, ud->tchan_map); 1584 1585 if (uc->tchan->tflow_id >= 0) 1586 clear_bit(uc->tchan->tflow_id, ud->tflow_map); 1587 1588 uc->tchan = NULL; 1589 } 1590 } 1591 1592 static void udma_put_rflow(struct udma_chan *uc) 1593 { 1594 struct udma_dev *ud = uc->ud; 1595 1596 if (uc->rflow) { 1597 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1598 uc->rflow->id); 1599 __udma_put_rflow(ud, uc->rflow); 1600 uc->rflow = NULL; 1601 } 1602 } 1603 1604 static void bcdma_free_bchan_resources(struct udma_chan *uc) 1605 { 1606 if (!uc->bchan) 1607 return; 1608 1609 k3_ringacc_ring_free(uc->bchan->tc_ring); 1610 k3_ringacc_ring_free(uc->bchan->t_ring); 1611 uc->bchan->tc_ring = NULL; 1612 uc->bchan->t_ring = NULL; 1613 k3_configure_chan_coherency(&uc->vc.chan, 0); 1614 1615 bcdma_put_bchan(uc); 1616 } 1617 1618 static int bcdma_alloc_bchan_resources(struct udma_chan *uc) 1619 { 1620 struct k3_ring_cfg ring_cfg; 1621 struct udma_dev *ud = uc->ud; 1622 int ret; 1623 1624 ret = bcdma_get_bchan(uc); 1625 if (ret) 1626 return ret; 1627 1628 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1, 1629 &uc->bchan->t_ring, 1630 &uc->bchan->tc_ring); 1631 if (ret) { 1632 ret = -EBUSY; 1633 goto err_ring; 1634 } 1635 1636 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1637 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1638 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1639 ring_cfg.mode = K3_RINGACC_RING_MODE_RING; 1640 1641 k3_configure_chan_coherency(&uc->vc.chan, ud->asel); 1642 ring_cfg.asel = ud->asel; 1643 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); 1644 1645 ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg); 1646 if (ret) 1647 goto err_ringcfg; 1648 1649 return 0; 1650 1651 err_ringcfg: 1652 k3_ringacc_ring_free(uc->bchan->tc_ring); 1653 uc->bchan->tc_ring = NULL; 1654 k3_ringacc_ring_free(uc->bchan->t_ring); 1655 uc->bchan->t_ring = NULL; 1656 k3_configure_chan_coherency(&uc->vc.chan, 0); 1657 err_ring: 1658 bcdma_put_bchan(uc); 1659 1660 return ret; 1661 } 1662 1663 static void udma_free_tx_resources(struct udma_chan *uc) 1664 { 1665 if (!uc->tchan) 1666 return; 1667 1668 k3_ringacc_ring_free(uc->tchan->t_ring); 1669 k3_ringacc_ring_free(uc->tchan->tc_ring); 1670 uc->tchan->t_ring = NULL; 1671 uc->tchan->tc_ring = NULL; 1672 1673 udma_put_tchan(uc); 1674 } 1675 1676 static int udma_alloc_tx_resources(struct udma_chan *uc) 1677 { 1678 struct k3_ring_cfg ring_cfg; 1679 struct udma_dev *ud = uc->ud; 1680 struct udma_tchan *tchan; 1681 int ring_idx, ret; 1682 1683 ret = udma_get_tchan(uc); 1684 if (ret) 1685 return ret; 1686 1687 tchan = uc->tchan; 1688 if (tchan->tflow_id >= 0) 1689 ring_idx = tchan->tflow_id; 1690 else 1691 ring_idx = ud->bchan_cnt + tchan->id; 1692 1693 ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1, 1694 &tchan->t_ring, 1695 &tchan->tc_ring); 1696 if (ret) { 1697 ret = -EBUSY; 1698 goto err_ring; 1699 } 1700 1701 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1702 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1703 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1704 if (ud->match_data->type == DMA_TYPE_UDMA) { 1705 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1706 } else { 1707 ring_cfg.mode = K3_RINGACC_RING_MODE_RING; 1708 1709 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel); 1710 ring_cfg.asel = uc->config.asel; 1711 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); 1712 } 1713 1714 ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg); 1715 ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg); 1716 1717 if (ret) 1718 goto err_ringcfg; 1719 1720 return 0; 1721 1722 err_ringcfg: 1723 k3_ringacc_ring_free(uc->tchan->tc_ring); 1724 uc->tchan->tc_ring = NULL; 1725 k3_ringacc_ring_free(uc->tchan->t_ring); 1726 uc->tchan->t_ring = NULL; 1727 err_ring: 1728 udma_put_tchan(uc); 1729 1730 return ret; 1731 } 1732 1733 static void udma_free_rx_resources(struct udma_chan *uc) 1734 { 1735 if (!uc->rchan) 1736 return; 1737 1738 if (uc->rflow) { 1739 struct udma_rflow *rflow = uc->rflow; 1740 1741 k3_ringacc_ring_free(rflow->fd_ring); 1742 k3_ringacc_ring_free(rflow->r_ring); 1743 rflow->fd_ring = NULL; 1744 rflow->r_ring = NULL; 1745 1746 udma_put_rflow(uc); 1747 } 1748 1749 udma_put_rchan(uc); 1750 } 1751 1752 static int udma_alloc_rx_resources(struct udma_chan *uc) 1753 { 1754 struct udma_dev *ud = uc->ud; 1755 struct k3_ring_cfg ring_cfg; 1756 struct udma_rflow *rflow; 1757 int fd_ring_id; 1758 int ret; 1759 1760 ret = udma_get_rchan(uc); 1761 if (ret) 1762 return ret; 1763 1764 /* For MEM_TO_MEM we don't need rflow or rings */ 1765 if (uc->config.dir == DMA_MEM_TO_MEM) 1766 return 0; 1767 1768 if (uc->config.default_flow_id >= 0) 1769 ret = udma_get_rflow(uc, uc->config.default_flow_id); 1770 else 1771 ret = udma_get_rflow(uc, uc->rchan->id); 1772 1773 if (ret) { 1774 ret = -EBUSY; 1775 goto err_rflow; 1776 } 1777 1778 rflow = uc->rflow; 1779 if (ud->tflow_cnt) 1780 fd_ring_id = ud->tflow_cnt + rflow->id; 1781 else 1782 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt + 1783 uc->rchan->id; 1784 1785 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, 1786 &rflow->fd_ring, &rflow->r_ring); 1787 if (ret) { 1788 ret = -EBUSY; 1789 goto err_ring; 1790 } 1791 1792 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1793 1794 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1795 if (ud->match_data->type == DMA_TYPE_UDMA) { 1796 if (uc->config.pkt_mode) 1797 ring_cfg.size = SG_MAX_SEGMENTS; 1798 else 1799 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1800 1801 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1802 } else { 1803 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1804 ring_cfg.mode = K3_RINGACC_RING_MODE_RING; 1805 1806 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel); 1807 ring_cfg.asel = uc->config.asel; 1808 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); 1809 } 1810 1811 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1812 1813 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1814 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1815 1816 if (ret) 1817 goto err_ringcfg; 1818 1819 return 0; 1820 1821 err_ringcfg: 1822 k3_ringacc_ring_free(rflow->r_ring); 1823 rflow->r_ring = NULL; 1824 k3_ringacc_ring_free(rflow->fd_ring); 1825 rflow->fd_ring = NULL; 1826 err_ring: 1827 udma_put_rflow(uc); 1828 err_rflow: 1829 udma_put_rchan(uc); 1830 1831 return ret; 1832 } 1833 1834 #define TISCI_BCDMA_BCHAN_VALID_PARAMS ( \ 1835 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1836 TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID) 1837 1838 #define TISCI_BCDMA_TCHAN_VALID_PARAMS ( \ 1839 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1840 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID) 1841 1842 #define TISCI_BCDMA_RCHAN_VALID_PARAMS ( \ 1843 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID) 1844 1845 #define TISCI_UDMA_TCHAN_VALID_PARAMS ( \ 1846 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1847 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1848 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1849 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1850 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1851 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1852 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1853 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1854 1855 #define TISCI_UDMA_RCHAN_VALID_PARAMS ( \ 1856 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1857 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1858 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1859 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1860 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1861 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1862 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1863 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1864 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1865 1866 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1867 { 1868 struct udma_dev *ud = uc->ud; 1869 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1870 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1871 struct udma_tchan *tchan = uc->tchan; 1872 struct udma_rchan *rchan = uc->rchan; 1873 u8 burst_size = 0; 1874 int ret; 1875 u8 tpl; 1876 1877 /* Non synchronized - mem to mem type of transfer */ 1878 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1879 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1880 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1881 1882 if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { 1883 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id); 1884 1885 burst_size = ud->match_data->burst_size[tpl]; 1886 } 1887 1888 req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; 1889 req_tx.nav_id = tisci_rm->tisci_dev_id; 1890 req_tx.index = tchan->id; 1891 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1892 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1893 req_tx.txcq_qnum = tc_ring; 1894 req_tx.tx_atype = ud->atype; 1895 if (burst_size) { 1896 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; 1897 req_tx.tx_burst_size = burst_size; 1898 } 1899 1900 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1901 if (ret) { 1902 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1903 return ret; 1904 } 1905 1906 req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS; 1907 req_rx.nav_id = tisci_rm->tisci_dev_id; 1908 req_rx.index = rchan->id; 1909 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1910 req_rx.rxcq_qnum = tc_ring; 1911 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1912 req_rx.rx_atype = ud->atype; 1913 if (burst_size) { 1914 req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; 1915 req_rx.rx_burst_size = burst_size; 1916 } 1917 1918 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1919 if (ret) 1920 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1921 1922 return ret; 1923 } 1924 1925 static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc) 1926 { 1927 struct udma_dev *ud = uc->ud; 1928 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1929 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1930 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1931 struct udma_bchan *bchan = uc->bchan; 1932 u8 burst_size = 0; 1933 int ret; 1934 u8 tpl; 1935 1936 if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { 1937 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id); 1938 1939 burst_size = ud->match_data->burst_size[tpl]; 1940 } 1941 1942 req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS; 1943 req_tx.nav_id = tisci_rm->tisci_dev_id; 1944 req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN; 1945 req_tx.index = bchan->id; 1946 if (burst_size) { 1947 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; 1948 req_tx.tx_burst_size = burst_size; 1949 } 1950 1951 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1952 if (ret) 1953 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret); 1954 1955 return ret; 1956 } 1957 1958 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1959 { 1960 struct udma_dev *ud = uc->ud; 1961 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1962 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1963 struct udma_tchan *tchan = uc->tchan; 1964 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1965 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1966 u32 mode, fetch_size; 1967 int ret; 1968 1969 if (uc->config.pkt_mode) { 1970 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1971 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1972 uc->config.psd_size, 0); 1973 } else { 1974 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1975 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1976 } 1977 1978 req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; 1979 req_tx.nav_id = tisci_rm->tisci_dev_id; 1980 req_tx.index = tchan->id; 1981 req_tx.tx_chan_type = mode; 1982 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1983 req_tx.tx_fetch_size = fetch_size >> 2; 1984 req_tx.txcq_qnum = tc_ring; 1985 req_tx.tx_atype = uc->config.atype; 1986 if (uc->config.ep_type == PSIL_EP_PDMA_XY && 1987 ud->match_data->flags & UDMA_FLAG_TDTYPE) { 1988 /* wait for peer to complete the teardown for PDMAs */ 1989 req_tx.valid_params |= 1990 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID; 1991 req_tx.tx_tdtype = 1; 1992 } 1993 1994 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1995 if (ret) 1996 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1997 1998 return ret; 1999 } 2000 2001 static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) 2002 { 2003 struct udma_dev *ud = uc->ud; 2004 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2005 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2006 struct udma_tchan *tchan = uc->tchan; 2007 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 2008 int ret; 2009 2010 req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS; 2011 req_tx.nav_id = tisci_rm->tisci_dev_id; 2012 req_tx.index = tchan->id; 2013 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 2014 if (ud->match_data->flags & UDMA_FLAG_TDTYPE) { 2015 /* wait for peer to complete the teardown for PDMAs */ 2016 req_tx.valid_params |= 2017 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID; 2018 req_tx.tx_tdtype = 1; 2019 } 2020 2021 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 2022 if (ret) 2023 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 2024 2025 return ret; 2026 } 2027 2028 #define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config 2029 2030 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 2031 { 2032 struct udma_dev *ud = uc->ud; 2033 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2034 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2035 struct udma_rchan *rchan = uc->rchan; 2036 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 2037 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2038 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 2039 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 2040 u32 mode, fetch_size; 2041 int ret; 2042 2043 if (uc->config.pkt_mode) { 2044 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 2045 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 2046 uc->config.psd_size, 0); 2047 } else { 2048 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 2049 fetch_size = sizeof(struct cppi5_desc_hdr_t); 2050 } 2051 2052 req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS; 2053 req_rx.nav_id = tisci_rm->tisci_dev_id; 2054 req_rx.index = rchan->id; 2055 req_rx.rx_fetch_size = fetch_size >> 2; 2056 req_rx.rxcq_qnum = rx_ring; 2057 req_rx.rx_chan_type = mode; 2058 req_rx.rx_atype = uc->config.atype; 2059 2060 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 2061 if (ret) { 2062 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 2063 return ret; 2064 } 2065 2066 flow_req.valid_params = 2067 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 2068 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 2069 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 2070 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 2071 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 2072 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 2073 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 2074 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 2075 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 2076 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 2077 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 2078 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 2079 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 2080 2081 flow_req.nav_id = tisci_rm->tisci_dev_id; 2082 flow_req.flow_index = rchan->id; 2083 2084 if (uc->config.needs_epib) 2085 flow_req.rx_einfo_present = 1; 2086 else 2087 flow_req.rx_einfo_present = 0; 2088 if (uc->config.psd_size) 2089 flow_req.rx_psinfo_present = 1; 2090 else 2091 flow_req.rx_psinfo_present = 0; 2092 flow_req.rx_error_handling = 1; 2093 flow_req.rx_dest_qnum = rx_ring; 2094 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 2095 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 2096 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 2097 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 2098 flow_req.rx_fdq0_sz0_qnum = fd_ring; 2099 flow_req.rx_fdq1_qnum = fd_ring; 2100 flow_req.rx_fdq2_qnum = fd_ring; 2101 flow_req.rx_fdq3_qnum = fd_ring; 2102 2103 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 2104 2105 if (ret) 2106 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 2107 2108 return 0; 2109 } 2110 2111 static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) 2112 { 2113 struct udma_dev *ud = uc->ud; 2114 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2115 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2116 struct udma_rchan *rchan = uc->rchan; 2117 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 2118 int ret; 2119 2120 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; 2121 req_rx.nav_id = tisci_rm->tisci_dev_id; 2122 req_rx.index = rchan->id; 2123 2124 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 2125 if (ret) 2126 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 2127 2128 return ret; 2129 } 2130 2131 static int pktdma_tisci_rx_channel_config(struct udma_chan *uc) 2132 { 2133 struct udma_dev *ud = uc->ud; 2134 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2135 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2136 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 2137 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 2138 int ret; 2139 2140 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; 2141 req_rx.nav_id = tisci_rm->tisci_dev_id; 2142 req_rx.index = uc->rchan->id; 2143 2144 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 2145 if (ret) { 2146 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret); 2147 return ret; 2148 } 2149 2150 flow_req.valid_params = 2151 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 2152 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 2153 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID; 2154 2155 flow_req.nav_id = tisci_rm->tisci_dev_id; 2156 flow_req.flow_index = uc->rflow->id; 2157 2158 if (uc->config.needs_epib) 2159 flow_req.rx_einfo_present = 1; 2160 else 2161 flow_req.rx_einfo_present = 0; 2162 if (uc->config.psd_size) 2163 flow_req.rx_psinfo_present = 1; 2164 else 2165 flow_req.rx_psinfo_present = 0; 2166 flow_req.rx_error_handling = 1; 2167 2168 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 2169 2170 if (ret) 2171 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id, 2172 ret); 2173 2174 return ret; 2175 } 2176 2177 static int udma_alloc_chan_resources(struct dma_chan *chan) 2178 { 2179 struct udma_chan *uc = to_udma_chan(chan); 2180 struct udma_dev *ud = to_udma_dev(chan->device); 2181 const struct udma_soc_data *soc_data = ud->soc_data; 2182 struct k3_ring *irq_ring; 2183 u32 irq_udma_idx; 2184 int ret; 2185 2186 uc->dma_dev = ud->dev; 2187 2188 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 2189 uc->use_dma_pool = true; 2190 /* in case of MEM_TO_MEM we have maximum of two TRs */ 2191 if (uc->config.dir == DMA_MEM_TO_MEM) { 2192 uc->config.hdesc_size = cppi5_trdesc_calc_size( 2193 sizeof(struct cppi5_tr_type15_t), 2); 2194 uc->config.pkt_mode = false; 2195 } 2196 } 2197 2198 if (uc->use_dma_pool) { 2199 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 2200 uc->config.hdesc_size, 2201 ud->desc_align, 2202 0); 2203 if (!uc->hdesc_pool) { 2204 dev_err(ud->ddev.dev, 2205 "Descriptor pool allocation failed\n"); 2206 uc->use_dma_pool = false; 2207 ret = -ENOMEM; 2208 goto err_cleanup; 2209 } 2210 } 2211 2212 /* 2213 * Make sure that the completion is in a known state: 2214 * No teardown, the channel is idle 2215 */ 2216 reinit_completion(&uc->teardown_completed); 2217 complete_all(&uc->teardown_completed); 2218 uc->state = UDMA_CHAN_IS_IDLE; 2219 2220 switch (uc->config.dir) { 2221 case DMA_MEM_TO_MEM: 2222 /* Non synchronized - mem to mem type of transfer */ 2223 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 2224 uc->id); 2225 2226 ret = udma_get_chan_pair(uc); 2227 if (ret) 2228 goto err_cleanup; 2229 2230 ret = udma_alloc_tx_resources(uc); 2231 if (ret) { 2232 udma_put_rchan(uc); 2233 goto err_cleanup; 2234 } 2235 2236 ret = udma_alloc_rx_resources(uc); 2237 if (ret) { 2238 udma_free_tx_resources(uc); 2239 goto err_cleanup; 2240 } 2241 2242 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2243 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2244 K3_PSIL_DST_THREAD_ID_OFFSET; 2245 2246 irq_ring = uc->tchan->tc_ring; 2247 irq_udma_idx = uc->tchan->id; 2248 2249 ret = udma_tisci_m2m_channel_config(uc); 2250 break; 2251 case DMA_MEM_TO_DEV: 2252 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 2253 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 2254 uc->id); 2255 2256 ret = udma_alloc_tx_resources(uc); 2257 if (ret) 2258 goto err_cleanup; 2259 2260 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2261 uc->config.dst_thread = uc->config.remote_thread_id; 2262 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 2263 2264 irq_ring = uc->tchan->tc_ring; 2265 irq_udma_idx = uc->tchan->id; 2266 2267 ret = udma_tisci_tx_channel_config(uc); 2268 break; 2269 case DMA_DEV_TO_MEM: 2270 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 2271 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 2272 uc->id); 2273 2274 ret = udma_alloc_rx_resources(uc); 2275 if (ret) 2276 goto err_cleanup; 2277 2278 uc->config.src_thread = uc->config.remote_thread_id; 2279 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2280 K3_PSIL_DST_THREAD_ID_OFFSET; 2281 2282 irq_ring = uc->rflow->r_ring; 2283 irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id; 2284 2285 ret = udma_tisci_rx_channel_config(uc); 2286 break; 2287 default: 2288 /* Can not happen */ 2289 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 2290 __func__, uc->id, uc->config.dir); 2291 ret = -EINVAL; 2292 goto err_cleanup; 2293 2294 } 2295 2296 /* check if the channel configuration was successful */ 2297 if (ret) 2298 goto err_res_free; 2299 2300 if (udma_is_chan_running(uc)) { 2301 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 2302 udma_reset_chan(uc, false); 2303 if (udma_is_chan_running(uc)) { 2304 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 2305 ret = -EBUSY; 2306 goto err_res_free; 2307 } 2308 } 2309 2310 /* PSI-L pairing */ 2311 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 2312 if (ret) { 2313 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 2314 uc->config.src_thread, uc->config.dst_thread); 2315 goto err_res_free; 2316 } 2317 2318 uc->psil_paired = true; 2319 2320 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 2321 if (uc->irq_num_ring <= 0) { 2322 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 2323 k3_ringacc_get_ring_id(irq_ring)); 2324 ret = -EINVAL; 2325 goto err_psi_free; 2326 } 2327 2328 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 2329 IRQF_TRIGGER_HIGH, uc->name, uc); 2330 if (ret) { 2331 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 2332 goto err_irq_free; 2333 } 2334 2335 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 2336 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 2337 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); 2338 if (uc->irq_num_udma <= 0) { 2339 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 2340 irq_udma_idx); 2341 free_irq(uc->irq_num_ring, uc); 2342 ret = -EINVAL; 2343 goto err_irq_free; 2344 } 2345 2346 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 2347 uc->name, uc); 2348 if (ret) { 2349 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 2350 uc->id); 2351 free_irq(uc->irq_num_ring, uc); 2352 goto err_irq_free; 2353 } 2354 } else { 2355 uc->irq_num_udma = 0; 2356 } 2357 2358 udma_reset_rings(uc); 2359 2360 return 0; 2361 2362 err_irq_free: 2363 uc->irq_num_ring = 0; 2364 uc->irq_num_udma = 0; 2365 err_psi_free: 2366 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 2367 uc->psil_paired = false; 2368 err_res_free: 2369 udma_free_tx_resources(uc); 2370 udma_free_rx_resources(uc); 2371 err_cleanup: 2372 udma_reset_uchan(uc); 2373 2374 if (uc->use_dma_pool) { 2375 dma_pool_destroy(uc->hdesc_pool); 2376 uc->use_dma_pool = false; 2377 } 2378 2379 return ret; 2380 } 2381 2382 static int bcdma_alloc_chan_resources(struct dma_chan *chan) 2383 { 2384 struct udma_chan *uc = to_udma_chan(chan); 2385 struct udma_dev *ud = to_udma_dev(chan->device); 2386 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 2387 u32 irq_udma_idx, irq_ring_idx; 2388 int ret; 2389 2390 /* Only TR mode is supported */ 2391 uc->config.pkt_mode = false; 2392 2393 /* 2394 * Make sure that the completion is in a known state: 2395 * No teardown, the channel is idle 2396 */ 2397 reinit_completion(&uc->teardown_completed); 2398 complete_all(&uc->teardown_completed); 2399 uc->state = UDMA_CHAN_IS_IDLE; 2400 2401 switch (uc->config.dir) { 2402 case DMA_MEM_TO_MEM: 2403 /* Non synchronized - mem to mem type of transfer */ 2404 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 2405 uc->id); 2406 2407 ret = bcdma_alloc_bchan_resources(uc); 2408 if (ret) 2409 return ret; 2410 2411 irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring; 2412 irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data; 2413 2414 ret = bcdma_tisci_m2m_channel_config(uc); 2415 break; 2416 case DMA_MEM_TO_DEV: 2417 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 2418 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 2419 uc->id); 2420 2421 ret = udma_alloc_tx_resources(uc); 2422 if (ret) { 2423 uc->config.remote_thread_id = -1; 2424 return ret; 2425 } 2426 2427 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2428 uc->config.dst_thread = uc->config.remote_thread_id; 2429 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 2430 2431 irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring; 2432 irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data; 2433 2434 ret = bcdma_tisci_tx_channel_config(uc); 2435 break; 2436 case DMA_DEV_TO_MEM: 2437 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 2438 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 2439 uc->id); 2440 2441 ret = udma_alloc_rx_resources(uc); 2442 if (ret) { 2443 uc->config.remote_thread_id = -1; 2444 return ret; 2445 } 2446 2447 uc->config.src_thread = uc->config.remote_thread_id; 2448 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2449 K3_PSIL_DST_THREAD_ID_OFFSET; 2450 2451 irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring; 2452 irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data; 2453 2454 ret = bcdma_tisci_rx_channel_config(uc); 2455 break; 2456 default: 2457 /* Can not happen */ 2458 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 2459 __func__, uc->id, uc->config.dir); 2460 return -EINVAL; 2461 } 2462 2463 /* check if the channel configuration was successful */ 2464 if (ret) 2465 goto err_res_free; 2466 2467 if (udma_is_chan_running(uc)) { 2468 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 2469 udma_reset_chan(uc, false); 2470 if (udma_is_chan_running(uc)) { 2471 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 2472 ret = -EBUSY; 2473 goto err_res_free; 2474 } 2475 } 2476 2477 uc->dma_dev = dmaengine_get_dma_device(chan); 2478 if (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type) { 2479 uc->config.hdesc_size = cppi5_trdesc_calc_size( 2480 sizeof(struct cppi5_tr_type15_t), 2); 2481 2482 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 2483 uc->config.hdesc_size, 2484 ud->desc_align, 2485 0); 2486 if (!uc->hdesc_pool) { 2487 dev_err(ud->ddev.dev, 2488 "Descriptor pool allocation failed\n"); 2489 uc->use_dma_pool = false; 2490 ret = -ENOMEM; 2491 goto err_res_free; 2492 } 2493 2494 uc->use_dma_pool = true; 2495 } else if (uc->config.dir != DMA_MEM_TO_MEM) { 2496 /* PSI-L pairing */ 2497 ret = navss_psil_pair(ud, uc->config.src_thread, 2498 uc->config.dst_thread); 2499 if (ret) { 2500 dev_err(ud->dev, 2501 "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 2502 uc->config.src_thread, uc->config.dst_thread); 2503 goto err_res_free; 2504 } 2505 2506 uc->psil_paired = true; 2507 } 2508 2509 uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); 2510 if (uc->irq_num_ring <= 0) { 2511 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 2512 irq_ring_idx); 2513 ret = -EINVAL; 2514 goto err_psi_free; 2515 } 2516 2517 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 2518 IRQF_TRIGGER_HIGH, uc->name, uc); 2519 if (ret) { 2520 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 2521 goto err_irq_free; 2522 } 2523 2524 /* Event from BCDMA (TR events) only needed for slave channels */ 2525 if (is_slave_direction(uc->config.dir)) { 2526 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); 2527 if (uc->irq_num_udma <= 0) { 2528 dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n", 2529 irq_udma_idx); 2530 free_irq(uc->irq_num_ring, uc); 2531 ret = -EINVAL; 2532 goto err_irq_free; 2533 } 2534 2535 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 2536 uc->name, uc); 2537 if (ret) { 2538 dev_err(ud->dev, "chan%d: BCDMA irq request failed\n", 2539 uc->id); 2540 free_irq(uc->irq_num_ring, uc); 2541 goto err_irq_free; 2542 } 2543 } else { 2544 uc->irq_num_udma = 0; 2545 } 2546 2547 udma_reset_rings(uc); 2548 2549 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 2550 udma_check_tx_completion); 2551 return 0; 2552 2553 err_irq_free: 2554 uc->irq_num_ring = 0; 2555 uc->irq_num_udma = 0; 2556 err_psi_free: 2557 if (uc->psil_paired) 2558 navss_psil_unpair(ud, uc->config.src_thread, 2559 uc->config.dst_thread); 2560 uc->psil_paired = false; 2561 err_res_free: 2562 bcdma_free_bchan_resources(uc); 2563 udma_free_tx_resources(uc); 2564 udma_free_rx_resources(uc); 2565 2566 udma_reset_uchan(uc); 2567 2568 if (uc->use_dma_pool) { 2569 dma_pool_destroy(uc->hdesc_pool); 2570 uc->use_dma_pool = false; 2571 } 2572 2573 return ret; 2574 } 2575 2576 static int bcdma_router_config(struct dma_chan *chan) 2577 { 2578 struct k3_event_route_data *router_data = chan->route_data; 2579 struct udma_chan *uc = to_udma_chan(chan); 2580 u32 trigger_event; 2581 2582 if (!uc->bchan) 2583 return -EINVAL; 2584 2585 if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2) 2586 return -EINVAL; 2587 2588 trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset; 2589 trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1; 2590 2591 return router_data->set_event(router_data->priv, trigger_event); 2592 } 2593 2594 static int pktdma_alloc_chan_resources(struct dma_chan *chan) 2595 { 2596 struct udma_chan *uc = to_udma_chan(chan); 2597 struct udma_dev *ud = to_udma_dev(chan->device); 2598 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 2599 u32 irq_ring_idx; 2600 int ret; 2601 2602 /* 2603 * Make sure that the completion is in a known state: 2604 * No teardown, the channel is idle 2605 */ 2606 reinit_completion(&uc->teardown_completed); 2607 complete_all(&uc->teardown_completed); 2608 uc->state = UDMA_CHAN_IS_IDLE; 2609 2610 switch (uc->config.dir) { 2611 case DMA_MEM_TO_DEV: 2612 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 2613 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 2614 uc->id); 2615 2616 ret = udma_alloc_tx_resources(uc); 2617 if (ret) { 2618 uc->config.remote_thread_id = -1; 2619 return ret; 2620 } 2621 2622 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2623 uc->config.dst_thread = uc->config.remote_thread_id; 2624 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 2625 2626 irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow; 2627 2628 ret = pktdma_tisci_tx_channel_config(uc); 2629 break; 2630 case DMA_DEV_TO_MEM: 2631 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 2632 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 2633 uc->id); 2634 2635 ret = udma_alloc_rx_resources(uc); 2636 if (ret) { 2637 uc->config.remote_thread_id = -1; 2638 return ret; 2639 } 2640 2641 uc->config.src_thread = uc->config.remote_thread_id; 2642 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2643 K3_PSIL_DST_THREAD_ID_OFFSET; 2644 2645 irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow; 2646 2647 ret = pktdma_tisci_rx_channel_config(uc); 2648 break; 2649 default: 2650 /* Can not happen */ 2651 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 2652 __func__, uc->id, uc->config.dir); 2653 return -EINVAL; 2654 } 2655 2656 /* check if the channel configuration was successful */ 2657 if (ret) 2658 goto err_res_free; 2659 2660 if (udma_is_chan_running(uc)) { 2661 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 2662 udma_reset_chan(uc, false); 2663 if (udma_is_chan_running(uc)) { 2664 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 2665 ret = -EBUSY; 2666 goto err_res_free; 2667 } 2668 } 2669 2670 uc->dma_dev = dmaengine_get_dma_device(chan); 2671 uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev, 2672 uc->config.hdesc_size, ud->desc_align, 2673 0); 2674 if (!uc->hdesc_pool) { 2675 dev_err(ud->ddev.dev, 2676 "Descriptor pool allocation failed\n"); 2677 uc->use_dma_pool = false; 2678 ret = -ENOMEM; 2679 goto err_res_free; 2680 } 2681 2682 uc->use_dma_pool = true; 2683 2684 /* PSI-L pairing */ 2685 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 2686 if (ret) { 2687 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 2688 uc->config.src_thread, uc->config.dst_thread); 2689 goto err_res_free; 2690 } 2691 2692 uc->psil_paired = true; 2693 2694 uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); 2695 if (uc->irq_num_ring <= 0) { 2696 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 2697 irq_ring_idx); 2698 ret = -EINVAL; 2699 goto err_psi_free; 2700 } 2701 2702 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 2703 IRQF_TRIGGER_HIGH, uc->name, uc); 2704 if (ret) { 2705 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 2706 goto err_irq_free; 2707 } 2708 2709 uc->irq_num_udma = 0; 2710 2711 udma_reset_rings(uc); 2712 2713 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 2714 udma_check_tx_completion); 2715 2716 if (uc->tchan) 2717 dev_dbg(ud->dev, 2718 "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n", 2719 uc->id, uc->tchan->id, uc->tchan->tflow_id, 2720 uc->config.remote_thread_id); 2721 else if (uc->rchan) 2722 dev_dbg(ud->dev, 2723 "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n", 2724 uc->id, uc->rchan->id, uc->rflow->id, 2725 uc->config.remote_thread_id); 2726 return 0; 2727 2728 err_irq_free: 2729 uc->irq_num_ring = 0; 2730 err_psi_free: 2731 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 2732 uc->psil_paired = false; 2733 err_res_free: 2734 udma_free_tx_resources(uc); 2735 udma_free_rx_resources(uc); 2736 2737 udma_reset_uchan(uc); 2738 2739 dma_pool_destroy(uc->hdesc_pool); 2740 uc->use_dma_pool = false; 2741 2742 return ret; 2743 } 2744 2745 static int udma_slave_config(struct dma_chan *chan, 2746 struct dma_slave_config *cfg) 2747 { 2748 struct udma_chan *uc = to_udma_chan(chan); 2749 2750 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 2751 2752 return 0; 2753 } 2754 2755 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 2756 size_t tr_size, int tr_count, 2757 enum dma_transfer_direction dir) 2758 { 2759 struct udma_hwdesc *hwdesc; 2760 struct cppi5_desc_hdr_t *tr_desc; 2761 struct udma_desc *d; 2762 u32 reload_count = 0; 2763 u32 ring_id; 2764 2765 switch (tr_size) { 2766 case 16: 2767 case 32: 2768 case 64: 2769 case 128: 2770 break; 2771 default: 2772 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 2773 return NULL; 2774 } 2775 2776 /* We have only one descriptor containing multiple TRs */ 2777 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 2778 if (!d) 2779 return NULL; 2780 2781 d->sglen = tr_count; 2782 2783 d->hwdesc_count = 1; 2784 hwdesc = &d->hwdesc[0]; 2785 2786 /* Allocate memory for DMA ring descriptor */ 2787 if (uc->use_dma_pool) { 2788 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2789 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2790 GFP_NOWAIT, 2791 &hwdesc->cppi5_desc_paddr); 2792 } else { 2793 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 2794 tr_count); 2795 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 2796 uc->ud->desc_align); 2797 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 2798 hwdesc->cppi5_desc_size, 2799 &hwdesc->cppi5_desc_paddr, 2800 GFP_NOWAIT); 2801 } 2802 2803 if (!hwdesc->cppi5_desc_vaddr) { 2804 kfree(d); 2805 return NULL; 2806 } 2807 2808 /* Start of the TR req records */ 2809 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 2810 /* Start address of the TR response array */ 2811 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 2812 2813 tr_desc = hwdesc->cppi5_desc_vaddr; 2814 2815 if (uc->cyclic) 2816 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 2817 2818 if (dir == DMA_DEV_TO_MEM) 2819 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2820 else 2821 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2822 2823 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 2824 cppi5_desc_set_pktids(tr_desc, uc->id, 2825 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2826 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 2827 2828 return d; 2829 } 2830 2831 /** 2832 * udma_get_tr_counters - calculate TR counters for a given length 2833 * @len: Length of the trasnfer 2834 * @align_to: Preferred alignment 2835 * @tr0_cnt0: First TR icnt0 2836 * @tr0_cnt1: First TR icnt1 2837 * @tr1_cnt0: Second (if used) TR icnt0 2838 * 2839 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated 2840 * For len >= SZ_64K two TRs are used in a simple way: 2841 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) 2842 * Second TR: the remaining length (tr1_cnt0) 2843 * 2844 * Returns the number of TRs the length needs (1 or 2) 2845 * -EINVAL if the length can not be supported 2846 */ 2847 static int udma_get_tr_counters(size_t len, unsigned long align_to, 2848 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) 2849 { 2850 if (len < SZ_64K) { 2851 *tr0_cnt0 = len; 2852 *tr0_cnt1 = 1; 2853 2854 return 1; 2855 } 2856 2857 if (align_to > 3) 2858 align_to = 3; 2859 2860 realign: 2861 *tr0_cnt0 = SZ_64K - BIT(align_to); 2862 if (len / *tr0_cnt0 >= SZ_64K) { 2863 if (align_to) { 2864 align_to--; 2865 goto realign; 2866 } 2867 return -EINVAL; 2868 } 2869 2870 *tr0_cnt1 = len / *tr0_cnt0; 2871 *tr1_cnt0 = len % *tr0_cnt0; 2872 2873 return 2; 2874 } 2875 2876 static struct udma_desc * 2877 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 2878 unsigned int sglen, enum dma_transfer_direction dir, 2879 unsigned long tx_flags, void *context) 2880 { 2881 struct scatterlist *sgent; 2882 struct udma_desc *d; 2883 struct cppi5_tr_type1_t *tr_req = NULL; 2884 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2885 unsigned int i; 2886 size_t tr_size; 2887 int num_tr = 0; 2888 int tr_idx = 0; 2889 u64 asel; 2890 2891 /* estimate the number of TRs we will need */ 2892 for_each_sg(sgl, sgent, sglen, i) { 2893 if (sg_dma_len(sgent) < SZ_64K) 2894 num_tr++; 2895 else 2896 num_tr += 2; 2897 } 2898 2899 /* Now allocate and setup the descriptor. */ 2900 tr_size = sizeof(struct cppi5_tr_type1_t); 2901 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 2902 if (!d) 2903 return NULL; 2904 2905 d->sglen = sglen; 2906 2907 if (uc->ud->match_data->type == DMA_TYPE_UDMA) 2908 asel = 0; 2909 else 2910 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 2911 2912 tr_req = d->hwdesc[0].tr_req_base; 2913 for_each_sg(sgl, sgent, sglen, i) { 2914 dma_addr_t sg_addr = sg_dma_address(sgent); 2915 2916 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), 2917 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); 2918 if (num_tr < 0) { 2919 dev_err(uc->ud->dev, "size %u is not supported\n", 2920 sg_dma_len(sgent)); 2921 udma_free_hwdesc(uc, d); 2922 kfree(d); 2923 return NULL; 2924 } 2925 2926 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2927 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2928 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); 2929 2930 sg_addr |= asel; 2931 tr_req[tr_idx].addr = sg_addr; 2932 tr_req[tr_idx].icnt0 = tr0_cnt0; 2933 tr_req[tr_idx].icnt1 = tr0_cnt1; 2934 tr_req[tr_idx].dim1 = tr0_cnt0; 2935 tr_idx++; 2936 2937 if (num_tr == 2) { 2938 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2939 false, false, 2940 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2941 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2942 CPPI5_TR_CSF_SUPR_EVT); 2943 2944 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; 2945 tr_req[tr_idx].icnt0 = tr1_cnt0; 2946 tr_req[tr_idx].icnt1 = 1; 2947 tr_req[tr_idx].dim1 = tr1_cnt0; 2948 tr_idx++; 2949 } 2950 2951 d->residue += sg_dma_len(sgent); 2952 } 2953 2954 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, 2955 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2956 2957 return d; 2958 } 2959 2960 static struct udma_desc * 2961 udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, 2962 unsigned int sglen, 2963 enum dma_transfer_direction dir, 2964 unsigned long tx_flags, void *context) 2965 { 2966 struct scatterlist *sgent; 2967 struct cppi5_tr_type15_t *tr_req = NULL; 2968 enum dma_slave_buswidth dev_width; 2969 u32 csf = CPPI5_TR_CSF_SUPR_EVT; 2970 u16 tr_cnt0, tr_cnt1; 2971 dma_addr_t dev_addr; 2972 struct udma_desc *d; 2973 unsigned int i; 2974 size_t tr_size, sg_len; 2975 int num_tr = 0; 2976 int tr_idx = 0; 2977 u32 burst, trigger_size, port_window; 2978 u64 asel; 2979 2980 if (dir == DMA_DEV_TO_MEM) { 2981 dev_addr = uc->cfg.src_addr; 2982 dev_width = uc->cfg.src_addr_width; 2983 burst = uc->cfg.src_maxburst; 2984 port_window = uc->cfg.src_port_window_size; 2985 } else if (dir == DMA_MEM_TO_DEV) { 2986 dev_addr = uc->cfg.dst_addr; 2987 dev_width = uc->cfg.dst_addr_width; 2988 burst = uc->cfg.dst_maxburst; 2989 port_window = uc->cfg.dst_port_window_size; 2990 } else { 2991 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2992 return NULL; 2993 } 2994 2995 if (!burst) 2996 burst = 1; 2997 2998 if (port_window) { 2999 if (port_window != burst) { 3000 dev_err(uc->ud->dev, 3001 "The burst must be equal to port_window\n"); 3002 return NULL; 3003 } 3004 3005 tr_cnt0 = dev_width * port_window; 3006 tr_cnt1 = 1; 3007 } else { 3008 tr_cnt0 = dev_width; 3009 tr_cnt1 = burst; 3010 } 3011 trigger_size = tr_cnt0 * tr_cnt1; 3012 3013 /* estimate the number of TRs we will need */ 3014 for_each_sg(sgl, sgent, sglen, i) { 3015 sg_len = sg_dma_len(sgent); 3016 3017 if (sg_len % trigger_size) { 3018 dev_err(uc->ud->dev, 3019 "Not aligned SG entry (%zu for %u)\n", sg_len, 3020 trigger_size); 3021 return NULL; 3022 } 3023 3024 if (sg_len / trigger_size < SZ_64K) 3025 num_tr++; 3026 else 3027 num_tr += 2; 3028 } 3029 3030 /* Now allocate and setup the descriptor. */ 3031 tr_size = sizeof(struct cppi5_tr_type15_t); 3032 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 3033 if (!d) 3034 return NULL; 3035 3036 d->sglen = sglen; 3037 3038 if (uc->ud->match_data->type == DMA_TYPE_UDMA) { 3039 asel = 0; 3040 csf |= CPPI5_TR_CSF_EOL_ICNT0; 3041 } else { 3042 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 3043 dev_addr |= asel; 3044 } 3045 3046 tr_req = d->hwdesc[0].tr_req_base; 3047 for_each_sg(sgl, sgent, sglen, i) { 3048 u16 tr0_cnt2, tr0_cnt3, tr1_cnt2; 3049 dma_addr_t sg_addr = sg_dma_address(sgent); 3050 3051 sg_len = sg_dma_len(sgent); 3052 num_tr = udma_get_tr_counters(sg_len / trigger_size, 0, 3053 &tr0_cnt2, &tr0_cnt3, &tr1_cnt2); 3054 if (num_tr < 0) { 3055 dev_err(uc->ud->dev, "size %zu is not supported\n", 3056 sg_len); 3057 udma_free_hwdesc(uc, d); 3058 kfree(d); 3059 return NULL; 3060 } 3061 3062 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, 3063 true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3064 cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); 3065 cppi5_tr_set_trigger(&tr_req[tr_idx].flags, 3066 uc->config.tr_trigger_type, 3067 CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0); 3068 3069 sg_addr |= asel; 3070 if (dir == DMA_DEV_TO_MEM) { 3071 tr_req[tr_idx].addr = dev_addr; 3072 tr_req[tr_idx].icnt0 = tr_cnt0; 3073 tr_req[tr_idx].icnt1 = tr_cnt1; 3074 tr_req[tr_idx].icnt2 = tr0_cnt2; 3075 tr_req[tr_idx].icnt3 = tr0_cnt3; 3076 tr_req[tr_idx].dim1 = (-1) * tr_cnt0; 3077 3078 tr_req[tr_idx].daddr = sg_addr; 3079 tr_req[tr_idx].dicnt0 = tr_cnt0; 3080 tr_req[tr_idx].dicnt1 = tr_cnt1; 3081 tr_req[tr_idx].dicnt2 = tr0_cnt2; 3082 tr_req[tr_idx].dicnt3 = tr0_cnt3; 3083 tr_req[tr_idx].ddim1 = tr_cnt0; 3084 tr_req[tr_idx].ddim2 = trigger_size; 3085 tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2; 3086 } else { 3087 tr_req[tr_idx].addr = sg_addr; 3088 tr_req[tr_idx].icnt0 = tr_cnt0; 3089 tr_req[tr_idx].icnt1 = tr_cnt1; 3090 tr_req[tr_idx].icnt2 = tr0_cnt2; 3091 tr_req[tr_idx].icnt3 = tr0_cnt3; 3092 tr_req[tr_idx].dim1 = tr_cnt0; 3093 tr_req[tr_idx].dim2 = trigger_size; 3094 tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2; 3095 3096 tr_req[tr_idx].daddr = dev_addr; 3097 tr_req[tr_idx].dicnt0 = tr_cnt0; 3098 tr_req[tr_idx].dicnt1 = tr_cnt1; 3099 tr_req[tr_idx].dicnt2 = tr0_cnt2; 3100 tr_req[tr_idx].dicnt3 = tr0_cnt3; 3101 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0; 3102 } 3103 3104 tr_idx++; 3105 3106 if (num_tr == 2) { 3107 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, 3108 false, true, 3109 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3110 cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); 3111 cppi5_tr_set_trigger(&tr_req[tr_idx].flags, 3112 uc->config.tr_trigger_type, 3113 CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 3114 0, 0); 3115 3116 sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3; 3117 if (dir == DMA_DEV_TO_MEM) { 3118 tr_req[tr_idx].addr = dev_addr; 3119 tr_req[tr_idx].icnt0 = tr_cnt0; 3120 tr_req[tr_idx].icnt1 = tr_cnt1; 3121 tr_req[tr_idx].icnt2 = tr1_cnt2; 3122 tr_req[tr_idx].icnt3 = 1; 3123 tr_req[tr_idx].dim1 = (-1) * tr_cnt0; 3124 3125 tr_req[tr_idx].daddr = sg_addr; 3126 tr_req[tr_idx].dicnt0 = tr_cnt0; 3127 tr_req[tr_idx].dicnt1 = tr_cnt1; 3128 tr_req[tr_idx].dicnt2 = tr1_cnt2; 3129 tr_req[tr_idx].dicnt3 = 1; 3130 tr_req[tr_idx].ddim1 = tr_cnt0; 3131 tr_req[tr_idx].ddim2 = trigger_size; 3132 } else { 3133 tr_req[tr_idx].addr = sg_addr; 3134 tr_req[tr_idx].icnt0 = tr_cnt0; 3135 tr_req[tr_idx].icnt1 = tr_cnt1; 3136 tr_req[tr_idx].icnt2 = tr1_cnt2; 3137 tr_req[tr_idx].icnt3 = 1; 3138 tr_req[tr_idx].dim1 = tr_cnt0; 3139 tr_req[tr_idx].dim2 = trigger_size; 3140 3141 tr_req[tr_idx].daddr = dev_addr; 3142 tr_req[tr_idx].dicnt0 = tr_cnt0; 3143 tr_req[tr_idx].dicnt1 = tr_cnt1; 3144 tr_req[tr_idx].dicnt2 = tr1_cnt2; 3145 tr_req[tr_idx].dicnt3 = 1; 3146 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0; 3147 } 3148 tr_idx++; 3149 } 3150 3151 d->residue += sg_len; 3152 } 3153 3154 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP); 3155 3156 return d; 3157 } 3158 3159 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 3160 enum dma_slave_buswidth dev_width, 3161 u16 elcnt) 3162 { 3163 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 3164 return 0; 3165 3166 /* Bus width translates to the element size (ES) */ 3167 switch (dev_width) { 3168 case DMA_SLAVE_BUSWIDTH_1_BYTE: 3169 d->static_tr.elsize = 0; 3170 break; 3171 case DMA_SLAVE_BUSWIDTH_2_BYTES: 3172 d->static_tr.elsize = 1; 3173 break; 3174 case DMA_SLAVE_BUSWIDTH_3_BYTES: 3175 d->static_tr.elsize = 2; 3176 break; 3177 case DMA_SLAVE_BUSWIDTH_4_BYTES: 3178 d->static_tr.elsize = 3; 3179 break; 3180 case DMA_SLAVE_BUSWIDTH_8_BYTES: 3181 d->static_tr.elsize = 4; 3182 break; 3183 default: /* not reached */ 3184 return -EINVAL; 3185 } 3186 3187 d->static_tr.elcnt = elcnt; 3188 3189 /* 3190 * PDMA must to close the packet when the channel is in packet mode. 3191 * For TR mode when the channel is not cyclic we also need PDMA to close 3192 * the packet otherwise the transfer will stall because PDMA holds on 3193 * the data it has received from the peripheral. 3194 */ 3195 if (uc->config.pkt_mode || !uc->cyclic) { 3196 unsigned int div = dev_width * elcnt; 3197 3198 if (uc->cyclic) 3199 d->static_tr.bstcnt = d->residue / d->sglen / div; 3200 else 3201 d->static_tr.bstcnt = d->residue / div; 3202 3203 if (uc->config.dir == DMA_DEV_TO_MEM && 3204 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 3205 return -EINVAL; 3206 } else { 3207 d->static_tr.bstcnt = 0; 3208 } 3209 3210 return 0; 3211 } 3212 3213 static struct udma_desc * 3214 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 3215 unsigned int sglen, enum dma_transfer_direction dir, 3216 unsigned long tx_flags, void *context) 3217 { 3218 struct scatterlist *sgent; 3219 struct cppi5_host_desc_t *h_desc = NULL; 3220 struct udma_desc *d; 3221 u32 ring_id; 3222 unsigned int i; 3223 u64 asel; 3224 3225 d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT); 3226 if (!d) 3227 return NULL; 3228 3229 d->sglen = sglen; 3230 d->hwdesc_count = sglen; 3231 3232 if (dir == DMA_DEV_TO_MEM) 3233 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 3234 else 3235 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 3236 3237 if (uc->ud->match_data->type == DMA_TYPE_UDMA) 3238 asel = 0; 3239 else 3240 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 3241 3242 for_each_sg(sgl, sgent, sglen, i) { 3243 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 3244 dma_addr_t sg_addr = sg_dma_address(sgent); 3245 struct cppi5_host_desc_t *desc; 3246 size_t sg_len = sg_dma_len(sgent); 3247 3248 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 3249 GFP_NOWAIT, 3250 &hwdesc->cppi5_desc_paddr); 3251 if (!hwdesc->cppi5_desc_vaddr) { 3252 dev_err(uc->ud->dev, 3253 "descriptor%d allocation failed\n", i); 3254 3255 udma_free_hwdesc(uc, d); 3256 kfree(d); 3257 return NULL; 3258 } 3259 3260 d->residue += sg_len; 3261 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 3262 desc = hwdesc->cppi5_desc_vaddr; 3263 3264 if (i == 0) { 3265 cppi5_hdesc_init(desc, 0, 0); 3266 /* Flow and Packed ID */ 3267 cppi5_desc_set_pktids(&desc->hdr, uc->id, 3268 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3269 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 3270 } else { 3271 cppi5_hdesc_reset_hbdesc(desc); 3272 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 3273 } 3274 3275 /* attach the sg buffer to the descriptor */ 3276 sg_addr |= asel; 3277 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 3278 3279 /* Attach link as host buffer descriptor */ 3280 if (h_desc) 3281 cppi5_hdesc_link_hbdesc(h_desc, 3282 hwdesc->cppi5_desc_paddr | asel); 3283 3284 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA || 3285 dir == DMA_MEM_TO_DEV) 3286 h_desc = desc; 3287 } 3288 3289 if (d->residue >= SZ_4M) { 3290 dev_err(uc->ud->dev, 3291 "%s: Transfer size %u is over the supported 4M range\n", 3292 __func__, d->residue); 3293 udma_free_hwdesc(uc, d); 3294 kfree(d); 3295 return NULL; 3296 } 3297 3298 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3299 cppi5_hdesc_set_pktlen(h_desc, d->residue); 3300 3301 return d; 3302 } 3303 3304 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 3305 void *data, size_t len) 3306 { 3307 struct udma_desc *d = to_udma_desc(desc); 3308 struct udma_chan *uc = to_udma_chan(desc->chan); 3309 struct cppi5_host_desc_t *h_desc; 3310 u32 psd_size = len; 3311 u32 flags = 0; 3312 3313 if (!uc->config.pkt_mode || !uc->config.metadata_size) 3314 return -ENOTSUPP; 3315 3316 if (!data || len > uc->config.metadata_size) 3317 return -EINVAL; 3318 3319 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 3320 return -EINVAL; 3321 3322 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3323 if (d->dir == DMA_MEM_TO_DEV) 3324 memcpy(h_desc->epib, data, len); 3325 3326 if (uc->config.needs_epib) 3327 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 3328 3329 d->metadata = data; 3330 d->metadata_size = len; 3331 if (uc->config.needs_epib) 3332 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 3333 3334 cppi5_hdesc_update_flags(h_desc, flags); 3335 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 3336 3337 return 0; 3338 } 3339 3340 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 3341 size_t *payload_len, size_t *max_len) 3342 { 3343 struct udma_desc *d = to_udma_desc(desc); 3344 struct udma_chan *uc = to_udma_chan(desc->chan); 3345 struct cppi5_host_desc_t *h_desc; 3346 3347 if (!uc->config.pkt_mode || !uc->config.metadata_size) 3348 return ERR_PTR(-ENOTSUPP); 3349 3350 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3351 3352 *max_len = uc->config.metadata_size; 3353 3354 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 3355 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 3356 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 3357 3358 return h_desc->epib; 3359 } 3360 3361 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 3362 size_t payload_len) 3363 { 3364 struct udma_desc *d = to_udma_desc(desc); 3365 struct udma_chan *uc = to_udma_chan(desc->chan); 3366 struct cppi5_host_desc_t *h_desc; 3367 u32 psd_size = payload_len; 3368 u32 flags = 0; 3369 3370 if (!uc->config.pkt_mode || !uc->config.metadata_size) 3371 return -ENOTSUPP; 3372 3373 if (payload_len > uc->config.metadata_size) 3374 return -EINVAL; 3375 3376 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 3377 return -EINVAL; 3378 3379 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3380 3381 if (uc->config.needs_epib) { 3382 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 3383 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 3384 } 3385 3386 cppi5_hdesc_update_flags(h_desc, flags); 3387 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 3388 3389 return 0; 3390 } 3391 3392 static struct dma_descriptor_metadata_ops metadata_ops = { 3393 .attach = udma_attach_metadata, 3394 .get_ptr = udma_get_metadata_ptr, 3395 .set_len = udma_set_metadata_len, 3396 }; 3397 3398 static struct dma_async_tx_descriptor * 3399 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 3400 unsigned int sglen, enum dma_transfer_direction dir, 3401 unsigned long tx_flags, void *context) 3402 { 3403 struct udma_chan *uc = to_udma_chan(chan); 3404 enum dma_slave_buswidth dev_width; 3405 struct udma_desc *d; 3406 u32 burst; 3407 3408 if (dir != uc->config.dir && 3409 (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) { 3410 dev_err(chan->device->dev, 3411 "%s: chan%d is for %s, not supporting %s\n", 3412 __func__, uc->id, 3413 dmaengine_get_direction_text(uc->config.dir), 3414 dmaengine_get_direction_text(dir)); 3415 return NULL; 3416 } 3417 3418 if (dir == DMA_DEV_TO_MEM) { 3419 dev_width = uc->cfg.src_addr_width; 3420 burst = uc->cfg.src_maxburst; 3421 } else if (dir == DMA_MEM_TO_DEV) { 3422 dev_width = uc->cfg.dst_addr_width; 3423 burst = uc->cfg.dst_maxburst; 3424 } else { 3425 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 3426 return NULL; 3427 } 3428 3429 if (!burst) 3430 burst = 1; 3431 3432 uc->config.tx_flags = tx_flags; 3433 3434 if (uc->config.pkt_mode) 3435 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 3436 context); 3437 else if (is_slave_direction(uc->config.dir)) 3438 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 3439 context); 3440 else 3441 d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir, 3442 tx_flags, context); 3443 3444 if (!d) 3445 return NULL; 3446 3447 d->dir = dir; 3448 d->desc_idx = 0; 3449 d->tr_idx = 0; 3450 3451 /* static TR for remote PDMA */ 3452 if (udma_configure_statictr(uc, d, dev_width, burst)) { 3453 dev_err(uc->ud->dev, 3454 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 3455 __func__, d->static_tr.bstcnt); 3456 3457 udma_free_hwdesc(uc, d); 3458 kfree(d); 3459 return NULL; 3460 } 3461 3462 if (uc->config.metadata_size) 3463 d->vd.tx.metadata_ops = &metadata_ops; 3464 3465 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 3466 } 3467 3468 static struct udma_desc * 3469 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 3470 size_t buf_len, size_t period_len, 3471 enum dma_transfer_direction dir, unsigned long flags) 3472 { 3473 struct udma_desc *d; 3474 size_t tr_size, period_addr; 3475 struct cppi5_tr_type1_t *tr_req; 3476 unsigned int periods = buf_len / period_len; 3477 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 3478 unsigned int i; 3479 int num_tr; 3480 3481 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, 3482 &tr0_cnt1, &tr1_cnt0); 3483 if (num_tr < 0) { 3484 dev_err(uc->ud->dev, "size %zu is not supported\n", 3485 period_len); 3486 return NULL; 3487 } 3488 3489 /* Now allocate and setup the descriptor. */ 3490 tr_size = sizeof(struct cppi5_tr_type1_t); 3491 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); 3492 if (!d) 3493 return NULL; 3494 3495 tr_req = d->hwdesc[0].tr_req_base; 3496 if (uc->ud->match_data->type == DMA_TYPE_UDMA) 3497 period_addr = buf_addr; 3498 else 3499 period_addr = buf_addr | 3500 ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT); 3501 3502 for (i = 0; i < periods; i++) { 3503 int tr_idx = i * num_tr; 3504 3505 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 3506 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3507 3508 tr_req[tr_idx].addr = period_addr; 3509 tr_req[tr_idx].icnt0 = tr0_cnt0; 3510 tr_req[tr_idx].icnt1 = tr0_cnt1; 3511 tr_req[tr_idx].dim1 = tr0_cnt0; 3512 3513 if (num_tr == 2) { 3514 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 3515 CPPI5_TR_CSF_SUPR_EVT); 3516 tr_idx++; 3517 3518 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 3519 false, false, 3520 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3521 3522 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; 3523 tr_req[tr_idx].icnt0 = tr1_cnt0; 3524 tr_req[tr_idx].icnt1 = 1; 3525 tr_req[tr_idx].dim1 = tr1_cnt0; 3526 } 3527 3528 if (!(flags & DMA_PREP_INTERRUPT)) 3529 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 3530 CPPI5_TR_CSF_SUPR_EVT); 3531 3532 period_addr += period_len; 3533 } 3534 3535 return d; 3536 } 3537 3538 static struct udma_desc * 3539 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 3540 size_t buf_len, size_t period_len, 3541 enum dma_transfer_direction dir, unsigned long flags) 3542 { 3543 struct udma_desc *d; 3544 u32 ring_id; 3545 int i; 3546 int periods = buf_len / period_len; 3547 3548 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 3549 return NULL; 3550 3551 if (period_len >= SZ_4M) 3552 return NULL; 3553 3554 d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT); 3555 if (!d) 3556 return NULL; 3557 3558 d->hwdesc_count = periods; 3559 3560 /* TODO: re-check this... */ 3561 if (dir == DMA_DEV_TO_MEM) 3562 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 3563 else 3564 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 3565 3566 if (uc->ud->match_data->type != DMA_TYPE_UDMA) 3567 buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 3568 3569 for (i = 0; i < periods; i++) { 3570 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 3571 dma_addr_t period_addr = buf_addr + (period_len * i); 3572 struct cppi5_host_desc_t *h_desc; 3573 3574 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 3575 GFP_NOWAIT, 3576 &hwdesc->cppi5_desc_paddr); 3577 if (!hwdesc->cppi5_desc_vaddr) { 3578 dev_err(uc->ud->dev, 3579 "descriptor%d allocation failed\n", i); 3580 3581 udma_free_hwdesc(uc, d); 3582 kfree(d); 3583 return NULL; 3584 } 3585 3586 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 3587 h_desc = hwdesc->cppi5_desc_vaddr; 3588 3589 cppi5_hdesc_init(h_desc, 0, 0); 3590 cppi5_hdesc_set_pktlen(h_desc, period_len); 3591 3592 /* Flow and Packed ID */ 3593 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 3594 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3595 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 3596 3597 /* attach each period to a new descriptor */ 3598 cppi5_hdesc_attach_buf(h_desc, 3599 period_addr, period_len, 3600 period_addr, period_len); 3601 } 3602 3603 return d; 3604 } 3605 3606 static struct dma_async_tx_descriptor * 3607 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 3608 size_t period_len, enum dma_transfer_direction dir, 3609 unsigned long flags) 3610 { 3611 struct udma_chan *uc = to_udma_chan(chan); 3612 enum dma_slave_buswidth dev_width; 3613 struct udma_desc *d; 3614 u32 burst; 3615 3616 if (dir != uc->config.dir) { 3617 dev_err(chan->device->dev, 3618 "%s: chan%d is for %s, not supporting %s\n", 3619 __func__, uc->id, 3620 dmaengine_get_direction_text(uc->config.dir), 3621 dmaengine_get_direction_text(dir)); 3622 return NULL; 3623 } 3624 3625 uc->cyclic = true; 3626 3627 if (dir == DMA_DEV_TO_MEM) { 3628 dev_width = uc->cfg.src_addr_width; 3629 burst = uc->cfg.src_maxburst; 3630 } else if (dir == DMA_MEM_TO_DEV) { 3631 dev_width = uc->cfg.dst_addr_width; 3632 burst = uc->cfg.dst_maxburst; 3633 } else { 3634 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 3635 return NULL; 3636 } 3637 3638 if (!burst) 3639 burst = 1; 3640 3641 if (uc->config.pkt_mode) 3642 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 3643 dir, flags); 3644 else 3645 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 3646 dir, flags); 3647 3648 if (!d) 3649 return NULL; 3650 3651 d->sglen = buf_len / period_len; 3652 3653 d->dir = dir; 3654 d->residue = buf_len; 3655 3656 /* static TR for remote PDMA */ 3657 if (udma_configure_statictr(uc, d, dev_width, burst)) { 3658 dev_err(uc->ud->dev, 3659 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 3660 __func__, d->static_tr.bstcnt); 3661 3662 udma_free_hwdesc(uc, d); 3663 kfree(d); 3664 return NULL; 3665 } 3666 3667 if (uc->config.metadata_size) 3668 d->vd.tx.metadata_ops = &metadata_ops; 3669 3670 return vchan_tx_prep(&uc->vc, &d->vd, flags); 3671 } 3672 3673 static struct dma_async_tx_descriptor * 3674 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 3675 size_t len, unsigned long tx_flags) 3676 { 3677 struct udma_chan *uc = to_udma_chan(chan); 3678 struct udma_desc *d; 3679 struct cppi5_tr_type15_t *tr_req; 3680 int num_tr; 3681 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 3682 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 3683 u32 csf = CPPI5_TR_CSF_SUPR_EVT; 3684 3685 if (uc->config.dir != DMA_MEM_TO_MEM) { 3686 dev_err(chan->device->dev, 3687 "%s: chan%d is for %s, not supporting %s\n", 3688 __func__, uc->id, 3689 dmaengine_get_direction_text(uc->config.dir), 3690 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 3691 return NULL; 3692 } 3693 3694 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, 3695 &tr0_cnt1, &tr1_cnt0); 3696 if (num_tr < 0) { 3697 dev_err(uc->ud->dev, "size %zu is not supported\n", 3698 len); 3699 return NULL; 3700 } 3701 3702 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 3703 if (!d) 3704 return NULL; 3705 3706 d->dir = DMA_MEM_TO_MEM; 3707 d->desc_idx = 0; 3708 d->tr_idx = 0; 3709 d->residue = len; 3710 3711 if (uc->ud->match_data->type != DMA_TYPE_UDMA) { 3712 src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; 3713 dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; 3714 } else { 3715 csf |= CPPI5_TR_CSF_EOL_ICNT0; 3716 } 3717 3718 tr_req = d->hwdesc[0].tr_req_base; 3719 3720 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 3721 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3722 cppi5_tr_csf_set(&tr_req[0].flags, csf); 3723 3724 tr_req[0].addr = src; 3725 tr_req[0].icnt0 = tr0_cnt0; 3726 tr_req[0].icnt1 = tr0_cnt1; 3727 tr_req[0].icnt2 = 1; 3728 tr_req[0].icnt3 = 1; 3729 tr_req[0].dim1 = tr0_cnt0; 3730 3731 tr_req[0].daddr = dest; 3732 tr_req[0].dicnt0 = tr0_cnt0; 3733 tr_req[0].dicnt1 = tr0_cnt1; 3734 tr_req[0].dicnt2 = 1; 3735 tr_req[0].dicnt3 = 1; 3736 tr_req[0].ddim1 = tr0_cnt0; 3737 3738 if (num_tr == 2) { 3739 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 3740 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3741 cppi5_tr_csf_set(&tr_req[1].flags, csf); 3742 3743 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 3744 tr_req[1].icnt0 = tr1_cnt0; 3745 tr_req[1].icnt1 = 1; 3746 tr_req[1].icnt2 = 1; 3747 tr_req[1].icnt3 = 1; 3748 3749 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 3750 tr_req[1].dicnt0 = tr1_cnt0; 3751 tr_req[1].dicnt1 = 1; 3752 tr_req[1].dicnt2 = 1; 3753 tr_req[1].dicnt3 = 1; 3754 } 3755 3756 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP); 3757 3758 if (uc->config.metadata_size) 3759 d->vd.tx.metadata_ops = &metadata_ops; 3760 3761 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 3762 } 3763 3764 static void udma_issue_pending(struct dma_chan *chan) 3765 { 3766 struct udma_chan *uc = to_udma_chan(chan); 3767 unsigned long flags; 3768 3769 spin_lock_irqsave(&uc->vc.lock, flags); 3770 3771 /* If we have something pending and no active descriptor, then */ 3772 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 3773 /* 3774 * start a descriptor if the channel is NOT [marked as 3775 * terminating _and_ it is still running (teardown has not 3776 * completed yet)]. 3777 */ 3778 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 3779 udma_is_chan_running(uc))) 3780 udma_start(uc); 3781 } 3782 3783 spin_unlock_irqrestore(&uc->vc.lock, flags); 3784 } 3785 3786 static enum dma_status udma_tx_status(struct dma_chan *chan, 3787 dma_cookie_t cookie, 3788 struct dma_tx_state *txstate) 3789 { 3790 struct udma_chan *uc = to_udma_chan(chan); 3791 enum dma_status ret; 3792 unsigned long flags; 3793 3794 spin_lock_irqsave(&uc->vc.lock, flags); 3795 3796 ret = dma_cookie_status(chan, cookie, txstate); 3797 3798 if (!udma_is_chan_running(uc)) 3799 ret = DMA_COMPLETE; 3800 3801 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 3802 ret = DMA_PAUSED; 3803 3804 if (ret == DMA_COMPLETE || !txstate) 3805 goto out; 3806 3807 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 3808 u32 peer_bcnt = 0; 3809 u32 bcnt = 0; 3810 u32 residue = uc->desc->residue; 3811 u32 delay = 0; 3812 3813 if (uc->desc->dir == DMA_MEM_TO_DEV) { 3814 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 3815 3816 if (uc->config.ep_type != PSIL_EP_NATIVE) { 3817 peer_bcnt = udma_tchanrt_read(uc, 3818 UDMA_CHAN_RT_PEER_BCNT_REG); 3819 3820 if (bcnt > peer_bcnt) 3821 delay = bcnt - peer_bcnt; 3822 } 3823 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 3824 bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 3825 3826 if (uc->config.ep_type != PSIL_EP_NATIVE) { 3827 peer_bcnt = udma_rchanrt_read(uc, 3828 UDMA_CHAN_RT_PEER_BCNT_REG); 3829 3830 if (peer_bcnt > bcnt) 3831 delay = peer_bcnt - bcnt; 3832 } 3833 } else { 3834 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 3835 } 3836 3837 if (bcnt && !(bcnt % uc->desc->residue)) 3838 residue = 0; 3839 else 3840 residue -= bcnt % uc->desc->residue; 3841 3842 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 3843 ret = DMA_COMPLETE; 3844 delay = 0; 3845 } 3846 3847 dma_set_residue(txstate, residue); 3848 dma_set_in_flight_bytes(txstate, delay); 3849 3850 } else { 3851 ret = DMA_COMPLETE; 3852 } 3853 3854 out: 3855 spin_unlock_irqrestore(&uc->vc.lock, flags); 3856 return ret; 3857 } 3858 3859 static int udma_pause(struct dma_chan *chan) 3860 { 3861 struct udma_chan *uc = to_udma_chan(chan); 3862 3863 /* pause the channel */ 3864 switch (uc->config.dir) { 3865 case DMA_DEV_TO_MEM: 3866 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3867 UDMA_PEER_RT_EN_PAUSE, 3868 UDMA_PEER_RT_EN_PAUSE); 3869 break; 3870 case DMA_MEM_TO_DEV: 3871 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3872 UDMA_PEER_RT_EN_PAUSE, 3873 UDMA_PEER_RT_EN_PAUSE); 3874 break; 3875 case DMA_MEM_TO_MEM: 3876 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, 3877 UDMA_CHAN_RT_CTL_PAUSE, 3878 UDMA_CHAN_RT_CTL_PAUSE); 3879 break; 3880 default: 3881 return -EINVAL; 3882 } 3883 3884 return 0; 3885 } 3886 3887 static int udma_resume(struct dma_chan *chan) 3888 { 3889 struct udma_chan *uc = to_udma_chan(chan); 3890 3891 /* resume the channel */ 3892 switch (uc->config.dir) { 3893 case DMA_DEV_TO_MEM: 3894 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3895 UDMA_PEER_RT_EN_PAUSE, 0); 3896 3897 break; 3898 case DMA_MEM_TO_DEV: 3899 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3900 UDMA_PEER_RT_EN_PAUSE, 0); 3901 break; 3902 case DMA_MEM_TO_MEM: 3903 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, 3904 UDMA_CHAN_RT_CTL_PAUSE, 0); 3905 break; 3906 default: 3907 return -EINVAL; 3908 } 3909 3910 return 0; 3911 } 3912 3913 static int udma_terminate_all(struct dma_chan *chan) 3914 { 3915 struct udma_chan *uc = to_udma_chan(chan); 3916 unsigned long flags; 3917 LIST_HEAD(head); 3918 3919 spin_lock_irqsave(&uc->vc.lock, flags); 3920 3921 if (udma_is_chan_running(uc)) 3922 udma_stop(uc); 3923 3924 if (uc->desc) { 3925 uc->terminated_desc = uc->desc; 3926 uc->desc = NULL; 3927 uc->terminated_desc->terminated = true; 3928 cancel_delayed_work(&uc->tx_drain.work); 3929 } 3930 3931 uc->paused = false; 3932 3933 vchan_get_all_descriptors(&uc->vc, &head); 3934 spin_unlock_irqrestore(&uc->vc.lock, flags); 3935 vchan_dma_desc_free_list(&uc->vc, &head); 3936 3937 return 0; 3938 } 3939 3940 static void udma_synchronize(struct dma_chan *chan) 3941 { 3942 struct udma_chan *uc = to_udma_chan(chan); 3943 unsigned long timeout = msecs_to_jiffies(1000); 3944 3945 vchan_synchronize(&uc->vc); 3946 3947 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 3948 timeout = wait_for_completion_timeout(&uc->teardown_completed, 3949 timeout); 3950 if (!timeout) { 3951 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 3952 uc->id); 3953 udma_dump_chan_stdata(uc); 3954 udma_reset_chan(uc, true); 3955 } 3956 } 3957 3958 udma_reset_chan(uc, false); 3959 if (udma_is_chan_running(uc)) 3960 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 3961 3962 cancel_delayed_work_sync(&uc->tx_drain.work); 3963 udma_reset_rings(uc); 3964 } 3965 3966 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 3967 struct virt_dma_desc *vd, 3968 struct dmaengine_result *result) 3969 { 3970 struct udma_chan *uc = to_udma_chan(&vc->chan); 3971 struct udma_desc *d; 3972 3973 if (!vd) 3974 return; 3975 3976 d = to_udma_desc(&vd->tx); 3977 3978 if (d->metadata_size) 3979 udma_fetch_epib(uc, d); 3980 3981 /* Provide residue information for the client */ 3982 if (result) { 3983 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 3984 3985 if (cppi5_desc_get_type(desc_vaddr) == 3986 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 3987 result->residue = d->residue - 3988 cppi5_hdesc_get_pktlen(desc_vaddr); 3989 if (result->residue) 3990 result->result = DMA_TRANS_ABORTED; 3991 else 3992 result->result = DMA_TRANS_NOERROR; 3993 } else { 3994 result->residue = 0; 3995 result->result = DMA_TRANS_NOERROR; 3996 } 3997 } 3998 } 3999 4000 /* 4001 * This tasklet handles the completion of a DMA descriptor by 4002 * calling its callback and freeing it. 4003 */ 4004 static void udma_vchan_complete(struct tasklet_struct *t) 4005 { 4006 struct virt_dma_chan *vc = from_tasklet(vc, t, task); 4007 struct virt_dma_desc *vd, *_vd; 4008 struct dmaengine_desc_callback cb; 4009 LIST_HEAD(head); 4010 4011 spin_lock_irq(&vc->lock); 4012 list_splice_tail_init(&vc->desc_completed, &head); 4013 vd = vc->cyclic; 4014 if (vd) { 4015 vc->cyclic = NULL; 4016 dmaengine_desc_get_callback(&vd->tx, &cb); 4017 } else { 4018 memset(&cb, 0, sizeof(cb)); 4019 } 4020 spin_unlock_irq(&vc->lock); 4021 4022 udma_desc_pre_callback(vc, vd, NULL); 4023 dmaengine_desc_callback_invoke(&cb, NULL); 4024 4025 list_for_each_entry_safe(vd, _vd, &head, node) { 4026 struct dmaengine_result result; 4027 4028 dmaengine_desc_get_callback(&vd->tx, &cb); 4029 4030 list_del(&vd->node); 4031 4032 udma_desc_pre_callback(vc, vd, &result); 4033 dmaengine_desc_callback_invoke(&cb, &result); 4034 4035 vchan_vdesc_fini(vd); 4036 } 4037 } 4038 4039 static void udma_free_chan_resources(struct dma_chan *chan) 4040 { 4041 struct udma_chan *uc = to_udma_chan(chan); 4042 struct udma_dev *ud = to_udma_dev(chan->device); 4043 4044 udma_terminate_all(chan); 4045 if (uc->terminated_desc) { 4046 udma_reset_chan(uc, false); 4047 udma_reset_rings(uc); 4048 } 4049 4050 cancel_delayed_work_sync(&uc->tx_drain.work); 4051 4052 if (uc->irq_num_ring > 0) { 4053 free_irq(uc->irq_num_ring, uc); 4054 4055 uc->irq_num_ring = 0; 4056 } 4057 if (uc->irq_num_udma > 0) { 4058 free_irq(uc->irq_num_udma, uc); 4059 4060 uc->irq_num_udma = 0; 4061 } 4062 4063 /* Release PSI-L pairing */ 4064 if (uc->psil_paired) { 4065 navss_psil_unpair(ud, uc->config.src_thread, 4066 uc->config.dst_thread); 4067 uc->psil_paired = false; 4068 } 4069 4070 vchan_free_chan_resources(&uc->vc); 4071 tasklet_kill(&uc->vc.task); 4072 4073 bcdma_free_bchan_resources(uc); 4074 udma_free_tx_resources(uc); 4075 udma_free_rx_resources(uc); 4076 udma_reset_uchan(uc); 4077 4078 if (uc->use_dma_pool) { 4079 dma_pool_destroy(uc->hdesc_pool); 4080 uc->use_dma_pool = false; 4081 } 4082 } 4083 4084 static struct platform_driver udma_driver; 4085 static struct platform_driver bcdma_driver; 4086 static struct platform_driver pktdma_driver; 4087 4088 struct udma_filter_param { 4089 int remote_thread_id; 4090 u32 atype; 4091 u32 asel; 4092 u32 tr_trigger_type; 4093 }; 4094 4095 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 4096 { 4097 struct udma_chan_config *ucc; 4098 struct psil_endpoint_config *ep_config; 4099 struct udma_filter_param *filter_param; 4100 struct udma_chan *uc; 4101 struct udma_dev *ud; 4102 4103 if (chan->device->dev->driver != &udma_driver.driver && 4104 chan->device->dev->driver != &bcdma_driver.driver && 4105 chan->device->dev->driver != &pktdma_driver.driver) 4106 return false; 4107 4108 uc = to_udma_chan(chan); 4109 ucc = &uc->config; 4110 ud = uc->ud; 4111 filter_param = param; 4112 4113 if (filter_param->atype > 2) { 4114 dev_err(ud->dev, "Invalid channel atype: %u\n", 4115 filter_param->atype); 4116 return false; 4117 } 4118 4119 if (filter_param->asel > 15) { 4120 dev_err(ud->dev, "Invalid channel asel: %u\n", 4121 filter_param->asel); 4122 return false; 4123 } 4124 4125 ucc->remote_thread_id = filter_param->remote_thread_id; 4126 ucc->atype = filter_param->atype; 4127 ucc->asel = filter_param->asel; 4128 ucc->tr_trigger_type = filter_param->tr_trigger_type; 4129 4130 if (ucc->tr_trigger_type) { 4131 ucc->dir = DMA_MEM_TO_MEM; 4132 goto triggered_bchan; 4133 } else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) { 4134 ucc->dir = DMA_MEM_TO_DEV; 4135 } else { 4136 ucc->dir = DMA_DEV_TO_MEM; 4137 } 4138 4139 ep_config = psil_get_ep_config(ucc->remote_thread_id); 4140 if (IS_ERR(ep_config)) { 4141 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 4142 ucc->remote_thread_id); 4143 ucc->dir = DMA_MEM_TO_MEM; 4144 ucc->remote_thread_id = -1; 4145 ucc->atype = 0; 4146 ucc->asel = 0; 4147 return false; 4148 } 4149 4150 if (ud->match_data->type == DMA_TYPE_BCDMA && 4151 ep_config->pkt_mode) { 4152 dev_err(ud->dev, 4153 "Only TR mode is supported (psi-l thread 0x%04x)\n", 4154 ucc->remote_thread_id); 4155 ucc->dir = DMA_MEM_TO_MEM; 4156 ucc->remote_thread_id = -1; 4157 ucc->atype = 0; 4158 ucc->asel = 0; 4159 return false; 4160 } 4161 4162 ucc->pkt_mode = ep_config->pkt_mode; 4163 ucc->channel_tpl = ep_config->channel_tpl; 4164 ucc->notdpkt = ep_config->notdpkt; 4165 ucc->ep_type = ep_config->ep_type; 4166 4167 if (ud->match_data->type == DMA_TYPE_PKTDMA && 4168 ep_config->mapped_channel_id >= 0) { 4169 ucc->mapped_channel_id = ep_config->mapped_channel_id; 4170 ucc->default_flow_id = ep_config->default_flow_id; 4171 } else { 4172 ucc->mapped_channel_id = -1; 4173 ucc->default_flow_id = -1; 4174 } 4175 4176 if (ucc->ep_type != PSIL_EP_NATIVE) { 4177 const struct udma_match_data *match_data = ud->match_data; 4178 4179 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 4180 ucc->enable_acc32 = ep_config->pdma_acc32; 4181 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 4182 ucc->enable_burst = ep_config->pdma_burst; 4183 } 4184 4185 ucc->needs_epib = ep_config->needs_epib; 4186 ucc->psd_size = ep_config->psd_size; 4187 ucc->metadata_size = 4188 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 4189 ucc->psd_size; 4190 4191 if (ucc->pkt_mode) 4192 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 4193 ucc->metadata_size, ud->desc_align); 4194 4195 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 4196 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 4197 4198 return true; 4199 4200 triggered_bchan: 4201 dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id, 4202 ucc->tr_trigger_type); 4203 4204 return true; 4205 4206 } 4207 4208 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 4209 struct of_dma *ofdma) 4210 { 4211 struct udma_dev *ud = ofdma->of_dma_data; 4212 dma_cap_mask_t mask = ud->ddev.cap_mask; 4213 struct udma_filter_param filter_param; 4214 struct dma_chan *chan; 4215 4216 if (ud->match_data->type == DMA_TYPE_BCDMA) { 4217 if (dma_spec->args_count != 3) 4218 return NULL; 4219 4220 filter_param.tr_trigger_type = dma_spec->args[0]; 4221 filter_param.remote_thread_id = dma_spec->args[1]; 4222 filter_param.asel = dma_spec->args[2]; 4223 filter_param.atype = 0; 4224 } else { 4225 if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 4226 return NULL; 4227 4228 filter_param.remote_thread_id = dma_spec->args[0]; 4229 filter_param.tr_trigger_type = 0; 4230 if (dma_spec->args_count == 2) { 4231 if (ud->match_data->type == DMA_TYPE_UDMA) { 4232 filter_param.atype = dma_spec->args[1]; 4233 filter_param.asel = 0; 4234 } else { 4235 filter_param.atype = 0; 4236 filter_param.asel = dma_spec->args[1]; 4237 } 4238 } else { 4239 filter_param.atype = 0; 4240 filter_param.asel = 0; 4241 } 4242 } 4243 4244 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 4245 ofdma->of_node); 4246 if (!chan) { 4247 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 4248 return ERR_PTR(-EINVAL); 4249 } 4250 4251 return chan; 4252 } 4253 4254 static struct udma_match_data am654_main_data = { 4255 .type = DMA_TYPE_UDMA, 4256 .psil_base = 0x1000, 4257 .enable_memcpy_support = true, 4258 .statictr_z_mask = GENMASK(11, 0), 4259 .burst_size = { 4260 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4261 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ 4262 0, /* No UH Channels */ 4263 }, 4264 }; 4265 4266 static struct udma_match_data am654_mcu_data = { 4267 .type = DMA_TYPE_UDMA, 4268 .psil_base = 0x6000, 4269 .enable_memcpy_support = false, 4270 .statictr_z_mask = GENMASK(11, 0), 4271 .burst_size = { 4272 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4273 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ 4274 0, /* No UH Channels */ 4275 }, 4276 }; 4277 4278 static struct udma_match_data j721e_main_data = { 4279 .type = DMA_TYPE_UDMA, 4280 .psil_base = 0x1000, 4281 .enable_memcpy_support = true, 4282 .flags = UDMA_FLAGS_J7_CLASS, 4283 .statictr_z_mask = GENMASK(23, 0), 4284 .burst_size = { 4285 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4286 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */ 4287 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */ 4288 }, 4289 }; 4290 4291 static struct udma_match_data j721e_mcu_data = { 4292 .type = DMA_TYPE_UDMA, 4293 .psil_base = 0x6000, 4294 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 4295 .flags = UDMA_FLAGS_J7_CLASS, 4296 .statictr_z_mask = GENMASK(23, 0), 4297 .burst_size = { 4298 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4299 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */ 4300 0, /* No UH Channels */ 4301 }, 4302 }; 4303 4304 static struct udma_soc_data am62a_dmss_csi_soc_data = { 4305 .oes = { 4306 .bcdma_rchan_data = 0xe00, 4307 .bcdma_rchan_ring = 0x1000, 4308 }, 4309 }; 4310 4311 static struct udma_match_data am62a_bcdma_csirx_data = { 4312 .type = DMA_TYPE_BCDMA, 4313 .psil_base = 0x3100, 4314 .enable_memcpy_support = false, 4315 .burst_size = { 4316 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4317 0, /* No H Channels */ 4318 0, /* No UH Channels */ 4319 }, 4320 .soc_data = &am62a_dmss_csi_soc_data, 4321 }; 4322 4323 static struct udma_match_data am64_bcdma_data = { 4324 .type = DMA_TYPE_BCDMA, 4325 .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ 4326 .enable_memcpy_support = true, /* Supported via bchan */ 4327 .flags = UDMA_FLAGS_J7_CLASS, 4328 .statictr_z_mask = GENMASK(23, 0), 4329 .burst_size = { 4330 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4331 0, /* No H Channels */ 4332 0, /* No UH Channels */ 4333 }, 4334 }; 4335 4336 static struct udma_match_data am64_pktdma_data = { 4337 .type = DMA_TYPE_PKTDMA, 4338 .psil_base = 0x1000, 4339 .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */ 4340 .flags = UDMA_FLAGS_J7_CLASS, 4341 .statictr_z_mask = GENMASK(23, 0), 4342 .burst_size = { 4343 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4344 0, /* No H Channels */ 4345 0, /* No UH Channels */ 4346 }, 4347 }; 4348 4349 static const struct of_device_id udma_of_match[] = { 4350 { 4351 .compatible = "ti,am654-navss-main-udmap", 4352 .data = &am654_main_data, 4353 }, 4354 { 4355 .compatible = "ti,am654-navss-mcu-udmap", 4356 .data = &am654_mcu_data, 4357 }, { 4358 .compatible = "ti,j721e-navss-main-udmap", 4359 .data = &j721e_main_data, 4360 }, { 4361 .compatible = "ti,j721e-navss-mcu-udmap", 4362 .data = &j721e_mcu_data, 4363 }, 4364 { 4365 .compatible = "ti,am64-dmss-bcdma", 4366 .data = &am64_bcdma_data, 4367 }, 4368 { 4369 .compatible = "ti,am64-dmss-pktdma", 4370 .data = &am64_pktdma_data, 4371 }, 4372 { 4373 .compatible = "ti,am62a-dmss-bcdma-csirx", 4374 .data = &am62a_bcdma_csirx_data, 4375 }, 4376 { /* Sentinel */ }, 4377 }; 4378 4379 static struct udma_soc_data am654_soc_data = { 4380 .oes = { 4381 .udma_rchan = 0x200, 4382 }, 4383 }; 4384 4385 static struct udma_soc_data j721e_soc_data = { 4386 .oes = { 4387 .udma_rchan = 0x400, 4388 }, 4389 }; 4390 4391 static struct udma_soc_data j7200_soc_data = { 4392 .oes = { 4393 .udma_rchan = 0x80, 4394 }, 4395 }; 4396 4397 static struct udma_soc_data am64_soc_data = { 4398 .oes = { 4399 .bcdma_bchan_data = 0x2200, 4400 .bcdma_bchan_ring = 0x2400, 4401 .bcdma_tchan_data = 0x2800, 4402 .bcdma_tchan_ring = 0x2a00, 4403 .bcdma_rchan_data = 0x2e00, 4404 .bcdma_rchan_ring = 0x3000, 4405 .pktdma_tchan_flow = 0x1200, 4406 .pktdma_rchan_flow = 0x1600, 4407 }, 4408 .bcdma_trigger_event_offset = 0xc400, 4409 }; 4410 4411 static const struct soc_device_attribute k3_soc_devices[] = { 4412 { .family = "AM65X", .data = &am654_soc_data }, 4413 { .family = "J721E", .data = &j721e_soc_data }, 4414 { .family = "J7200", .data = &j7200_soc_data }, 4415 { .family = "AM64X", .data = &am64_soc_data }, 4416 { .family = "J721S2", .data = &j721e_soc_data}, 4417 { .family = "AM62X", .data = &am64_soc_data }, 4418 { .family = "AM62AX", .data = &am64_soc_data }, 4419 { .family = "J784S4", .data = &j721e_soc_data }, 4420 { /* sentinel */ } 4421 }; 4422 4423 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 4424 { 4425 u32 cap2, cap3, cap4; 4426 int i; 4427 4428 ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]); 4429 if (IS_ERR(ud->mmrs[MMR_GCFG])) 4430 return PTR_ERR(ud->mmrs[MMR_GCFG]); 4431 4432 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); 4433 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4434 4435 switch (ud->match_data->type) { 4436 case DMA_TYPE_UDMA: 4437 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); 4438 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); 4439 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); 4440 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); 4441 break; 4442 case DMA_TYPE_BCDMA: 4443 ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); 4444 ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); 4445 ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); 4446 ud->rflow_cnt = ud->rchan_cnt; 4447 break; 4448 case DMA_TYPE_PKTDMA: 4449 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30); 4450 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); 4451 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); 4452 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); 4453 ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4); 4454 break; 4455 default: 4456 return -EINVAL; 4457 } 4458 4459 for (i = 1; i < MMR_LAST; i++) { 4460 if (i == MMR_BCHANRT && ud->bchan_cnt == 0) 4461 continue; 4462 if (i == MMR_TCHANRT && ud->tchan_cnt == 0) 4463 continue; 4464 if (i == MMR_RCHANRT && ud->rchan_cnt == 0) 4465 continue; 4466 4467 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]); 4468 if (IS_ERR(ud->mmrs[i])) 4469 return PTR_ERR(ud->mmrs[i]); 4470 } 4471 4472 return 0; 4473 } 4474 4475 static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map, 4476 struct ti_sci_resource_desc *rm_desc, 4477 char *name) 4478 { 4479 bitmap_clear(map, rm_desc->start, rm_desc->num); 4480 bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec); 4481 dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name, 4482 rm_desc->start, rm_desc->num, rm_desc->start_sec, 4483 rm_desc->num_sec); 4484 } 4485 4486 static const char * const range_names[] = { 4487 [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan", 4488 [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan", 4489 [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan", 4490 [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow", 4491 [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow", 4492 }; 4493 4494 static int udma_setup_resources(struct udma_dev *ud) 4495 { 4496 int ret, i, j; 4497 struct device *dev = ud->dev; 4498 struct ti_sci_resource *rm_res, irq_res; 4499 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 4500 u32 cap3; 4501 4502 /* Set up the throughput level start indexes */ 4503 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4504 if (of_device_is_compatible(dev->of_node, 4505 "ti,am654-navss-main-udmap")) { 4506 ud->tchan_tpl.levels = 2; 4507 ud->tchan_tpl.start_idx[0] = 8; 4508 } else if (of_device_is_compatible(dev->of_node, 4509 "ti,am654-navss-mcu-udmap")) { 4510 ud->tchan_tpl.levels = 2; 4511 ud->tchan_tpl.start_idx[0] = 2; 4512 } else if (UDMA_CAP3_UCHAN_CNT(cap3)) { 4513 ud->tchan_tpl.levels = 3; 4514 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); 4515 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4516 } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { 4517 ud->tchan_tpl.levels = 2; 4518 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4519 } else { 4520 ud->tchan_tpl.levels = 1; 4521 } 4522 4523 ud->rchan_tpl.levels = ud->tchan_tpl.levels; 4524 ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0]; 4525 ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1]; 4526 4527 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 4528 sizeof(unsigned long), GFP_KERNEL); 4529 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 4530 GFP_KERNEL); 4531 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 4532 sizeof(unsigned long), GFP_KERNEL); 4533 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 4534 GFP_KERNEL); 4535 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 4536 sizeof(unsigned long), 4537 GFP_KERNEL); 4538 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 4539 BITS_TO_LONGS(ud->rflow_cnt), 4540 sizeof(unsigned long), 4541 GFP_KERNEL); 4542 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 4543 sizeof(unsigned long), 4544 GFP_KERNEL); 4545 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 4546 GFP_KERNEL); 4547 4548 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 4549 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 4550 !ud->rflows || !ud->rflow_in_use) 4551 return -ENOMEM; 4552 4553 /* 4554 * RX flows with the same Ids as RX channels are reserved to be used 4555 * as default flows if remote HW can't generate flow_ids. Those 4556 * RX flows can be requested only explicitly by id. 4557 */ 4558 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 4559 4560 /* by default no GP rflows are assigned to Linux */ 4561 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 4562 4563 /* Get resource ranges from tisci */ 4564 for (i = 0; i < RM_RANGE_LAST; i++) { 4565 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW) 4566 continue; 4567 4568 tisci_rm->rm_ranges[i] = 4569 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 4570 tisci_rm->tisci_dev_id, 4571 (char *)range_names[i]); 4572 } 4573 4574 /* tchan ranges */ 4575 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4576 if (IS_ERR(rm_res)) { 4577 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 4578 irq_res.sets = 1; 4579 } else { 4580 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 4581 for (i = 0; i < rm_res->sets; i++) 4582 udma_mark_resource_ranges(ud, ud->tchan_map, 4583 &rm_res->desc[i], "tchan"); 4584 irq_res.sets = rm_res->sets; 4585 } 4586 4587 /* rchan and matching default flow ranges */ 4588 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4589 if (IS_ERR(rm_res)) { 4590 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 4591 irq_res.sets++; 4592 } else { 4593 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 4594 for (i = 0; i < rm_res->sets; i++) 4595 udma_mark_resource_ranges(ud, ud->rchan_map, 4596 &rm_res->desc[i], "rchan"); 4597 irq_res.sets += rm_res->sets; 4598 } 4599 4600 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 4601 if (!irq_res.desc) 4602 return -ENOMEM; 4603 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4604 if (IS_ERR(rm_res)) { 4605 irq_res.desc[0].start = 0; 4606 irq_res.desc[0].num = ud->tchan_cnt; 4607 i = 1; 4608 } else { 4609 for (i = 0; i < rm_res->sets; i++) { 4610 irq_res.desc[i].start = rm_res->desc[i].start; 4611 irq_res.desc[i].num = rm_res->desc[i].num; 4612 irq_res.desc[i].start_sec = rm_res->desc[i].start_sec; 4613 irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; 4614 } 4615 } 4616 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4617 if (IS_ERR(rm_res)) { 4618 irq_res.desc[i].start = 0; 4619 irq_res.desc[i].num = ud->rchan_cnt; 4620 } else { 4621 for (j = 0; j < rm_res->sets; j++, i++) { 4622 if (rm_res->desc[j].num) { 4623 irq_res.desc[i].start = rm_res->desc[j].start + 4624 ud->soc_data->oes.udma_rchan; 4625 irq_res.desc[i].num = rm_res->desc[j].num; 4626 } 4627 if (rm_res->desc[j].num_sec) { 4628 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + 4629 ud->soc_data->oes.udma_rchan; 4630 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; 4631 } 4632 } 4633 } 4634 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 4635 kfree(irq_res.desc); 4636 if (ret) { 4637 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 4638 return ret; 4639 } 4640 4641 /* GP rflow ranges */ 4642 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 4643 if (IS_ERR(rm_res)) { 4644 /* all gp flows are assigned exclusively to Linux */ 4645 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 4646 ud->rflow_cnt - ud->rchan_cnt); 4647 } else { 4648 for (i = 0; i < rm_res->sets; i++) 4649 udma_mark_resource_ranges(ud, ud->rflow_gp_map, 4650 &rm_res->desc[i], "gp-rflow"); 4651 } 4652 4653 return 0; 4654 } 4655 4656 static int bcdma_setup_resources(struct udma_dev *ud) 4657 { 4658 int ret, i, j; 4659 struct device *dev = ud->dev; 4660 struct ti_sci_resource *rm_res, irq_res; 4661 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 4662 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 4663 u32 cap; 4664 4665 /* Set up the throughput level start indexes */ 4666 cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4667 if (BCDMA_CAP3_UBCHAN_CNT(cap)) { 4668 ud->bchan_tpl.levels = 3; 4669 ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap); 4670 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap); 4671 } else if (BCDMA_CAP3_HBCHAN_CNT(cap)) { 4672 ud->bchan_tpl.levels = 2; 4673 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap); 4674 } else { 4675 ud->bchan_tpl.levels = 1; 4676 } 4677 4678 cap = udma_read(ud->mmrs[MMR_GCFG], 0x30); 4679 if (BCDMA_CAP4_URCHAN_CNT(cap)) { 4680 ud->rchan_tpl.levels = 3; 4681 ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap); 4682 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap); 4683 } else if (BCDMA_CAP4_HRCHAN_CNT(cap)) { 4684 ud->rchan_tpl.levels = 2; 4685 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap); 4686 } else { 4687 ud->rchan_tpl.levels = 1; 4688 } 4689 4690 if (BCDMA_CAP4_UTCHAN_CNT(cap)) { 4691 ud->tchan_tpl.levels = 3; 4692 ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap); 4693 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap); 4694 } else if (BCDMA_CAP4_HTCHAN_CNT(cap)) { 4695 ud->tchan_tpl.levels = 2; 4696 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap); 4697 } else { 4698 ud->tchan_tpl.levels = 1; 4699 } 4700 4701 ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt), 4702 sizeof(unsigned long), GFP_KERNEL); 4703 ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans), 4704 GFP_KERNEL); 4705 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 4706 sizeof(unsigned long), GFP_KERNEL); 4707 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 4708 GFP_KERNEL); 4709 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 4710 sizeof(unsigned long), GFP_KERNEL); 4711 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 4712 GFP_KERNEL); 4713 /* BCDMA do not really have flows, but the driver expect it */ 4714 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt), 4715 sizeof(unsigned long), 4716 GFP_KERNEL); 4717 ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows), 4718 GFP_KERNEL); 4719 4720 if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map || 4721 !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans || 4722 !ud->rflows) 4723 return -ENOMEM; 4724 4725 /* Get resource ranges from tisci */ 4726 for (i = 0; i < RM_RANGE_LAST; i++) { 4727 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW) 4728 continue; 4729 if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0) 4730 continue; 4731 if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0) 4732 continue; 4733 if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0) 4734 continue; 4735 4736 tisci_rm->rm_ranges[i] = 4737 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 4738 tisci_rm->tisci_dev_id, 4739 (char *)range_names[i]); 4740 } 4741 4742 irq_res.sets = 0; 4743 4744 /* bchan ranges */ 4745 if (ud->bchan_cnt) { 4746 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; 4747 if (IS_ERR(rm_res)) { 4748 bitmap_zero(ud->bchan_map, ud->bchan_cnt); 4749 irq_res.sets++; 4750 } else { 4751 bitmap_fill(ud->bchan_map, ud->bchan_cnt); 4752 for (i = 0; i < rm_res->sets; i++) 4753 udma_mark_resource_ranges(ud, ud->bchan_map, 4754 &rm_res->desc[i], 4755 "bchan"); 4756 irq_res.sets += rm_res->sets; 4757 } 4758 } 4759 4760 /* tchan ranges */ 4761 if (ud->tchan_cnt) { 4762 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4763 if (IS_ERR(rm_res)) { 4764 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 4765 irq_res.sets += 2; 4766 } else { 4767 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 4768 for (i = 0; i < rm_res->sets; i++) 4769 udma_mark_resource_ranges(ud, ud->tchan_map, 4770 &rm_res->desc[i], 4771 "tchan"); 4772 irq_res.sets += rm_res->sets * 2; 4773 } 4774 } 4775 4776 /* rchan ranges */ 4777 if (ud->rchan_cnt) { 4778 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4779 if (IS_ERR(rm_res)) { 4780 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 4781 irq_res.sets += 2; 4782 } else { 4783 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 4784 for (i = 0; i < rm_res->sets; i++) 4785 udma_mark_resource_ranges(ud, ud->rchan_map, 4786 &rm_res->desc[i], 4787 "rchan"); 4788 irq_res.sets += rm_res->sets * 2; 4789 } 4790 } 4791 4792 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 4793 if (!irq_res.desc) 4794 return -ENOMEM; 4795 if (ud->bchan_cnt) { 4796 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; 4797 if (IS_ERR(rm_res)) { 4798 irq_res.desc[0].start = oes->bcdma_bchan_ring; 4799 irq_res.desc[0].num = ud->bchan_cnt; 4800 i = 1; 4801 } else { 4802 for (i = 0; i < rm_res->sets; i++) { 4803 irq_res.desc[i].start = rm_res->desc[i].start + 4804 oes->bcdma_bchan_ring; 4805 irq_res.desc[i].num = rm_res->desc[i].num; 4806 } 4807 } 4808 } else { 4809 i = 0; 4810 } 4811 4812 if (ud->tchan_cnt) { 4813 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4814 if (IS_ERR(rm_res)) { 4815 irq_res.desc[i].start = oes->bcdma_tchan_data; 4816 irq_res.desc[i].num = ud->tchan_cnt; 4817 irq_res.desc[i + 1].start = oes->bcdma_tchan_ring; 4818 irq_res.desc[i + 1].num = ud->tchan_cnt; 4819 i += 2; 4820 } else { 4821 for (j = 0; j < rm_res->sets; j++, i += 2) { 4822 irq_res.desc[i].start = rm_res->desc[j].start + 4823 oes->bcdma_tchan_data; 4824 irq_res.desc[i].num = rm_res->desc[j].num; 4825 4826 irq_res.desc[i + 1].start = rm_res->desc[j].start + 4827 oes->bcdma_tchan_ring; 4828 irq_res.desc[i + 1].num = rm_res->desc[j].num; 4829 } 4830 } 4831 } 4832 if (ud->rchan_cnt) { 4833 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4834 if (IS_ERR(rm_res)) { 4835 irq_res.desc[i].start = oes->bcdma_rchan_data; 4836 irq_res.desc[i].num = ud->rchan_cnt; 4837 irq_res.desc[i + 1].start = oes->bcdma_rchan_ring; 4838 irq_res.desc[i + 1].num = ud->rchan_cnt; 4839 i += 2; 4840 } else { 4841 for (j = 0; j < rm_res->sets; j++, i += 2) { 4842 irq_res.desc[i].start = rm_res->desc[j].start + 4843 oes->bcdma_rchan_data; 4844 irq_res.desc[i].num = rm_res->desc[j].num; 4845 4846 irq_res.desc[i + 1].start = rm_res->desc[j].start + 4847 oes->bcdma_rchan_ring; 4848 irq_res.desc[i + 1].num = rm_res->desc[j].num; 4849 } 4850 } 4851 } 4852 4853 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 4854 kfree(irq_res.desc); 4855 if (ret) { 4856 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 4857 return ret; 4858 } 4859 4860 return 0; 4861 } 4862 4863 static int pktdma_setup_resources(struct udma_dev *ud) 4864 { 4865 int ret, i, j; 4866 struct device *dev = ud->dev; 4867 struct ti_sci_resource *rm_res, irq_res; 4868 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 4869 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 4870 u32 cap3; 4871 4872 /* Set up the throughput level start indexes */ 4873 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4874 if (UDMA_CAP3_UCHAN_CNT(cap3)) { 4875 ud->tchan_tpl.levels = 3; 4876 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); 4877 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4878 } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { 4879 ud->tchan_tpl.levels = 2; 4880 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4881 } else { 4882 ud->tchan_tpl.levels = 1; 4883 } 4884 4885 ud->rchan_tpl.levels = ud->tchan_tpl.levels; 4886 ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0]; 4887 ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1]; 4888 4889 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 4890 sizeof(unsigned long), GFP_KERNEL); 4891 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 4892 GFP_KERNEL); 4893 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 4894 sizeof(unsigned long), GFP_KERNEL); 4895 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 4896 GFP_KERNEL); 4897 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 4898 sizeof(unsigned long), 4899 GFP_KERNEL); 4900 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 4901 GFP_KERNEL); 4902 ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt), 4903 sizeof(unsigned long), GFP_KERNEL); 4904 4905 if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans || 4906 !ud->rchans || !ud->rflows || !ud->rflow_in_use) 4907 return -ENOMEM; 4908 4909 /* Get resource ranges from tisci */ 4910 for (i = 0; i < RM_RANGE_LAST; i++) { 4911 if (i == RM_RANGE_BCHAN) 4912 continue; 4913 4914 tisci_rm->rm_ranges[i] = 4915 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 4916 tisci_rm->tisci_dev_id, 4917 (char *)range_names[i]); 4918 } 4919 4920 /* tchan ranges */ 4921 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4922 if (IS_ERR(rm_res)) { 4923 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 4924 } else { 4925 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 4926 for (i = 0; i < rm_res->sets; i++) 4927 udma_mark_resource_ranges(ud, ud->tchan_map, 4928 &rm_res->desc[i], "tchan"); 4929 } 4930 4931 /* rchan ranges */ 4932 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4933 if (IS_ERR(rm_res)) { 4934 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 4935 } else { 4936 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 4937 for (i = 0; i < rm_res->sets; i++) 4938 udma_mark_resource_ranges(ud, ud->rchan_map, 4939 &rm_res->desc[i], "rchan"); 4940 } 4941 4942 /* rflow ranges */ 4943 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 4944 if (IS_ERR(rm_res)) { 4945 /* all rflows are assigned exclusively to Linux */ 4946 bitmap_zero(ud->rflow_in_use, ud->rflow_cnt); 4947 irq_res.sets = 1; 4948 } else { 4949 bitmap_fill(ud->rflow_in_use, ud->rflow_cnt); 4950 for (i = 0; i < rm_res->sets; i++) 4951 udma_mark_resource_ranges(ud, ud->rflow_in_use, 4952 &rm_res->desc[i], "rflow"); 4953 irq_res.sets = rm_res->sets; 4954 } 4955 4956 /* tflow ranges */ 4957 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; 4958 if (IS_ERR(rm_res)) { 4959 /* all tflows are assigned exclusively to Linux */ 4960 bitmap_zero(ud->tflow_map, ud->tflow_cnt); 4961 irq_res.sets++; 4962 } else { 4963 bitmap_fill(ud->tflow_map, ud->tflow_cnt); 4964 for (i = 0; i < rm_res->sets; i++) 4965 udma_mark_resource_ranges(ud, ud->tflow_map, 4966 &rm_res->desc[i], "tflow"); 4967 irq_res.sets += rm_res->sets; 4968 } 4969 4970 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 4971 if (!irq_res.desc) 4972 return -ENOMEM; 4973 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; 4974 if (IS_ERR(rm_res)) { 4975 irq_res.desc[0].start = oes->pktdma_tchan_flow; 4976 irq_res.desc[0].num = ud->tflow_cnt; 4977 i = 1; 4978 } else { 4979 for (i = 0; i < rm_res->sets; i++) { 4980 irq_res.desc[i].start = rm_res->desc[i].start + 4981 oes->pktdma_tchan_flow; 4982 irq_res.desc[i].num = rm_res->desc[i].num; 4983 } 4984 } 4985 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 4986 if (IS_ERR(rm_res)) { 4987 irq_res.desc[i].start = oes->pktdma_rchan_flow; 4988 irq_res.desc[i].num = ud->rflow_cnt; 4989 } else { 4990 for (j = 0; j < rm_res->sets; j++, i++) { 4991 irq_res.desc[i].start = rm_res->desc[j].start + 4992 oes->pktdma_rchan_flow; 4993 irq_res.desc[i].num = rm_res->desc[j].num; 4994 } 4995 } 4996 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 4997 kfree(irq_res.desc); 4998 if (ret) { 4999 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 5000 return ret; 5001 } 5002 5003 return 0; 5004 } 5005 5006 static int setup_resources(struct udma_dev *ud) 5007 { 5008 struct device *dev = ud->dev; 5009 int ch_count, ret; 5010 5011 switch (ud->match_data->type) { 5012 case DMA_TYPE_UDMA: 5013 ret = udma_setup_resources(ud); 5014 break; 5015 case DMA_TYPE_BCDMA: 5016 ret = bcdma_setup_resources(ud); 5017 break; 5018 case DMA_TYPE_PKTDMA: 5019 ret = pktdma_setup_resources(ud); 5020 break; 5021 default: 5022 return -EINVAL; 5023 } 5024 5025 if (ret) 5026 return ret; 5027 5028 ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt; 5029 if (ud->bchan_cnt) 5030 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt); 5031 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 5032 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 5033 if (!ch_count) 5034 return -ENODEV; 5035 5036 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 5037 GFP_KERNEL); 5038 if (!ud->channels) 5039 return -ENOMEM; 5040 5041 switch (ud->match_data->type) { 5042 case DMA_TYPE_UDMA: 5043 dev_info(dev, 5044 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 5045 ch_count, 5046 ud->tchan_cnt - bitmap_weight(ud->tchan_map, 5047 ud->tchan_cnt), 5048 ud->rchan_cnt - bitmap_weight(ud->rchan_map, 5049 ud->rchan_cnt), 5050 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 5051 ud->rflow_cnt)); 5052 break; 5053 case DMA_TYPE_BCDMA: 5054 dev_info(dev, 5055 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n", 5056 ch_count, 5057 ud->bchan_cnt - bitmap_weight(ud->bchan_map, 5058 ud->bchan_cnt), 5059 ud->tchan_cnt - bitmap_weight(ud->tchan_map, 5060 ud->tchan_cnt), 5061 ud->rchan_cnt - bitmap_weight(ud->rchan_map, 5062 ud->rchan_cnt)); 5063 break; 5064 case DMA_TYPE_PKTDMA: 5065 dev_info(dev, 5066 "Channels: %d (tchan: %u, rchan: %u)\n", 5067 ch_count, 5068 ud->tchan_cnt - bitmap_weight(ud->tchan_map, 5069 ud->tchan_cnt), 5070 ud->rchan_cnt - bitmap_weight(ud->rchan_map, 5071 ud->rchan_cnt)); 5072 break; 5073 default: 5074 break; 5075 } 5076 5077 return ch_count; 5078 } 5079 5080 static int udma_setup_rx_flush(struct udma_dev *ud) 5081 { 5082 struct udma_rx_flush *rx_flush = &ud->rx_flush; 5083 struct cppi5_desc_hdr_t *tr_desc; 5084 struct cppi5_tr_type1_t *tr_req; 5085 struct cppi5_host_desc_t *desc; 5086 struct device *dev = ud->dev; 5087 struct udma_hwdesc *hwdesc; 5088 size_t tr_size; 5089 5090 /* Allocate 1K buffer for discarded data on RX channel teardown */ 5091 rx_flush->buffer_size = SZ_1K; 5092 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, 5093 GFP_KERNEL); 5094 if (!rx_flush->buffer_vaddr) 5095 return -ENOMEM; 5096 5097 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, 5098 rx_flush->buffer_size, 5099 DMA_TO_DEVICE); 5100 if (dma_mapping_error(dev, rx_flush->buffer_paddr)) 5101 return -ENOMEM; 5102 5103 /* Set up descriptor to be used for TR mode */ 5104 hwdesc = &rx_flush->hwdescs[0]; 5105 tr_size = sizeof(struct cppi5_tr_type1_t); 5106 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); 5107 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 5108 ud->desc_align); 5109 5110 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 5111 GFP_KERNEL); 5112 if (!hwdesc->cppi5_desc_vaddr) 5113 return -ENOMEM; 5114 5115 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 5116 hwdesc->cppi5_desc_size, 5117 DMA_TO_DEVICE); 5118 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 5119 return -ENOMEM; 5120 5121 /* Start of the TR req records */ 5122 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 5123 /* Start address of the TR response array */ 5124 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; 5125 5126 tr_desc = hwdesc->cppi5_desc_vaddr; 5127 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); 5128 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 5129 cppi5_desc_set_retpolicy(tr_desc, 0, 0); 5130 5131 tr_req = hwdesc->tr_req_base; 5132 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, 5133 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 5134 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); 5135 5136 tr_req->addr = rx_flush->buffer_paddr; 5137 tr_req->icnt0 = rx_flush->buffer_size; 5138 tr_req->icnt1 = 1; 5139 5140 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 5141 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 5142 5143 /* Set up descriptor to be used for packet mode */ 5144 hwdesc = &rx_flush->hwdescs[1]; 5145 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 5146 CPPI5_INFO0_HDESC_EPIB_SIZE + 5147 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, 5148 ud->desc_align); 5149 5150 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 5151 GFP_KERNEL); 5152 if (!hwdesc->cppi5_desc_vaddr) 5153 return -ENOMEM; 5154 5155 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 5156 hwdesc->cppi5_desc_size, 5157 DMA_TO_DEVICE); 5158 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 5159 return -ENOMEM; 5160 5161 desc = hwdesc->cppi5_desc_vaddr; 5162 cppi5_hdesc_init(desc, 0, 0); 5163 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 5164 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); 5165 5166 cppi5_hdesc_attach_buf(desc, 5167 rx_flush->buffer_paddr, rx_flush->buffer_size, 5168 rx_flush->buffer_paddr, rx_flush->buffer_size); 5169 5170 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 5171 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 5172 return 0; 5173 } 5174 5175 #ifdef CONFIG_DEBUG_FS 5176 static void udma_dbg_summary_show_chan(struct seq_file *s, 5177 struct dma_chan *chan) 5178 { 5179 struct udma_chan *uc = to_udma_chan(chan); 5180 struct udma_chan_config *ucc = &uc->config; 5181 5182 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 5183 chan->dbg_client_name ?: "in-use"); 5184 if (ucc->tr_trigger_type) 5185 seq_puts(s, " (triggered, "); 5186 else 5187 seq_printf(s, " (%s, ", 5188 dmaengine_get_direction_text(uc->config.dir)); 5189 5190 switch (uc->config.dir) { 5191 case DMA_MEM_TO_MEM: 5192 if (uc->ud->match_data->type == DMA_TYPE_BCDMA) { 5193 seq_printf(s, "bchan%d)\n", uc->bchan->id); 5194 return; 5195 } 5196 5197 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 5198 ucc->src_thread, ucc->dst_thread); 5199 break; 5200 case DMA_DEV_TO_MEM: 5201 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 5202 ucc->src_thread, ucc->dst_thread); 5203 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA) 5204 seq_printf(s, "rflow%d, ", uc->rflow->id); 5205 break; 5206 case DMA_MEM_TO_DEV: 5207 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 5208 ucc->src_thread, ucc->dst_thread); 5209 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA) 5210 seq_printf(s, "tflow%d, ", uc->tchan->tflow_id); 5211 break; 5212 default: 5213 seq_printf(s, ")\n"); 5214 return; 5215 } 5216 5217 if (ucc->ep_type == PSIL_EP_NATIVE) { 5218 seq_printf(s, "PSI-L Native"); 5219 if (ucc->metadata_size) { 5220 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 5221 if (ucc->psd_size) 5222 seq_printf(s, " PSDsize:%u", ucc->psd_size); 5223 seq_printf(s, " ]"); 5224 } 5225 } else { 5226 seq_printf(s, "PDMA"); 5227 if (ucc->enable_acc32 || ucc->enable_burst) 5228 seq_printf(s, "[%s%s ]", 5229 ucc->enable_acc32 ? " ACC32" : "", 5230 ucc->enable_burst ? " BURST" : ""); 5231 } 5232 5233 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 5234 } 5235 5236 static void udma_dbg_summary_show(struct seq_file *s, 5237 struct dma_device *dma_dev) 5238 { 5239 struct dma_chan *chan; 5240 5241 list_for_each_entry(chan, &dma_dev->channels, device_node) { 5242 if (chan->client_count) 5243 udma_dbg_summary_show_chan(s, chan); 5244 } 5245 } 5246 #endif /* CONFIG_DEBUG_FS */ 5247 5248 static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud) 5249 { 5250 const struct udma_match_data *match_data = ud->match_data; 5251 u8 tpl; 5252 5253 if (!match_data->enable_memcpy_support) 5254 return DMAENGINE_ALIGN_8_BYTES; 5255 5256 /* Get the highest TPL level the device supports for memcpy */ 5257 if (ud->bchan_cnt) 5258 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0); 5259 else if (ud->tchan_cnt) 5260 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0); 5261 else 5262 return DMAENGINE_ALIGN_8_BYTES; 5263 5264 switch (match_data->burst_size[tpl]) { 5265 case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES: 5266 return DMAENGINE_ALIGN_256_BYTES; 5267 case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES: 5268 return DMAENGINE_ALIGN_128_BYTES; 5269 case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES: 5270 fallthrough; 5271 default: 5272 return DMAENGINE_ALIGN_64_BYTES; 5273 } 5274 } 5275 5276 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 5277 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 5278 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 5279 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 5280 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 5281 5282 static int udma_probe(struct platform_device *pdev) 5283 { 5284 struct device_node *navss_node = pdev->dev.parent->of_node; 5285 const struct soc_device_attribute *soc; 5286 struct device *dev = &pdev->dev; 5287 struct udma_dev *ud; 5288 const struct of_device_id *match; 5289 int i, ret; 5290 int ch_count; 5291 5292 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 5293 if (ret) 5294 dev_err(dev, "failed to set dma mask stuff\n"); 5295 5296 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 5297 if (!ud) 5298 return -ENOMEM; 5299 5300 match = of_match_node(udma_of_match, dev->of_node); 5301 if (!match) { 5302 dev_err(dev, "No compatible match found\n"); 5303 return -ENODEV; 5304 } 5305 ud->match_data = match->data; 5306 5307 ud->soc_data = ud->match_data->soc_data; 5308 if (!ud->soc_data) { 5309 soc = soc_device_match(k3_soc_devices); 5310 if (!soc) { 5311 dev_err(dev, "No compatible SoC found\n"); 5312 return -ENODEV; 5313 } 5314 ud->soc_data = soc->data; 5315 } 5316 5317 ret = udma_get_mmrs(pdev, ud); 5318 if (ret) 5319 return ret; 5320 5321 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 5322 if (IS_ERR(ud->tisci_rm.tisci)) 5323 return PTR_ERR(ud->tisci_rm.tisci); 5324 5325 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 5326 &ud->tisci_rm.tisci_dev_id); 5327 if (ret) { 5328 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 5329 return ret; 5330 } 5331 pdev->id = ud->tisci_rm.tisci_dev_id; 5332 5333 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 5334 &ud->tisci_rm.tisci_navss_dev_id); 5335 if (ret) { 5336 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 5337 return ret; 5338 } 5339 5340 if (ud->match_data->type == DMA_TYPE_UDMA) { 5341 ret = of_property_read_u32(dev->of_node, "ti,udma-atype", 5342 &ud->atype); 5343 if (!ret && ud->atype > 2) { 5344 dev_err(dev, "Invalid atype: %u\n", ud->atype); 5345 return -EINVAL; 5346 } 5347 } else { 5348 ret = of_property_read_u32(dev->of_node, "ti,asel", 5349 &ud->asel); 5350 if (!ret && ud->asel > 15) { 5351 dev_err(dev, "Invalid asel: %u\n", ud->asel); 5352 return -EINVAL; 5353 } 5354 } 5355 5356 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 5357 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 5358 5359 if (ud->match_data->type == DMA_TYPE_UDMA) { 5360 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 5361 } else { 5362 struct k3_ringacc_init_data ring_init_data; 5363 5364 ring_init_data.tisci = ud->tisci_rm.tisci; 5365 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id; 5366 if (ud->match_data->type == DMA_TYPE_BCDMA) { 5367 ring_init_data.num_rings = ud->bchan_cnt + 5368 ud->tchan_cnt + 5369 ud->rchan_cnt; 5370 } else { 5371 ring_init_data.num_rings = ud->rflow_cnt + 5372 ud->tflow_cnt; 5373 } 5374 5375 ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data); 5376 } 5377 5378 if (IS_ERR(ud->ringacc)) 5379 return PTR_ERR(ud->ringacc); 5380 5381 dev->msi.domain = of_msi_get_domain(dev, dev->of_node, 5382 DOMAIN_BUS_TI_SCI_INTA_MSI); 5383 if (!dev->msi.domain) { 5384 return -EPROBE_DEFER; 5385 } 5386 5387 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 5388 /* cyclic operation is not supported via PKTDMA */ 5389 if (ud->match_data->type != DMA_TYPE_PKTDMA) { 5390 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 5391 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 5392 } 5393 5394 ud->ddev.device_config = udma_slave_config; 5395 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 5396 ud->ddev.device_issue_pending = udma_issue_pending; 5397 ud->ddev.device_tx_status = udma_tx_status; 5398 ud->ddev.device_pause = udma_pause; 5399 ud->ddev.device_resume = udma_resume; 5400 ud->ddev.device_terminate_all = udma_terminate_all; 5401 ud->ddev.device_synchronize = udma_synchronize; 5402 #ifdef CONFIG_DEBUG_FS 5403 ud->ddev.dbg_summary_show = udma_dbg_summary_show; 5404 #endif 5405 5406 switch (ud->match_data->type) { 5407 case DMA_TYPE_UDMA: 5408 ud->ddev.device_alloc_chan_resources = 5409 udma_alloc_chan_resources; 5410 break; 5411 case DMA_TYPE_BCDMA: 5412 ud->ddev.device_alloc_chan_resources = 5413 bcdma_alloc_chan_resources; 5414 ud->ddev.device_router_config = bcdma_router_config; 5415 break; 5416 case DMA_TYPE_PKTDMA: 5417 ud->ddev.device_alloc_chan_resources = 5418 pktdma_alloc_chan_resources; 5419 break; 5420 default: 5421 return -EINVAL; 5422 } 5423 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 5424 5425 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 5426 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 5427 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 5428 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 5429 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 5430 DESC_METADATA_ENGINE; 5431 if (ud->match_data->enable_memcpy_support && 5432 !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) { 5433 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 5434 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 5435 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 5436 } 5437 5438 ud->ddev.dev = dev; 5439 ud->dev = dev; 5440 ud->psil_base = ud->match_data->psil_base; 5441 5442 INIT_LIST_HEAD(&ud->ddev.channels); 5443 INIT_LIST_HEAD(&ud->desc_to_purge); 5444 5445 ch_count = setup_resources(ud); 5446 if (ch_count <= 0) 5447 return ch_count; 5448 5449 spin_lock_init(&ud->lock); 5450 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 5451 5452 ud->desc_align = 64; 5453 if (ud->desc_align < dma_get_cache_alignment()) 5454 ud->desc_align = dma_get_cache_alignment(); 5455 5456 ret = udma_setup_rx_flush(ud); 5457 if (ret) 5458 return ret; 5459 5460 for (i = 0; i < ud->bchan_cnt; i++) { 5461 struct udma_bchan *bchan = &ud->bchans[i]; 5462 5463 bchan->id = i; 5464 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000; 5465 } 5466 5467 for (i = 0; i < ud->tchan_cnt; i++) { 5468 struct udma_tchan *tchan = &ud->tchans[i]; 5469 5470 tchan->id = i; 5471 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 5472 } 5473 5474 for (i = 0; i < ud->rchan_cnt; i++) { 5475 struct udma_rchan *rchan = &ud->rchans[i]; 5476 5477 rchan->id = i; 5478 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 5479 } 5480 5481 for (i = 0; i < ud->rflow_cnt; i++) { 5482 struct udma_rflow *rflow = &ud->rflows[i]; 5483 5484 rflow->id = i; 5485 } 5486 5487 for (i = 0; i < ch_count; i++) { 5488 struct udma_chan *uc = &ud->channels[i]; 5489 5490 uc->ud = ud; 5491 uc->vc.desc_free = udma_desc_free; 5492 uc->id = i; 5493 uc->bchan = NULL; 5494 uc->tchan = NULL; 5495 uc->rchan = NULL; 5496 uc->config.remote_thread_id = -1; 5497 uc->config.mapped_channel_id = -1; 5498 uc->config.default_flow_id = -1; 5499 uc->config.dir = DMA_MEM_TO_MEM; 5500 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 5501 dev_name(dev), i); 5502 5503 vchan_init(&uc->vc, &ud->ddev); 5504 /* Use custom vchan completion handling */ 5505 tasklet_setup(&uc->vc.task, udma_vchan_complete); 5506 init_completion(&uc->teardown_completed); 5507 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); 5508 } 5509 5510 /* Configure the copy_align to the maximum burst size the device supports */ 5511 ud->ddev.copy_align = udma_get_copy_align(ud); 5512 5513 ret = dma_async_device_register(&ud->ddev); 5514 if (ret) { 5515 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 5516 return ret; 5517 } 5518 5519 platform_set_drvdata(pdev, ud); 5520 5521 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 5522 if (ret) { 5523 dev_err(dev, "failed to register of_dma controller\n"); 5524 dma_async_device_unregister(&ud->ddev); 5525 } 5526 5527 return ret; 5528 } 5529 5530 static int udma_pm_suspend(struct device *dev) 5531 { 5532 struct udma_dev *ud = dev_get_drvdata(dev); 5533 struct dma_device *dma_dev = &ud->ddev; 5534 struct dma_chan *chan; 5535 struct udma_chan *uc; 5536 5537 list_for_each_entry(chan, &dma_dev->channels, device_node) { 5538 if (chan->client_count) { 5539 uc = to_udma_chan(chan); 5540 /* backup the channel configuration */ 5541 memcpy(&uc->backup_config, &uc->config, 5542 sizeof(struct udma_chan_config)); 5543 dev_dbg(dev, "Suspending channel %s\n", 5544 dma_chan_name(chan)); 5545 ud->ddev.device_free_chan_resources(chan); 5546 } 5547 } 5548 5549 return 0; 5550 } 5551 5552 static int udma_pm_resume(struct device *dev) 5553 { 5554 struct udma_dev *ud = dev_get_drvdata(dev); 5555 struct dma_device *dma_dev = &ud->ddev; 5556 struct dma_chan *chan; 5557 struct udma_chan *uc; 5558 int ret; 5559 5560 list_for_each_entry(chan, &dma_dev->channels, device_node) { 5561 if (chan->client_count) { 5562 uc = to_udma_chan(chan); 5563 /* restore the channel configuration */ 5564 memcpy(&uc->config, &uc->backup_config, 5565 sizeof(struct udma_chan_config)); 5566 dev_dbg(dev, "Resuming channel %s\n", 5567 dma_chan_name(chan)); 5568 ret = ud->ddev.device_alloc_chan_resources(chan); 5569 if (ret) 5570 return ret; 5571 } 5572 } 5573 5574 return 0; 5575 } 5576 5577 static const struct dev_pm_ops udma_pm_ops = { 5578 SET_LATE_SYSTEM_SLEEP_PM_OPS(udma_pm_suspend, udma_pm_resume) 5579 }; 5580 5581 static struct platform_driver udma_driver = { 5582 .driver = { 5583 .name = "ti-udma", 5584 .of_match_table = udma_of_match, 5585 .suppress_bind_attrs = true, 5586 .pm = &udma_pm_ops, 5587 }, 5588 .probe = udma_probe, 5589 }; 5590 5591 module_platform_driver(udma_driver); 5592 MODULE_LICENSE("GPL v2"); 5593 5594 /* Private interfaces to UDMA */ 5595 #include "k3-udma-private.c" 5596