Lines Matching +full:sci +full:- +full:rm +full:- +full:range +full:- +full:gp +full:- +full:rings
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
11 #include <linux/dma-mapping.h>
26 #include <linux/soc/ti/k3-ringacc.h>
29 #include <linux/dma/k3-event-router.h>
30 #include <linux/dma/ti-cppi5.h>
32 #include "../virt-dma.h"
33 #include "k3-udma.h"
34 #include "k3-psil-priv.h"
359 if (!uc->tchan)
361 return udma_read(uc->tchan->reg_rt, reg);
366 if (!uc->tchan)
368 udma_write(uc->tchan->reg_rt, reg, val);
374 if (!uc->tchan)
376 udma_update_bits(uc->tchan->reg_rt, reg, mask, val);
382 if (!uc->rchan)
384 return udma_read(uc->rchan->reg_rt, reg);
389 if (!uc->rchan)
391 udma_write(uc->rchan->reg_rt, reg, val);
397 if (!uc->rchan)
399 udma_update_bits(uc->rchan->reg_rt, reg, mask, val);
404 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
407 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
408 tisci_rm->tisci_navss_dev_id,
415 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
418 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
419 tisci_rm->tisci_navss_dev_id,
425 struct device *chan_dev = &chan->dev->device;
429 chan->dev->chan_dma_dev = false;
431 chan_dev->dma_coherent = false;
432 chan_dev->dma_parms = NULL;
434 chan->dev->chan_dma_dev = true;
436 chan_dev->dma_coherent = true;
438 chan_dev->dma_parms = chan_dev->parent->dma_parms;
440 dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);
442 chan_dev->dma_coherent = false;
443 chan_dev->dma_parms = NULL;
451 for (i = 0; i < tpl_map->levels; i++) {
452 if (chan_id >= tpl_map->start_idx[i])
461 memset(&uc->config, 0, sizeof(uc->config));
462 uc->config.remote_thread_id = -1;
463 uc->config.mapped_channel_id = -1;
464 uc->config.default_flow_id = -1;
465 uc->state = UDMA_CHAN_IS_IDLE;
470 struct device *dev = uc->ud->dev;
474 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {
483 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {
496 return d->hwdesc[idx].cppi5_desc_paddr;
501 return d->hwdesc[idx].cppi5_desc_vaddr;
507 struct udma_desc *d = uc->terminated_desc;
511 d->desc_idx);
518 d = uc->desc;
521 d->desc_idx);
533 if (uc->use_dma_pool) {
536 for (i = 0; i < d->hwdesc_count; i++) {
537 if (!d->hwdesc[i].cppi5_desc_vaddr)
540 dma_pool_free(uc->hdesc_pool,
541 d->hwdesc[i].cppi5_desc_vaddr,
542 d->hwdesc[i].cppi5_desc_paddr);
544 d->hwdesc[i].cppi5_desc_vaddr = NULL;
546 } else if (d->hwdesc[0].cppi5_desc_vaddr) {
547 dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,
548 d->hwdesc[0].cppi5_desc_vaddr,
549 d->hwdesc[0].cppi5_desc_paddr);
551 d->hwdesc[0].cppi5_desc_vaddr = NULL;
562 spin_lock_irqsave(&ud->lock, flags);
563 list_splice_tail_init(&ud->desc_to_purge, &head);
564 spin_unlock_irqrestore(&ud->lock, flags);
567 struct udma_chan *uc = to_udma_chan(vd->tx.chan);
568 struct udma_desc *d = to_udma_desc(&vd->tx);
571 list_del(&vd->node);
576 if (!list_empty(&ud->desc_to_purge))
577 schedule_work(&ud->purge_work);
582 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
583 struct udma_chan *uc = to_udma_chan(vd->tx.chan);
584 struct udma_desc *d = to_udma_desc(&vd->tx);
587 if (uc->terminated_desc == d)
588 uc->terminated_desc = NULL;
590 if (uc->use_dma_pool) {
596 spin_lock_irqsave(&ud->lock, flags);
597 list_add_tail(&vd->node, &ud->desc_to_purge);
598 spin_unlock_irqrestore(&ud->lock, flags);
600 schedule_work(&ud->purge_work);
608 if (uc->tchan)
610 if (uc->rchan)
623 switch (uc->config.dir) {
648 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;
653 struct udma_desc *d = uc->desc;
657 switch (uc->config.dir) {
659 ring = uc->rflow->fd_ring;
663 ring = uc->tchan->t_ring;
666 return -EINVAL;
669 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
670 if (idx == -1) {
683 if (uc->config.dir != DMA_DEV_TO_MEM)
697 switch (uc->config.dir) {
699 ring = uc->rflow->r_ring;
703 ring = uc->tchan->tc_ring;
706 return -ENOENT;
721 return -ENOENT;
731 switch (uc->config.dir) {
733 if (uc->rchan) {
734 ring1 = uc->rflow->fd_ring;
735 ring2 = uc->rflow->r_ring;
740 if (uc->tchan) {
741 ring1 = uc->tchan->t_ring;
742 ring2 = uc->tchan->tc_ring;
756 if (uc->terminated_desc) {
757 udma_desc_free(&uc->terminated_desc->vd);
758 uc->terminated_desc = NULL;
764 if (uc->desc->dir == DMA_DEV_TO_MEM) {
767 if (uc->config.ep_type != PSIL_EP_NATIVE)
772 if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE)
781 if (uc->tchan) {
791 if (!uc->bchan) {
797 if (uc->rchan) {
814 switch (uc->config.dir) {
828 return -EINVAL;
834 /* Hard reset: re-initialize the channel to reset */
839 memcpy(&ucc_backup, &uc->config, sizeof(uc->config));
840 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
843 memcpy(&uc->config, &ucc_backup, sizeof(uc->config));
844 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
852 if (uc->config.dir == DMA_DEV_TO_MEM)
858 uc->state = UDMA_CHAN_IS_IDLE;
865 struct udma_chan_config *ucc = &uc->config;
867 if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode &&
868 (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {
874 * PKTDMA supports pre-linked descriptor and cyclic is not
877 for (i = 0; i < uc->desc->sglen; i++)
887 if (uc->config.ep_type == PSIL_EP_NATIVE)
891 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
899 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
902 uc->desc = NULL;
903 return -ENOENT;
906 list_del(&vd->node);
908 uc->desc = to_udma_desc(&vd->tx);
922 switch (uc->desc->dir) {
925 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
926 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
927 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
929 uc->ud->match_data;
931 if (uc->config.enable_acc32)
933 if (uc->config.enable_burst)
942 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
943 match_data->statictr_z_mask));
946 memcpy(&uc->static_tr, &uc->desc->static_tr,
947 sizeof(uc->static_tr));
960 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
961 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
962 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
964 if (uc->config.enable_acc32)
966 if (uc->config.enable_burst)
974 memcpy(&uc->static_tr, &uc->desc->static_tr,
975 sizeof(uc->static_tr));
994 return -EINVAL;
997 uc->state = UDMA_CHAN_IS_ACTIVE;
1005 enum udma_chan_state old_state = uc->state;
1007 uc->state = UDMA_CHAN_IS_TERMINATING;
1008 reinit_completion(&uc->teardown_completed);
1010 switch (uc->config.dir) {
1012 if (!uc->cyclic && !uc->desc)
1013 udma_push_to_ring(uc, -1);
1033 uc->state = old_state;
1034 complete_all(&uc->teardown_completed);
1035 return -EINVAL;
1043 struct udma_desc *d = uc->desc;
1046 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
1048 udma_push_to_ring(uc, d->desc_idx);
1049 d->desc_idx = (d->desc_idx + 1) % d->sglen;
1054 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
1056 memcpy(d->metadata, h_desc->epib, d->metadata_size);
1069 if (uc->config.ep_type == PSIL_EP_NATIVE ||
1070 uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))
1078 uc->tx_drain.residue = bcnt - peer_bcnt;
1079 uc->tx_drain.tstamp = ktime_get();
1096 if (uc->desc) {
1098 residue_diff = uc->tx_drain.residue;
1099 time_diff = uc->tx_drain.tstamp;
1104 desc_done = udma_is_desc_really_done(uc, uc->desc);
1112 time_diff = ktime_sub(uc->tx_drain.tstamp,
1114 residue_diff -= uc->tx_drain.residue;
1123 uc->tx_drain.residue;
1126 schedule_delayed_work(&uc->tx_drain.work, HZ);
1135 if (uc->desc) {
1136 struct udma_desc *d = uc->desc;
1138 udma_decrement_byte_counters(uc, d->residue);
1140 vchan_cookie_complete(&d->vd);
1157 spin_lock(&uc->vc.lock);
1161 complete_all(&uc->teardown_completed);
1163 if (uc->terminated_desc) {
1164 udma_desc_free(&uc->terminated_desc->vd);
1165 uc->terminated_desc = NULL;
1168 if (!uc->desc)
1178 d->desc_idx);
1180 dev_err(uc->ud->dev, "not matching descriptors!\n");
1184 if (d == uc->desc) {
1186 if (uc->cyclic) {
1188 vchan_cyclic_callback(&d->vd);
1191 udma_decrement_byte_counters(uc, d->residue);
1193 vchan_cookie_complete(&d->vd);
1195 schedule_delayed_work(&uc->tx_drain.work,
1204 dma_cookie_complete(&d->vd.tx);
1208 spin_unlock(&uc->vc.lock);
1218 spin_lock(&uc->vc.lock);
1219 d = uc->desc;
1221 d->tr_idx = (d->tr_idx + 1) % d->sglen;
1223 if (uc->cyclic) {
1224 vchan_cyclic_callback(&d->vd);
1227 udma_decrement_byte_counters(uc, d->residue);
1229 vchan_cookie_complete(&d->vd);
1233 spin_unlock(&uc->vc.lock);
1239 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
1244 * Allocate range of RX flow ids for future use, those flows can be requested
1245 * only using explicit flow id number. if @from is set to -1 it will try to find
1246 * first free range. if @from is positive value it will force allocation only
1247 * of the specified range of flows.
1249 * Returns -ENOMEM if can't find free range.
1250 * -EEXIST if requested range is busy.
1251 * -EINVAL if wrong input values passed.
1261 tmp_from = ud->rchan_cnt;
1263 if (tmp_from < ud->rchan_cnt)
1264 return -EINVAL;
1266 if (tmp_from + cnt > ud->rflow_cnt)
1267 return -EINVAL;
1269 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
1270 ud->rflow_cnt);
1273 ud->rflow_cnt,
1275 if (start >= ud->rflow_cnt)
1276 return -ENOMEM;
1279 return -EEXIST;
1281 bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
1287 if (from < ud->rchan_cnt)
1288 return -EINVAL;
1289 if (from + cnt > ud->rflow_cnt)
1290 return -EINVAL;
1292 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
1301 * TI-SCI FW will perform additional permission check ant way, it's
1305 if (id < 0 || id >= ud->rflow_cnt)
1306 return ERR_PTR(-ENOENT);
1308 if (test_bit(id, ud->rflow_in_use))
1309 return ERR_PTR(-ENOENT);
1311 if (ud->rflow_gp_map) {
1312 /* GP rflow has to be allocated first */
1313 if (!test_bit(id, ud->rflow_gp_map) &&
1314 !test_bit(id, ud->rflow_gp_map_allocated))
1315 return ERR_PTR(-EINVAL);
1318 dev_dbg(ud->dev, "get rflow%d\n", id);
1319 set_bit(id, ud->rflow_in_use);
1320 return &ud->rflows[id];
1325 if (!test_bit(rflow->id, ud->rflow_in_use)) {
1326 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
1330 dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
1331 clear_bit(rflow->id, ud->rflow_in_use);
1340 if (test_bit(id, ud->res##_map)) { \
1341 dev_err(ud->dev, "res##%d is in use\n", id); \
1342 return ERR_PTR(-ENOENT); \
1347 if (tpl >= ud->res##_tpl.levels) \
1348 tpl = ud->res##_tpl.levels - 1; \
1350 start = ud->res##_tpl.start_idx[tpl]; \
1352 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \
1354 if (id == ud->res##_cnt) { \
1355 return ERR_PTR(-ENOENT); \
1359 set_bit(id, ud->res##_map); \
1360 return &ud->res##s[id]; \
1369 struct udma_dev *ud = uc->ud;
1373 if (uc->bchan) {
1374 dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",
1375 uc->id, uc->bchan->id);
1383 if (uc->config.tr_trigger_type)
1386 tpl = ud->bchan_tpl.levels - 1;
1388 uc->bchan = __udma_reserve_bchan(ud, tpl, -1);
1389 if (IS_ERR(uc->bchan)) {
1390 ret = PTR_ERR(uc->bchan);
1391 uc->bchan = NULL;
1395 uc->tchan = uc->bchan;
1402 struct udma_dev *ud = uc->ud;
1405 if (uc->tchan) {
1406 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
1407 uc->id, uc->tchan->id);
1412 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1416 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,
1417 uc->config.mapped_channel_id);
1418 if (IS_ERR(uc->tchan)) {
1419 ret = PTR_ERR(uc->tchan);
1420 uc->tchan = NULL;
1424 if (ud->tflow_cnt) {
1428 if (uc->config.default_flow_id >= 0)
1429 tflow_id = uc->config.default_flow_id;
1431 tflow_id = uc->tchan->id;
1433 if (test_bit(tflow_id, ud->tflow_map)) {
1434 dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
1435 clear_bit(uc->tchan->id, ud->tchan_map);
1436 uc->tchan = NULL;
1437 return -ENOENT;
1440 uc->tchan->tflow_id = tflow_id;
1441 set_bit(tflow_id, ud->tflow_map);
1443 uc->tchan->tflow_id = -1;
1451 struct udma_dev *ud = uc->ud;
1454 if (uc->rchan) {
1455 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
1456 uc->id, uc->rchan->id);
1461 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1465 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,
1466 uc->config.mapped_channel_id);
1467 if (IS_ERR(uc->rchan)) {
1468 ret = PTR_ERR(uc->rchan);
1469 uc->rchan = NULL;
1478 struct udma_dev *ud = uc->ud;
1481 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
1482 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
1483 uc->id, uc->tchan->id);
1487 if (uc->tchan) {
1488 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
1489 uc->id, uc->tchan->id);
1490 return -EBUSY;
1491 } else if (uc->rchan) {
1492 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
1493 uc->id, uc->rchan->id);
1494 return -EBUSY;
1498 end = min(ud->tchan_cnt, ud->rchan_cnt);
1503 chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1];
1505 if (!test_bit(chan_id, ud->tchan_map) &&
1506 !test_bit(chan_id, ud->rchan_map))
1511 return -ENOENT;
1513 set_bit(chan_id, ud->tchan_map);
1514 set_bit(chan_id, ud->rchan_map);
1515 uc->tchan = &ud->tchans[chan_id];
1516 uc->rchan = &ud->rchans[chan_id];
1519 uc->tchan->tflow_id = -1;
1526 struct udma_dev *ud = uc->ud;
1529 if (!uc->rchan) {
1530 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
1531 return -EINVAL;
1534 if (uc->rflow) {
1535 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
1536 uc->id, uc->rflow->id);
1540 uc->rflow = __udma_get_rflow(ud, flow_id);
1541 if (IS_ERR(uc->rflow)) {
1542 ret = PTR_ERR(uc->rflow);
1543 uc->rflow = NULL;
1552 struct udma_dev *ud = uc->ud;
1554 if (uc->bchan) {
1555 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
1556 uc->bchan->id);
1557 clear_bit(uc->bchan->id, ud->bchan_map);
1558 uc->bchan = NULL;
1559 uc->tchan = NULL;
1565 struct udma_dev *ud = uc->ud;
1567 if (uc->rchan) {
1568 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
1569 uc->rchan->id);
1570 clear_bit(uc->rchan->id, ud->rchan_map);
1571 uc->rchan = NULL;
1577 struct udma_dev *ud = uc->ud;
1579 if (uc->tchan) {
1580 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
1581 uc->tchan->id);
1582 clear_bit(uc->tchan->id, ud->tchan_map);
1584 if (uc->tchan->tflow_id >= 0)
1585 clear_bit(uc->tchan->tflow_id, ud->tflow_map);
1587 uc->tchan = NULL;
1593 struct udma_dev *ud = uc->ud;
1595 if (uc->rflow) {
1596 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
1597 uc->rflow->id);
1598 __udma_put_rflow(ud, uc->rflow);
1599 uc->rflow = NULL;
1605 if (!uc->bchan)
1608 k3_ringacc_ring_free(uc->bchan->tc_ring);
1609 k3_ringacc_ring_free(uc->bchan->t_ring);
1610 uc->bchan->tc_ring = NULL;
1611 uc->bchan->t_ring = NULL;
1612 k3_configure_chan_coherency(&uc->vc.chan, 0);
1620 struct udma_dev *ud = uc->ud;
1627 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
1628 &uc->bchan->t_ring,
1629 &uc->bchan->tc_ring);
1631 ret = -EBUSY;
1640 k3_configure_chan_coherency(&uc->vc.chan, ud->asel);
1641 ring_cfg.asel = ud->asel;
1642 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1644 ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
1651 k3_ringacc_ring_free(uc->bchan->tc_ring);
1652 uc->bchan->tc_ring = NULL;
1653 k3_ringacc_ring_free(uc->bchan->t_ring);
1654 uc->bchan->t_ring = NULL;
1655 k3_configure_chan_coherency(&uc->vc.chan, 0);
1664 if (!uc->tchan)
1667 k3_ringacc_ring_free(uc->tchan->t_ring);
1668 k3_ringacc_ring_free(uc->tchan->tc_ring);
1669 uc->tchan->t_ring = NULL;
1670 uc->tchan->tc_ring = NULL;
1678 struct udma_dev *ud = uc->ud;
1686 tchan = uc->tchan;
1687 if (tchan->tflow_id >= 0)
1688 ring_idx = tchan->tflow_id;
1690 ring_idx = ud->bchan_cnt + tchan->id;
1692 ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
1693 &tchan->t_ring,
1694 &tchan->tc_ring);
1696 ret = -EBUSY;
1703 if (ud->match_data->type == DMA_TYPE_UDMA) {
1708 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1709 ring_cfg.asel = uc->config.asel;
1710 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1713 ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg);
1714 ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg);
1722 k3_ringacc_ring_free(uc->tchan->tc_ring);
1723 uc->tchan->tc_ring = NULL;
1724 k3_ringacc_ring_free(uc->tchan->t_ring);
1725 uc->tchan->t_ring = NULL;
1734 if (!uc->rchan)
1737 if (uc->rflow) {
1738 struct udma_rflow *rflow = uc->rflow;
1740 k3_ringacc_ring_free(rflow->fd_ring);
1741 k3_ringacc_ring_free(rflow->r_ring);
1742 rflow->fd_ring = NULL;
1743 rflow->r_ring = NULL;
1753 struct udma_dev *ud = uc->ud;
1763 /* For MEM_TO_MEM we don't need rflow or rings */
1764 if (uc->config.dir == DMA_MEM_TO_MEM)
1767 if (uc->config.default_flow_id >= 0)
1768 ret = udma_get_rflow(uc, uc->config.default_flow_id);
1770 ret = udma_get_rflow(uc, uc->rchan->id);
1773 ret = -EBUSY;
1777 rflow = uc->rflow;
1778 if (ud->tflow_cnt)
1779 fd_ring_id = ud->tflow_cnt + rflow->id;
1781 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
1782 uc->rchan->id;
1784 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
1785 &rflow->fd_ring, &rflow->r_ring);
1787 ret = -EBUSY;
1794 if (ud->match_data->type == DMA_TYPE_UDMA) {
1795 if (uc->config.pkt_mode)
1805 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1806 ring_cfg.asel = uc->config.asel;
1807 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1810 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
1813 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
1821 k3_ringacc_ring_free(rflow->r_ring);
1822 rflow->r_ring = NULL;
1823 k3_ringacc_ring_free(rflow->fd_ring);
1824 rflow->fd_ring = NULL;
1867 struct udma_dev *ud = uc->ud;
1868 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1869 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1870 struct udma_tchan *tchan = uc->tchan;
1871 struct udma_rchan *rchan = uc->rchan;
1876 /* Non synchronized - mem to mem type of transfer */
1877 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1881 if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1882 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id);
1884 burst_size = ud->match_data->burst_size[tpl];
1888 req_tx.nav_id = tisci_rm->tisci_dev_id;
1889 req_tx.index = tchan->id;
1893 req_tx.tx_atype = ud->atype;
1899 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1901 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1906 req_rx.nav_id = tisci_rm->tisci_dev_id;
1907 req_rx.index = rchan->id;
1911 req_rx.rx_atype = ud->atype;
1917 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
1919 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);
1926 struct udma_dev *ud = uc->ud;
1927 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1928 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1930 struct udma_bchan *bchan = uc->bchan;
1935 if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1936 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id);
1938 burst_size = ud->match_data->burst_size[tpl];
1942 req_tx.nav_id = tisci_rm->tisci_dev_id;
1944 req_tx.index = bchan->id;
1950 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1952 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1959 struct udma_dev *ud = uc->ud;
1960 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1961 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1962 struct udma_tchan *tchan = uc->tchan;
1963 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1968 if (uc->config.pkt_mode) {
1970 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1971 uc->config.psd_size, 0);
1978 req_tx.nav_id = tisci_rm->tisci_dev_id;
1979 req_tx.index = tchan->id;
1981 req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1984 req_tx.tx_atype = uc->config.atype;
1985 if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
1986 ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1993 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1995 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
2002 struct udma_dev *ud = uc->ud;
2003 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2004 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2005 struct udma_tchan *tchan = uc->tchan;
2010 req_tx.nav_id = tisci_rm->tisci_dev_id;
2011 req_tx.index = tchan->id;
2012 req_tx.tx_supr_tdpkt = uc->config.notdpkt;
2013 if (ud->match_data->flags & UDMA_FLAG_TDTYPE) {
2020 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
2022 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
2031 struct udma_dev *ud = uc->ud;
2032 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2033 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2034 struct udma_rchan *rchan = uc->rchan;
2035 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);
2036 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2042 if (uc->config.pkt_mode) {
2044 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
2045 uc->config.psd_size, 0);
2052 req_rx.nav_id = tisci_rm->tisci_dev_id;
2053 req_rx.index = rchan->id;
2057 req_rx.rx_atype = uc->config.atype;
2059 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2061 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2080 flow_req.nav_id = tisci_rm->tisci_dev_id;
2081 flow_req.flow_index = rchan->id;
2083 if (uc->config.needs_epib)
2087 if (uc->config.psd_size)
2102 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2105 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);
2112 struct udma_dev *ud = uc->ud;
2113 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2114 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2115 struct udma_rchan *rchan = uc->rchan;
2120 req_rx.nav_id = tisci_rm->tisci_dev_id;
2121 req_rx.index = rchan->id;
2123 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2125 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2132 struct udma_dev *ud = uc->ud;
2133 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2134 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2140 req_rx.nav_id = tisci_rm->tisci_dev_id;
2141 req_rx.index = uc->rchan->id;
2143 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2145 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2154 flow_req.nav_id = tisci_rm->tisci_dev_id;
2155 flow_req.flow_index = uc->rflow->id;
2157 if (uc->config.needs_epib)
2161 if (uc->config.psd_size)
2167 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2170 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2179 struct udma_dev *ud = to_udma_dev(chan->device);
2180 const struct udma_soc_data *soc_data = ud->soc_data;
2185 uc->dma_dev = ud->dev;
2187 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {
2188 uc->use_dma_pool = true;
2190 if (uc->config.dir == DMA_MEM_TO_MEM) {
2191 uc->config.hdesc_size = cppi5_trdesc_calc_size(
2193 uc->config.pkt_mode = false;
2197 if (uc->use_dma_pool) {
2198 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2199 uc->config.hdesc_size,
2200 ud->desc_align,
2202 if (!uc->hdesc_pool) {
2203 dev_err(ud->ddev.dev,
2205 uc->use_dma_pool = false;
2206 ret = -ENOMEM;
2215 reinit_completion(&uc->teardown_completed);
2216 complete_all(&uc->teardown_completed);
2217 uc->state = UDMA_CHAN_IS_IDLE;
2219 switch (uc->config.dir) {
2221 /* Non synchronized - mem to mem type of transfer */
2222 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2223 uc->id);
2241 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2242 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2245 irq_ring = uc->tchan->tc_ring;
2246 irq_udma_idx = uc->tchan->id;
2251 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2252 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2253 uc->id);
2259 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2260 uc->config.dst_thread = uc->config.remote_thread_id;
2261 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2263 irq_ring = uc->tchan->tc_ring;
2264 irq_udma_idx = uc->tchan->id;
2269 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2270 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2271 uc->id);
2277 uc->config.src_thread = uc->config.remote_thread_id;
2278 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2281 irq_ring = uc->rflow->r_ring;
2282 irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id;
2288 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2289 __func__, uc->id, uc->config.dir);
2290 ret = -EINVAL;
2300 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2303 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2304 ret = -EBUSY;
2309 /* PSI-L pairing */
2310 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2312 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2313 uc->config.src_thread, uc->config.dst_thread);
2317 uc->psil_paired = true;
2319 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);
2320 if (uc->irq_num_ring <= 0) {
2321 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2323 ret = -EINVAL;
2327 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2328 IRQF_TRIGGER_HIGH, uc->name, uc);
2330 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2335 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {
2336 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);
2337 if (uc->irq_num_udma <= 0) {
2338 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",
2340 free_irq(uc->irq_num_ring, uc);
2341 ret = -EINVAL;
2345 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2346 uc->name, uc);
2348 dev_err(ud->dev, "chan%d: UDMA irq request failed\n",
2349 uc->id);
2350 free_irq(uc->irq_num_ring, uc);
2354 uc->irq_num_udma = 0;
2362 uc->irq_num_ring = 0;
2363 uc->irq_num_udma = 0;
2365 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2366 uc->psil_paired = false;
2373 if (uc->use_dma_pool) {
2374 dma_pool_destroy(uc->hdesc_pool);
2375 uc->use_dma_pool = false;
2384 struct udma_dev *ud = to_udma_dev(chan->device);
2385 const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2390 uc->config.pkt_mode = false;
2396 reinit_completion(&uc->teardown_completed);
2397 complete_all(&uc->teardown_completed);
2398 uc->state = UDMA_CHAN_IS_IDLE;
2400 switch (uc->config.dir) {
2402 /* Non synchronized - mem to mem type of transfer */
2403 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2404 uc->id);
2410 irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring;
2411 irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data;
2416 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2417 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2418 uc->id);
2422 uc->config.remote_thread_id = -1;
2426 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2427 uc->config.dst_thread = uc->config.remote_thread_id;
2428 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2430 irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring;
2431 irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data;
2436 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2437 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2438 uc->id);
2442 uc->config.remote_thread_id = -1;
2446 uc->config.src_thread = uc->config.remote_thread_id;
2447 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2450 irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring;
2451 irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data;
2457 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2458 __func__, uc->id, uc->config.dir);
2459 return -EINVAL;
2467 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2470 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2471 ret = -EBUSY;
2476 uc->dma_dev = dmaengine_get_dma_device(chan);
2477 if (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type) {
2478 uc->config.hdesc_size = cppi5_trdesc_calc_size(
2481 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2482 uc->config.hdesc_size,
2483 ud->desc_align,
2485 if (!uc->hdesc_pool) {
2486 dev_err(ud->ddev.dev,
2488 uc->use_dma_pool = false;
2489 ret = -ENOMEM;
2493 uc->use_dma_pool = true;
2494 } else if (uc->config.dir != DMA_MEM_TO_MEM) {
2495 /* PSI-L pairing */
2496 ret = navss_psil_pair(ud, uc->config.src_thread,
2497 uc->config.dst_thread);
2499 dev_err(ud->dev,
2500 "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2501 uc->config.src_thread, uc->config.dst_thread);
2505 uc->psil_paired = true;
2508 uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);
2509 if (uc->irq_num_ring <= 0) {
2510 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2512 ret = -EINVAL;
2516 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2517 IRQF_TRIGGER_HIGH, uc->name, uc);
2519 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2524 if (is_slave_direction(uc->config.dir)) {
2525 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);
2526 if (uc->irq_num_udma <= 0) {
2527 dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n",
2529 free_irq(uc->irq_num_ring, uc);
2530 ret = -EINVAL;
2534 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2535 uc->name, uc);
2537 dev_err(ud->dev, "chan%d: BCDMA irq request failed\n",
2538 uc->id);
2539 free_irq(uc->irq_num_ring, uc);
2543 uc->irq_num_udma = 0;
2548 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2553 uc->irq_num_ring = 0;
2554 uc->irq_num_udma = 0;
2556 if (uc->psil_paired)
2557 navss_psil_unpair(ud, uc->config.src_thread,
2558 uc->config.dst_thread);
2559 uc->psil_paired = false;
2567 if (uc->use_dma_pool) {
2568 dma_pool_destroy(uc->hdesc_pool);
2569 uc->use_dma_pool = false;
2577 struct k3_event_route_data *router_data = chan->route_data;
2581 if (!uc->bchan)
2582 return -EINVAL;
2584 if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2)
2585 return -EINVAL;
2587 trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset;
2588 trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1;
2590 return router_data->set_event(router_data->priv, trigger_event);
2596 struct udma_dev *ud = to_udma_dev(chan->device);
2597 const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2605 reinit_completion(&uc->teardown_completed);
2606 complete_all(&uc->teardown_completed);
2607 uc->state = UDMA_CHAN_IS_IDLE;
2609 switch (uc->config.dir) {
2611 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2612 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2613 uc->id);
2617 uc->config.remote_thread_id = -1;
2621 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2622 uc->config.dst_thread = uc->config.remote_thread_id;
2623 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2625 irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow;
2630 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2631 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2632 uc->id);
2636 uc->config.remote_thread_id = -1;
2640 uc->config.src_thread = uc->config.remote_thread_id;
2641 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2644 irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow;
2650 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2651 __func__, uc->id, uc->config.dir);
2652 return -EINVAL;
2660 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2663 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2664 ret = -EBUSY;
2669 uc->dma_dev = dmaengine_get_dma_device(chan);
2670 uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev,
2671 uc->config.hdesc_size, ud->desc_align,
2673 if (!uc->hdesc_pool) {
2674 dev_err(ud->ddev.dev,
2676 uc->use_dma_pool = false;
2677 ret = -ENOMEM;
2681 uc->use_dma_pool = true;
2683 /* PSI-L pairing */
2684 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2686 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2687 uc->config.src_thread, uc->config.dst_thread);
2691 uc->psil_paired = true;
2693 uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);
2694 if (uc->irq_num_ring <= 0) {
2695 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2697 ret = -EINVAL;
2701 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2702 IRQF_TRIGGER_HIGH, uc->name, uc);
2704 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2708 uc->irq_num_udma = 0;
2712 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2715 if (uc->tchan)
2716 dev_dbg(ud->dev,
2718 uc->id, uc->tchan->id, uc->tchan->tflow_id,
2719 uc->config.remote_thread_id);
2720 else if (uc->rchan)
2721 dev_dbg(ud->dev,
2723 uc->id, uc->rchan->id, uc->rflow->id,
2724 uc->config.remote_thread_id);
2728 uc->irq_num_ring = 0;
2730 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2731 uc->psil_paired = false;
2738 dma_pool_destroy(uc->hdesc_pool);
2739 uc->use_dma_pool = false;
2749 memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
2771 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
2776 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
2780 d->sglen = tr_count;
2782 d->hwdesc_count = 1;
2783 hwdesc = &d->hwdesc[0];
2786 if (uc->use_dma_pool) {
2787 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
2788 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
2790 &hwdesc->cppi5_desc_paddr);
2792 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
2794 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
2795 uc->ud->desc_align);
2796 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
2797 hwdesc->cppi5_desc_size,
2798 &hwdesc->cppi5_desc_paddr,
2802 if (!hwdesc->cppi5_desc_vaddr) {
2808 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
2810 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
2812 tr_desc = hwdesc->cppi5_desc_vaddr;
2814 if (uc->cyclic)
2818 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2820 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
2823 cppi5_desc_set_pktids(tr_desc, uc->id,
2831 * udma_get_tr_counters - calculate TR counters for a given length
2840 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
2844 * -EINVAL if the length can not be supported
2860 *tr0_cnt0 = SZ_64K - BIT(align_to);
2863 align_to--;
2866 return -EINVAL;
2904 d->sglen = sglen;
2906 if (uc->ud->match_data->type == DMA_TYPE_UDMA)
2909 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
2911 tr_req = d->hwdesc[0].tr_req_base;
2918 dev_err(uc->ud->dev, "size %u is not supported\n",
2950 d->residue += sg_dma_len(sgent);
2953 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
2980 dev_addr = uc->cfg.src_addr;
2981 dev_width = uc->cfg.src_addr_width;
2982 burst = uc->cfg.src_maxburst;
2983 port_window = uc->cfg.src_port_window_size;
2985 dev_addr = uc->cfg.dst_addr;
2986 dev_width = uc->cfg.dst_addr_width;
2987 burst = uc->cfg.dst_maxburst;
2988 port_window = uc->cfg.dst_port_window_size;
2990 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
2999 dev_err(uc->ud->dev,
3017 dev_err(uc->ud->dev,
3035 d->sglen = sglen;
3037 if (uc->ud->match_data->type == DMA_TYPE_UDMA) {
3041 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3045 tr_req = d->hwdesc[0].tr_req_base;
3054 dev_err(uc->ud->dev, "size %zu is not supported\n",
3065 uc->config.tr_trigger_type,
3075 tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3100 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3111 uc->config.tr_trigger_type,
3122 tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3145 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3150 d->residue += sg_len;
3153 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP);
3162 if (uc->config.ep_type != PSIL_EP_PDMA_XY)
3168 d->static_tr.elsize = 0;
3171 d->static_tr.elsize = 1;
3174 d->static_tr.elsize = 2;
3177 d->static_tr.elsize = 3;
3180 d->static_tr.elsize = 4;
3183 return -EINVAL;
3186 d->static_tr.elcnt = elcnt;
3188 if (uc->config.pkt_mode || !uc->cyclic) {
3197 if (uc->cyclic)
3198 d->static_tr.bstcnt = d->residue / d->sglen / div;
3200 d->static_tr.bstcnt = d->residue / div;
3201 } else if (uc->ud->match_data->type == DMA_TYPE_BCDMA &&
3202 uc->config.dir == DMA_DEV_TO_MEM &&
3203 uc->cyclic) {
3210 struct cppi5_tr_type1_t *tr_req = d->hwdesc[0].tr_req_base;
3212 d->static_tr.bstcnt =
3213 (tr_req->icnt0 * tr_req->icnt1) / dev_width;
3215 d->static_tr.bstcnt = 0;
3218 if (uc->config.dir == DMA_DEV_TO_MEM &&
3219 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
3220 return -EINVAL;
3241 d->sglen = sglen;
3242 d->hwdesc_count = sglen;
3245 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3247 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3249 if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3252 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3255 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3260 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3262 &hwdesc->cppi5_desc_paddr);
3263 if (!hwdesc->cppi5_desc_vaddr) {
3264 dev_err(uc->ud->dev,
3272 d->residue += sg_len;
3273 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3274 desc = hwdesc->cppi5_desc_vaddr;
3279 cppi5_desc_set_pktids(&desc->hdr, uc->id,
3281 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
3284 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
3294 hwdesc->cppi5_desc_paddr | asel);
3296 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||
3301 if (d->residue >= SZ_4M) {
3302 dev_err(uc->ud->dev,
3303 "%s: Transfer size %u is over the supported 4M range\n",
3304 __func__, d->residue);
3310 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3311 cppi5_hdesc_set_pktlen(h_desc, d->residue);
3320 struct udma_chan *uc = to_udma_chan(desc->chan);
3325 if (!uc->config.pkt_mode || !uc->config.metadata_size)
3326 return -ENOTSUPP;
3328 if (!data || len > uc->config.metadata_size)
3329 return -EINVAL;
3331 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3332 return -EINVAL;
3334 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3335 if (d->dir == DMA_MEM_TO_DEV)
3336 memcpy(h_desc->epib, data, len);
3338 if (uc->config.needs_epib)
3339 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3341 d->metadata = data;
3342 d->metadata_size = len;
3343 if (uc->config.needs_epib)
3356 struct udma_chan *uc = to_udma_chan(desc->chan);
3359 if (!uc->config.pkt_mode || !uc->config.metadata_size)
3360 return ERR_PTR(-ENOTSUPP);
3362 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3364 *max_len = uc->config.metadata_size;
3366 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
3370 return h_desc->epib;
3377 struct udma_chan *uc = to_udma_chan(desc->chan);
3382 if (!uc->config.pkt_mode || !uc->config.metadata_size)
3383 return -ENOTSUPP;
3385 if (payload_len > uc->config.metadata_size)
3386 return -EINVAL;
3388 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3389 return -EINVAL;
3391 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3393 if (uc->config.needs_epib) {
3394 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3420 if (dir != uc->config.dir &&
3421 (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {
3422 dev_err(chan->device->dev,
3424 __func__, uc->id,
3425 dmaengine_get_direction_text(uc->config.dir),
3431 dev_width = uc->cfg.src_addr_width;
3432 burst = uc->cfg.src_maxburst;
3434 dev_width = uc->cfg.dst_addr_width;
3435 burst = uc->cfg.dst_maxburst;
3437 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
3444 uc->config.tx_flags = tx_flags;
3446 if (uc->config.pkt_mode)
3449 else if (is_slave_direction(uc->config.dir))
3459 d->dir = dir;
3460 d->desc_idx = 0;
3461 d->tr_idx = 0;
3465 dev_err(uc->ud->dev,
3467 __func__, uc->ud->match_data->statictr_z_mask,
3468 d->static_tr.bstcnt);
3475 if (uc->config.metadata_size)
3476 d->vd.tx.metadata_ops = &metadata_ops;
3478 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3498 dev_err(uc->ud->dev, "size %zu is not supported\n",
3509 tr_req = d->hwdesc[0].tr_req_base;
3510 if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3514 ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);
3517 * For BCDMA <-> PDMA transfers, the EOP flag needs to be set on the
3520 * of TX, and to avoid short-packet error in case of RX.
3525 if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
3526 uc->ud->match_data->type == DMA_TYPE_BCDMA) {
3578 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
3588 d->hwdesc_count = periods;
3590 /* TODO: re-check this... */
3592 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3594 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3596 if (uc->ud->match_data->type != DMA_TYPE_UDMA)
3597 buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3600 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3604 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3606 &hwdesc->cppi5_desc_paddr);
3607 if (!hwdesc->cppi5_desc_vaddr) {
3608 dev_err(uc->ud->dev,
3616 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3617 h_desc = hwdesc->cppi5_desc_vaddr;
3623 cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
3625 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
3646 if (dir != uc->config.dir) {
3647 dev_err(chan->device->dev,
3649 __func__, uc->id,
3650 dmaengine_get_direction_text(uc->config.dir),
3655 uc->cyclic = true;
3658 dev_width = uc->cfg.src_addr_width;
3659 burst = uc->cfg.src_maxburst;
3661 dev_width = uc->cfg.dst_addr_width;
3662 burst = uc->cfg.dst_maxburst;
3664 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
3671 if (uc->config.pkt_mode)
3681 d->sglen = buf_len / period_len;
3683 d->dir = dir;
3684 d->residue = buf_len;
3688 dev_err(uc->ud->dev,
3690 __func__, uc->ud->match_data->statictr_z_mask,
3691 d->static_tr.bstcnt);
3698 if (uc->config.metadata_size)
3699 d->vd.tx.metadata_ops = &metadata_ops;
3701 return vchan_tx_prep(&uc->vc, &d->vd, flags);
3716 if (uc->config.dir != DMA_MEM_TO_MEM) {
3717 dev_err(chan->device->dev,
3719 __func__, uc->id,
3720 dmaengine_get_direction_text(uc->config.dir),
3728 dev_err(uc->ud->dev, "size %zu is not supported\n",
3737 d->dir = DMA_MEM_TO_MEM;
3738 d->desc_idx = 0;
3739 d->tr_idx = 0;
3740 d->residue = len;
3742 if (uc->ud->match_data->type != DMA_TYPE_UDMA) {
3743 src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3744 dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3749 tr_req = d->hwdesc[0].tr_req_base;
3787 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP);
3789 if (uc->config.metadata_size)
3790 d->vd.tx.metadata_ops = &metadata_ops;
3792 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3800 spin_lock_irqsave(&uc->vc.lock, flags);
3803 if (vchan_issue_pending(&uc->vc) && !uc->desc) {
3809 if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
3814 spin_unlock_irqrestore(&uc->vc.lock, flags);
3825 spin_lock_irqsave(&uc->vc.lock, flags);
3838 if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
3841 u32 residue = uc->desc->residue;
3844 if (uc->desc->dir == DMA_MEM_TO_DEV) {
3847 if (uc->config.ep_type != PSIL_EP_NATIVE) {
3852 delay = bcnt - peer_bcnt;
3854 } else if (uc->desc->dir == DMA_DEV_TO_MEM) {
3857 if (uc->config.ep_type != PSIL_EP_NATIVE) {
3862 delay = peer_bcnt - bcnt;
3868 if (bcnt && !(bcnt % uc->desc->residue))
3871 residue -= bcnt % uc->desc->residue;
3873 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {
3886 spin_unlock_irqrestore(&uc->vc.lock, flags);
3895 switch (uc->config.dir) {
3912 return -EINVAL;
3923 switch (uc->config.dir) {
3938 return -EINVAL;
3950 spin_lock_irqsave(&uc->vc.lock, flags);
3955 if (uc->desc) {
3956 uc->terminated_desc = uc->desc;
3957 uc->desc = NULL;
3958 uc->terminated_desc->terminated = true;
3959 cancel_delayed_work(&uc->tx_drain.work);
3962 uc->paused = false;
3964 vchan_get_all_descriptors(&uc->vc, &head);
3965 spin_unlock_irqrestore(&uc->vc.lock, flags);
3966 vchan_dma_desc_free_list(&uc->vc, &head);
3976 vchan_synchronize(&uc->vc);
3978 if (uc->state == UDMA_CHAN_IS_TERMINATING) {
3979 timeout = wait_for_completion_timeout(&uc->teardown_completed,
3982 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
3983 uc->id);
3991 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
3993 cancel_delayed_work_sync(&uc->tx_drain.work);
4001 struct udma_chan *uc = to_udma_chan(&vc->chan);
4008 d = to_udma_desc(&vd->tx);
4010 if (d->metadata_size)
4014 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
4019 result->residue = d->residue -
4021 if (result->residue)
4022 result->result = DMA_TRANS_ABORTED;
4024 result->result = DMA_TRANS_NOERROR;
4026 result->residue = 0;
4028 status = d->hwdesc[0].tr_resp_base->status;
4030 result->result = DMA_TRANS_ABORTED;
4032 result->result = DMA_TRANS_NOERROR;
4048 spin_lock_irq(&vc->lock);
4049 list_splice_tail_init(&vc->desc_completed, &head);
4050 vd = vc->cyclic;
4052 vc->cyclic = NULL;
4053 dmaengine_desc_get_callback(&vd->tx, &cb);
4057 spin_unlock_irq(&vc->lock);
4065 dmaengine_desc_get_callback(&vd->tx, &cb);
4067 list_del(&vd->node);
4079 struct udma_dev *ud = to_udma_dev(chan->device);
4082 if (uc->terminated_desc) {
4087 cancel_delayed_work_sync(&uc->tx_drain.work);
4089 if (uc->irq_num_ring > 0) {
4090 free_irq(uc->irq_num_ring, uc);
4092 uc->irq_num_ring = 0;
4094 if (uc->irq_num_udma > 0) {
4095 free_irq(uc->irq_num_udma, uc);
4097 uc->irq_num_udma = 0;
4100 /* Release PSI-L pairing */
4101 if (uc->psil_paired) {
4102 navss_psil_unpair(ud, uc->config.src_thread,
4103 uc->config.dst_thread);
4104 uc->psil_paired = false;
4107 vchan_free_chan_resources(&uc->vc);
4108 tasklet_kill(&uc->vc.task);
4115 if (uc->use_dma_pool) {
4116 dma_pool_destroy(uc->hdesc_pool);
4117 uc->use_dma_pool = false;
4140 if (chan->device->dev->driver != &udma_driver.driver &&
4141 chan->device->dev->driver != &bcdma_driver.driver &&
4142 chan->device->dev->driver != &pktdma_driver.driver)
4146 ucc = &uc->config;
4147 ud = uc->ud;
4150 if (filter_param->atype > 2) {
4151 dev_err(ud->dev, "Invalid channel atype: %u\n",
4152 filter_param->atype);
4156 if (filter_param->asel > 15) {
4157 dev_err(ud->dev, "Invalid channel asel: %u\n",
4158 filter_param->asel);
4162 ucc->remote_thread_id = filter_param->remote_thread_id;
4163 ucc->atype = filter_param->atype;
4164 ucc->asel = filter_param->asel;
4165 ucc->tr_trigger_type = filter_param->tr_trigger_type;
4167 if (ucc->tr_trigger_type) {
4168 ucc->dir = DMA_MEM_TO_MEM;
4170 } else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) {
4171 ucc->dir = DMA_MEM_TO_DEV;
4173 ucc->dir = DMA_DEV_TO_MEM;
4176 ep_config = psil_get_ep_config(ucc->remote_thread_id);
4178 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
4179 ucc->remote_thread_id);
4180 ucc->dir = DMA_MEM_TO_MEM;
4181 ucc->remote_thread_id = -1;
4182 ucc->atype = 0;
4183 ucc->asel = 0;
4187 if (ud->match_data->type == DMA_TYPE_BCDMA &&
4188 ep_config->pkt_mode) {
4189 dev_err(ud->dev,
4190 "Only TR mode is supported (psi-l thread 0x%04x)\n",
4191 ucc->remote_thread_id);
4192 ucc->dir = DMA_MEM_TO_MEM;
4193 ucc->remote_thread_id = -1;
4194 ucc->atype = 0;
4195 ucc->asel = 0;
4199 ucc->pkt_mode = ep_config->pkt_mode;
4200 ucc->channel_tpl = ep_config->channel_tpl;
4201 ucc->notdpkt = ep_config->notdpkt;
4202 ucc->ep_type = ep_config->ep_type;
4204 if (ud->match_data->type == DMA_TYPE_PKTDMA &&
4205 ep_config->mapped_channel_id >= 0) {
4206 ucc->mapped_channel_id = ep_config->mapped_channel_id;
4207 ucc->default_flow_id = ep_config->default_flow_id;
4209 ucc->mapped_channel_id = -1;
4210 ucc->default_flow_id = -1;
4213 if (ucc->ep_type != PSIL_EP_NATIVE) {
4214 const struct udma_match_data *match_data = ud->match_data;
4216 if (match_data->flags & UDMA_FLAG_PDMA_ACC32)
4217 ucc->enable_acc32 = ep_config->pdma_acc32;
4218 if (match_data->flags & UDMA_FLAG_PDMA_BURST)
4219 ucc->enable_burst = ep_config->pdma_burst;
4222 ucc->needs_epib = ep_config->needs_epib;
4223 ucc->psd_size = ep_config->psd_size;
4224 ucc->metadata_size =
4225 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
4226 ucc->psd_size;
4228 if (ucc->pkt_mode)
4229 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
4230 ucc->metadata_size, ud->desc_align);
4232 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
4233 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));
4238 dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id,
4239 ucc->tr_trigger_type);
4248 struct udma_dev *ud = ofdma->of_dma_data;
4249 dma_cap_mask_t mask = ud->ddev.cap_mask;
4253 if (ud->match_data->type == DMA_TYPE_BCDMA) {
4254 if (dma_spec->args_count != 3)
4257 filter_param.tr_trigger_type = dma_spec->args[0];
4258 filter_param.remote_thread_id = dma_spec->args[1];
4259 filter_param.asel = dma_spec->args[2];
4262 if (dma_spec->args_count != 1 && dma_spec->args_count != 2)
4265 filter_param.remote_thread_id = dma_spec->args[0];
4267 if (dma_spec->args_count == 2) {
4268 if (ud->match_data->type == DMA_TYPE_UDMA) {
4269 filter_param.atype = dma_spec->args[1];
4273 filter_param.asel = dma_spec->args[1];
4282 ofdma->of_node);
4284 dev_err(ud->dev, "get channel fail in %s.\n", __func__);
4285 return ERR_PTR(-EINVAL);
4421 .compatible = "ti,am654-navss-main-udmap",
4425 .compatible = "ti,am654-navss-mcu-udmap",
4428 .compatible = "ti,j721e-navss-main-udmap",
4431 .compatible = "ti,j721e-navss-mcu-udmap",
4435 .compatible = "ti,am64-dmss-bcdma",
4439 .compatible = "ti,am64-dmss-pktdma",
4443 .compatible = "ti,am62a-dmss-bcdma-csirx",
4447 .compatible = "ti,j721s2-dmss-bcdma-csi",
4451 .compatible = "ti,j722s-dmss-bcdma-csi",
4509 ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]);
4510 if (IS_ERR(ud->mmrs[MMR_GCFG]))
4511 return PTR_ERR(ud->mmrs[MMR_GCFG]);
4513 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
4514 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4516 switch (ud->match_data->type) {
4518 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4519 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4520 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);
4521 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4524 ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2) +
4527 ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2);
4528 ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2);
4529 ud->rflow_cnt = ud->rchan_cnt;
4532 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4533 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4534 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4535 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4536 ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4);
4539 return -EINVAL;
4543 if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
4545 if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
4547 if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
4550 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);
4551 if (IS_ERR(ud->mmrs[i]))
4552 return PTR_ERR(ud->mmrs[i]);
4562 bitmap_clear(map, rm_desc->start, rm_desc->num);
4563 bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec);
4564 dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name,
4565 rm_desc->start, rm_desc->num, rm_desc->start_sec,
4566 rm_desc->num_sec);
4570 [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
4571 [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
4572 [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
4573 [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
4574 [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
4580 struct device *dev = ud->dev;
4582 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4586 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4587 if (of_device_is_compatible(dev->of_node,
4588 "ti,am654-navss-main-udmap")) {
4589 ud->tchan_tpl.levels = 2;
4590 ud->tchan_tpl.start_idx[0] = 8;
4591 } else if (of_device_is_compatible(dev->of_node,
4592 "ti,am654-navss-mcu-udmap")) {
4593 ud->tchan_tpl.levels = 2;
4594 ud->tchan_tpl.start_idx[0] = 2;
4596 ud->tchan_tpl.levels = 3;
4597 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4598 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4600 ud->tchan_tpl.levels = 2;
4601 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4603 ud->tchan_tpl.levels = 1;
4606 ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4607 ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4608 ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4610 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4612 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4614 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4616 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4618 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
4621 ud->rflow_gp_map_allocated = devm_kcalloc(dev,
4622 BITS_TO_LONGS(ud->rflow_cnt),
4625 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4628 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4631 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
4632 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
4633 !ud->rflows || !ud->rflow_in_use)
4634 return -ENOMEM;
4641 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
4643 /* by default no GP rflows are assigned to Linux */
4644 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
4651 tisci_rm->rm_ranges[i] =
4652 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4653 tisci_rm->tisci_dev_id,
4658 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4660 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4663 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4664 for (i = 0; i < rm_res->sets; i++)
4665 udma_mark_resource_ranges(ud, ud->tchan_map,
4666 &rm_res->desc[i], "tchan");
4667 irq_res.sets = rm_res->sets;
4671 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4673 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4676 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4677 for (i = 0; i < rm_res->sets; i++)
4678 udma_mark_resource_ranges(ud, ud->rchan_map,
4679 &rm_res->desc[i], "rchan");
4680 irq_res.sets += rm_res->sets;
4685 return -ENOMEM;
4686 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4689 irq_res.desc[0].num = ud->tchan_cnt;
4692 for (i = 0; i < rm_res->sets; i++) {
4693 irq_res.desc[i].start = rm_res->desc[i].start;
4694 irq_res.desc[i].num = rm_res->desc[i].num;
4695 irq_res.desc[i].start_sec = rm_res->desc[i].start_sec;
4696 irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;
4699 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4702 irq_res.desc[i].num = ud->rchan_cnt;
4704 for (j = 0; j < rm_res->sets; j++, i++) {
4705 if (rm_res->desc[j].num) {
4706 irq_res.desc[i].start = rm_res->desc[j].start +
4707 ud->soc_data->oes.udma_rchan;
4708 irq_res.desc[i].num = rm_res->desc[j].num;
4710 if (rm_res->desc[j].num_sec) {
4711 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
4712 ud->soc_data->oes.udma_rchan;
4713 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
4717 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4720 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4724 /* GP rflow ranges */
4725 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4727 /* all gp flows are assigned exclusively to Linux */
4728 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
4729 ud->rflow_cnt - ud->rchan_cnt);
4731 for (i = 0; i < rm_res->sets; i++)
4732 udma_mark_resource_ranges(ud, ud->rflow_gp_map,
4733 &rm_res->desc[i], "gp-rflow");
4742 struct device *dev = ud->dev;
4744 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4745 const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4749 cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4751 ud->bchan_tpl.levels = 3;
4752 ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap);
4753 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4755 ud->bchan_tpl.levels = 2;
4756 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4758 ud->bchan_tpl.levels = 1;
4761 cap = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4763 ud->rchan_tpl.levels = 3;
4764 ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap);
4765 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4767 ud->rchan_tpl.levels = 2;
4768 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4770 ud->rchan_tpl.levels = 1;
4774 ud->tchan_tpl.levels = 3;
4775 ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap);
4776 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4778 ud->tchan_tpl.levels = 2;
4779 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4781 ud->tchan_tpl.levels = 1;
4784 ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
4786 ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
4788 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4790 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4792 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4794 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4797 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt),
4800 ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
4803 if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
4804 !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans ||
4805 !ud->rflows)
4806 return -ENOMEM;
4812 if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0)
4814 if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0)
4816 if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0)
4819 tisci_rm->rm_ranges[i] =
4820 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4821 tisci_rm->tisci_dev_id,
4828 if (ud->bchan_cnt) {
4829 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4831 bitmap_zero(ud->bchan_map, ud->bchan_cnt);
4834 bitmap_fill(ud->bchan_map, ud->bchan_cnt);
4835 for (i = 0; i < rm_res->sets; i++)
4836 udma_mark_resource_ranges(ud, ud->bchan_map,
4837 &rm_res->desc[i],
4839 irq_res.sets += rm_res->sets;
4844 if (ud->tchan_cnt) {
4845 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4847 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4850 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4851 for (i = 0; i < rm_res->sets; i++)
4852 udma_mark_resource_ranges(ud, ud->tchan_map,
4853 &rm_res->desc[i],
4855 irq_res.sets += rm_res->sets * 2;
4860 if (ud->rchan_cnt) {
4861 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4863 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4866 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4867 for (i = 0; i < rm_res->sets; i++)
4868 udma_mark_resource_ranges(ud, ud->rchan_map,
4869 &rm_res->desc[i],
4871 irq_res.sets += rm_res->sets * 2;
4877 return -ENOMEM;
4878 if (ud->bchan_cnt) {
4879 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4881 irq_res.desc[0].start = oes->bcdma_bchan_ring;
4882 irq_res.desc[0].num = ud->bchan_cnt;
4885 for (i = 0; i < rm_res->sets; i++) {
4886 irq_res.desc[i].start = rm_res->desc[i].start +
4887 oes->bcdma_bchan_ring;
4888 irq_res.desc[i].num = rm_res->desc[i].num;
4890 if (rm_res->desc[i].num_sec) {
4891 irq_res.desc[i].start_sec = rm_res->desc[i].start_sec +
4892 oes->bcdma_bchan_ring;
4893 irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;
4901 if (ud->tchan_cnt) {
4902 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4904 irq_res.desc[i].start = oes->bcdma_tchan_data;
4905 irq_res.desc[i].num = ud->tchan_cnt;
4906 irq_res.desc[i + 1].start = oes->bcdma_tchan_ring;
4907 irq_res.desc[i + 1].num = ud->tchan_cnt;
4910 for (j = 0; j < rm_res->sets; j++, i += 2) {
4911 irq_res.desc[i].start = rm_res->desc[j].start +
4912 oes->bcdma_tchan_data;
4913 irq_res.desc[i].num = rm_res->desc[j].num;
4915 irq_res.desc[i + 1].start = rm_res->desc[j].start +
4916 oes->bcdma_tchan_ring;
4917 irq_res.desc[i + 1].num = rm_res->desc[j].num;
4919 if (rm_res->desc[j].num_sec) {
4920 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
4921 oes->bcdma_tchan_data;
4922 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
4923 irq_res.desc[i + 1].start_sec = rm_res->desc[j].start_sec +
4924 oes->bcdma_tchan_ring;
4925 irq_res.desc[i + 1].num_sec = rm_res->desc[j].num_sec;
4930 if (ud->rchan_cnt) {
4931 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4933 irq_res.desc[i].start = oes->bcdma_rchan_data;
4934 irq_res.desc[i].num = ud->rchan_cnt;
4935 irq_res.desc[i + 1].start = oes->bcdma_rchan_ring;
4936 irq_res.desc[i + 1].num = ud->rchan_cnt;
4939 for (j = 0; j < rm_res->sets; j++, i += 2) {
4940 irq_res.desc[i].start = rm_res->desc[j].start +
4941 oes->bcdma_rchan_data;
4942 irq_res.desc[i].num = rm_res->desc[j].num;
4944 irq_res.desc[i + 1].start = rm_res->desc[j].start +
4945 oes->bcdma_rchan_ring;
4946 irq_res.desc[i + 1].num = rm_res->desc[j].num;
4948 if (rm_res->desc[j].num_sec) {
4949 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
4950 oes->bcdma_rchan_data;
4951 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
4952 irq_res.desc[i + 1].start_sec = rm_res->desc[j].start_sec +
4953 oes->bcdma_rchan_ring;
4954 irq_res.desc[i + 1].num_sec = rm_res->desc[j].num_sec;
4960 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4963 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4973 struct device *dev = ud->dev;
4975 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4976 const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4980 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4982 ud->tchan_tpl.levels = 3;
4983 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4984 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4986 ud->tchan_tpl.levels = 2;
4987 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4989 ud->tchan_tpl.levels = 1;
4992 ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4993 ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4994 ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4996 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4998 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
5000 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
5002 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
5004 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
5007 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
5009 ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
5012 if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
5013 !ud->rchans || !ud->rflows || !ud->rflow_in_use)
5014 return -ENOMEM;
5021 tisci_rm->rm_ranges[i] =
5022 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
5023 tisci_rm->tisci_dev_id,
5028 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
5030 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
5032 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
5033 for (i = 0; i < rm_res->sets; i++)
5034 udma_mark_resource_ranges(ud, ud->tchan_map,
5035 &rm_res->desc[i], "tchan");
5039 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
5041 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
5043 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
5044 for (i = 0; i < rm_res->sets; i++)
5045 udma_mark_resource_ranges(ud, ud->rchan_map,
5046 &rm_res->desc[i], "rchan");
5050 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
5053 bitmap_zero(ud->rflow_in_use, ud->rflow_cnt);
5056 bitmap_fill(ud->rflow_in_use, ud->rflow_cnt);
5057 for (i = 0; i < rm_res->sets; i++)
5058 udma_mark_resource_ranges(ud, ud->rflow_in_use,
5059 &rm_res->desc[i], "rflow");
5060 irq_res.sets = rm_res->sets;
5064 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
5067 bitmap_zero(ud->tflow_map, ud->tflow_cnt);
5070 bitmap_fill(ud->tflow_map, ud->tflow_cnt);
5071 for (i = 0; i < rm_res->sets; i++)
5072 udma_mark_resource_ranges(ud, ud->tflow_map,
5073 &rm_res->desc[i], "tflow");
5074 irq_res.sets += rm_res->sets;
5079 return -ENOMEM;
5080 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
5082 irq_res.desc[0].start = oes->pktdma_tchan_flow;
5083 irq_res.desc[0].num = ud->tflow_cnt;
5086 for (i = 0; i < rm_res->sets; i++) {
5087 irq_res.desc[i].start = rm_res->desc[i].start +
5088 oes->pktdma_tchan_flow;
5089 irq_res.desc[i].num = rm_res->desc[i].num;
5091 if (rm_res->desc[i].num_sec) {
5092 irq_res.desc[i].start_sec = rm_res->desc[i].start_sec +
5093 oes->pktdma_tchan_flow;
5094 irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;
5098 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
5100 irq_res.desc[i].start = oes->pktdma_rchan_flow;
5101 irq_res.desc[i].num = ud->rflow_cnt;
5103 for (j = 0; j < rm_res->sets; j++, i++) {
5104 irq_res.desc[i].start = rm_res->desc[j].start +
5105 oes->pktdma_rchan_flow;
5106 irq_res.desc[i].num = rm_res->desc[j].num;
5108 if (rm_res->desc[j].num_sec) {
5109 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
5110 oes->pktdma_rchan_flow;
5111 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
5115 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
5118 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
5127 struct device *dev = ud->dev;
5130 switch (ud->match_data->type) {
5141 return -EINVAL;
5147 ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
5148 if (ud->bchan_cnt)
5149 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
5150 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
5151 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
5153 return -ENODEV;
5155 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
5157 if (!ud->channels)
5158 return -ENOMEM;
5160 switch (ud->match_data->type) {
5163 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
5165 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5166 ud->tchan_cnt),
5167 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5168 ud->rchan_cnt),
5169 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,
5170 ud->rflow_cnt));
5176 ud->bchan_cnt - bitmap_weight(ud->bchan_map,
5177 ud->bchan_cnt),
5178 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5179 ud->tchan_cnt),
5180 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5181 ud->rchan_cnt));
5187 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5188 ud->tchan_cnt),
5189 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5190 ud->rchan_cnt));
5201 struct udma_rx_flush *rx_flush = &ud->rx_flush;
5205 struct device *dev = ud->dev;
5210 rx_flush->buffer_size = SZ_1K;
5211 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
5213 if (!rx_flush->buffer_vaddr)
5214 return -ENOMEM;
5216 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
5217 rx_flush->buffer_size,
5219 if (dma_mapping_error(dev, rx_flush->buffer_paddr))
5220 return -ENOMEM;
5223 hwdesc = &rx_flush->hwdescs[0];
5225 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
5226 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
5227 ud->desc_align);
5229 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5231 if (!hwdesc->cppi5_desc_vaddr)
5232 return -ENOMEM;
5234 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5235 hwdesc->cppi5_desc_size,
5237 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5238 return -ENOMEM;
5241 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
5243 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
5245 tr_desc = hwdesc->cppi5_desc_vaddr;
5250 tr_req = hwdesc->tr_req_base;
5251 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
5253 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
5255 tr_req->addr = rx_flush->buffer_paddr;
5256 tr_req->icnt0 = rx_flush->buffer_size;
5257 tr_req->icnt1 = 1;
5259 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5260 hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5263 hwdesc = &rx_flush->hwdescs[1];
5264 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
5267 ud->desc_align);
5269 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5271 if (!hwdesc->cppi5_desc_vaddr)
5272 return -ENOMEM;
5274 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5275 hwdesc->cppi5_desc_size,
5277 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5278 return -ENOMEM;
5280 desc = hwdesc->cppi5_desc_vaddr;
5282 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5283 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
5286 rx_flush->buffer_paddr, rx_flush->buffer_size,
5287 rx_flush->buffer_paddr, rx_flush->buffer_size);
5289 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5290 hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5299 struct udma_chan_config *ucc = &uc->config;
5301 seq_printf(s, " %-13s| %s", dma_chan_name(chan),
5302 chan->dbg_client_name ?: "in-use");
5303 if (ucc->tr_trigger_type)
5307 dmaengine_get_direction_text(uc->config.dir));
5309 switch (uc->config.dir) {
5311 if (uc->ud->match_data->type == DMA_TYPE_BCDMA) {
5312 seq_printf(s, "bchan%d)\n", uc->bchan->id);
5316 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,
5317 ucc->src_thread, ucc->dst_thread);
5320 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,
5321 ucc->src_thread, ucc->dst_thread);
5322 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5323 seq_printf(s, "rflow%d, ", uc->rflow->id);
5326 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,
5327 ucc->src_thread, ucc->dst_thread);
5328 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5329 seq_printf(s, "tflow%d, ", uc->tchan->tflow_id);
5336 if (ucc->ep_type == PSIL_EP_NATIVE) {
5337 seq_printf(s, "PSI-L Native");
5338 if (ucc->metadata_size) {
5339 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");
5340 if (ucc->psd_size)
5341 seq_printf(s, " PSDsize:%u", ucc->psd_size);
5346 if (ucc->enable_acc32 || ucc->enable_burst)
5348 ucc->enable_acc32 ? " ACC32" : "",
5349 ucc->enable_burst ? " BURST" : "");
5352 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");
5360 list_for_each_entry(chan, &dma_dev->channels, device_node) {
5361 if (chan->client_count)
5369 const struct udma_match_data *match_data = ud->match_data;
5372 if (!match_data->enable_memcpy_support)
5376 if (ud->bchan_cnt)
5377 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0);
5378 else if (ud->tchan_cnt)
5379 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0);
5383 switch (match_data->burst_size[tpl]) {
5403 struct device_node *navss_node = pdev->dev.parent->of_node;
5405 struct device *dev = &pdev->dev;
5417 return -ENOMEM;
5419 match = of_match_node(udma_of_match, dev->of_node);
5422 return -ENODEV;
5424 ud->match_data = match->data;
5426 ud->soc_data = ud->match_data->soc_data;
5427 if (!ud->soc_data) {
5431 return -ENODEV;
5433 ud->soc_data = soc->data;
5440 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
5441 if (IS_ERR(ud->tisci_rm.tisci))
5442 return PTR_ERR(ud->tisci_rm.tisci);
5444 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
5445 &ud->tisci_rm.tisci_dev_id);
5447 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
5450 pdev->id = ud->tisci_rm.tisci_dev_id;
5452 ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
5453 &ud->tisci_rm.tisci_navss_dev_id);
5455 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
5459 if (ud->match_data->type == DMA_TYPE_UDMA) {
5460 ret = of_property_read_u32(dev->of_node, "ti,udma-atype",
5461 &ud->atype);
5462 if (!ret && ud->atype > 2) {
5463 dev_err(dev, "Invalid atype: %u\n", ud->atype);
5464 return -EINVAL;
5467 ret = of_property_read_u32(dev->of_node, "ti,asel",
5468 &ud->asel);
5469 if (!ret && ud->asel > 15) {
5470 dev_err(dev, "Invalid asel: %u\n", ud->asel);
5471 return -EINVAL;
5475 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
5476 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
5478 if (ud->match_data->type == DMA_TYPE_UDMA) {
5479 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
5483 ring_init_data.tisci = ud->tisci_rm.tisci;
5484 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
5485 if (ud->match_data->type == DMA_TYPE_BCDMA) {
5486 ring_init_data.num_rings = ud->bchan_cnt +
5487 ud->tchan_cnt +
5488 ud->rchan_cnt;
5490 ring_init_data.num_rings = ud->rflow_cnt +
5491 ud->tflow_cnt;
5494 ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data);
5497 if (IS_ERR(ud->ringacc))
5498 return PTR_ERR(ud->ringacc);
5500 dev->msi.domain = of_msi_get_domain(dev, dev->of_node,
5502 if (!dev->msi.domain) {
5503 return -EPROBE_DEFER;
5506 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
5508 if (ud->match_data->type != DMA_TYPE_PKTDMA) {
5509 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
5510 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
5513 ud->ddev.device_config = udma_slave_config;
5514 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
5515 ud->ddev.device_issue_pending = udma_issue_pending;
5516 ud->ddev.device_tx_status = udma_tx_status;
5517 ud->ddev.device_pause = udma_pause;
5518 ud->ddev.device_resume = udma_resume;
5519 ud->ddev.device_terminate_all = udma_terminate_all;
5520 ud->ddev.device_synchronize = udma_synchronize;
5522 ud->ddev.dbg_summary_show = udma_dbg_summary_show;
5525 switch (ud->match_data->type) {
5527 ud->ddev.device_alloc_chan_resources =
5531 ud->ddev.device_alloc_chan_resources =
5533 ud->ddev.device_router_config = bcdma_router_config;
5536 ud->ddev.device_alloc_chan_resources =
5540 return -EINVAL;
5542 ud->ddev.device_free_chan_resources = udma_free_chan_resources;
5544 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
5545 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
5546 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
5547 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
5548 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
5550 if (ud->match_data->enable_memcpy_support &&
5551 !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) {
5552 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
5553 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
5554 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
5557 ud->ddev.dev = dev;
5558 ud->dev = dev;
5559 ud->psil_base = ud->match_data->psil_base;
5561 INIT_LIST_HEAD(&ud->ddev.channels);
5562 INIT_LIST_HEAD(&ud->desc_to_purge);
5568 spin_lock_init(&ud->lock);
5569 INIT_WORK(&ud->purge_work, udma_purge_desc_work);
5571 ud->desc_align = 64;
5572 if (ud->desc_align < dma_get_cache_alignment())
5573 ud->desc_align = dma_get_cache_alignment();
5579 for (i = 0; i < ud->bchan_cnt; i++) {
5580 struct udma_bchan *bchan = &ud->bchans[i];
5582 bchan->id = i;
5583 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
5586 for (i = 0; i < ud->tchan_cnt; i++) {
5587 struct udma_tchan *tchan = &ud->tchans[i];
5589 tchan->id = i;
5590 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
5593 for (i = 0; i < ud->rchan_cnt; i++) {
5594 struct udma_rchan *rchan = &ud->rchans[i];
5596 rchan->id = i;
5597 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
5600 for (i = 0; i < ud->rflow_cnt; i++) {
5601 struct udma_rflow *rflow = &ud->rflows[i];
5603 rflow->id = i;
5607 struct udma_chan *uc = &ud->channels[i];
5609 uc->ud = ud;
5610 uc->vc.desc_free = udma_desc_free;
5611 uc->id = i;
5612 uc->bchan = NULL;
5613 uc->tchan = NULL;
5614 uc->rchan = NULL;
5615 uc->config.remote_thread_id = -1;
5616 uc->config.mapped_channel_id = -1;
5617 uc->config.default_flow_id = -1;
5618 uc->config.dir = DMA_MEM_TO_MEM;
5619 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
5622 vchan_init(&uc->vc, &ud->ddev);
5624 tasklet_setup(&uc->vc.task, udma_vchan_complete);
5625 init_completion(&uc->teardown_completed);
5626 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
5630 ud->ddev.copy_align = udma_get_copy_align(ud);
5632 ret = dma_async_device_register(&ud->ddev);
5640 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
5643 dma_async_device_unregister(&ud->ddev);
5652 struct dma_device *dma_dev = &ud->ddev;
5656 list_for_each_entry(chan, &dma_dev->channels, device_node) {
5657 if (chan->client_count) {
5660 memcpy(&uc->backup_config, &uc->config,
5664 ud->ddev.device_free_chan_resources(chan);
5674 struct dma_device *dma_dev = &ud->ddev;
5679 list_for_each_entry(chan, &dma_dev->channels, device_node) {
5680 if (chan->client_count) {
5683 memcpy(&uc->config, &uc->backup_config,
5687 ret = ud->ddev.device_alloc_chan_resources(chan);
5702 .name = "ti-udma",
5715 #include "k3-udma-private.c"