Lines Matching +full:xlnx +full:- +full:zynqmp +full:- +full:dpdma
1 // SPDX-License-Identifier: GPL-2.0
3 * Xilinx ZynqMP DPDMA Engine driver
5 * Copyright (C) 2015 - 2020 Xilinx, Inc.
28 #include <dt-bindings/dma/xlnx-zynqmp-dpdma.h>
31 #include "../virt-dma.h"
33 /* DPDMA registers */
119 /* DPDMA descriptor fields */
142 * struct xilinx_dpdma_hw_desc - DPDMA hardware descriptor
180 * struct xilinx_dpdma_sw_desc - DPDMA software descriptor
181 * @hw: DPDMA hardware descriptor
192 * struct xilinx_dpdma_tx_desc - DPDMA transaction descriptor
209 * struct xilinx_dpdma_chan - DPDMA channel
216 * @video_group: flag if multi-channel operation is needed for video channels
224 * @xdev: DPDMA device
252 * struct xilinx_dpdma_device - DPDMA device
258 * @chan: DPDMA channels
273 /* -----------------------------------------------------------------------------
301 if (IS_ENABLED(CONFIG_DEBUG_FS) && chan->id == dpdma_debugfs.chan_id) in xilinx_dpdma_debugfs_desc_done_irq()
328 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
332 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
339 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
366 return -EINVAL; in xilinx_dpdma_debugfs_read()
371 return -ENOMEM; in xilinx_dpdma_debugfs_read()
386 ret = -EFAULT; in xilinx_dpdma_debugfs_read()
407 return -EINVAL; in xilinx_dpdma_debugfs_write()
411 return -EBUSY; in xilinx_dpdma_debugfs_write()
415 return -ENOMEM; in xilinx_dpdma_debugfs_write()
431 ret = -EINVAL; in xilinx_dpdma_debugfs_write()
458 dent = debugfs_create_file("testcase", 0444, xdev->common.dbg_dev_root, in xilinx_dpdma_debugfs_init()
461 dev_err(xdev->dev, "Failed to create debugfs testcase file\n"); in xilinx_dpdma_debugfs_init()
464 /* -----------------------------------------------------------------------------
488 /* -----------------------------------------------------------------------------
493 * xilinx_dpdma_sw_desc_set_dma_addrs - Set DMA addresses in the descriptor
494 * @xdev: DPDMA device
511 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_sw_desc_set_dma_addrs()
514 hw_desc->src_addr = lower_32_bits(dma_addr[0]); in xilinx_dpdma_sw_desc_set_dma_addrs()
515 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
516 hw_desc->addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
521 u32 *addr = &hw_desc->src_addr2; in xilinx_dpdma_sw_desc_set_dma_addrs()
523 addr[i - 1] = lower_32_bits(dma_addr[i]); in xilinx_dpdma_sw_desc_set_dma_addrs()
525 if (xdev->ext_addr) { in xilinx_dpdma_sw_desc_set_dma_addrs()
526 u32 *addr_ext = &hw_desc->addr_ext_23; in xilinx_dpdma_sw_desc_set_dma_addrs()
530 addr_msb <<= 16 * ((i - 1) % 2); in xilinx_dpdma_sw_desc_set_dma_addrs()
531 addr_ext[(i - 1) / 2] |= addr_msb; in xilinx_dpdma_sw_desc_set_dma_addrs()
538 prev->hw.next_desc = lower_32_bits(sw_desc->dma_addr); in xilinx_dpdma_sw_desc_set_dma_addrs()
539 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
540 prev->hw.addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
542 upper_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_sw_desc_set_dma_addrs()
546 * xilinx_dpdma_chan_alloc_sw_desc - Allocate a software descriptor
547 * @chan: DPDMA channel
559 sw_desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &dma_addr); in xilinx_dpdma_chan_alloc_sw_desc()
563 sw_desc->dma_addr = dma_addr; in xilinx_dpdma_chan_alloc_sw_desc()
569 * xilinx_dpdma_chan_free_sw_desc - Free a software descriptor
570 * @chan: DPDMA channel
579 dma_pool_free(chan->desc_pool, sw_desc, sw_desc->dma_addr); in xilinx_dpdma_chan_free_sw_desc()
583 * xilinx_dpdma_chan_dump_tx_desc - Dump a tx descriptor
584 * @chan: DPDMA channel
593 struct device *dev = chan->xdev->dev; in xilinx_dpdma_chan_dump_tx_desc()
596 dev_dbg(dev, "------- TX descriptor dump start -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
597 dev_dbg(dev, "------- channel ID = %d -------\n", chan->id); in xilinx_dpdma_chan_dump_tx_desc()
599 list_for_each_entry(sw_desc, &tx_desc->descriptors, node) { in xilinx_dpdma_chan_dump_tx_desc()
600 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_dump_tx_desc()
602 dev_dbg(dev, "------- HW descriptor %d -------\n", i++); in xilinx_dpdma_chan_dump_tx_desc()
603 dev_dbg(dev, "descriptor DMA addr: %pad\n", &sw_desc->dma_addr); in xilinx_dpdma_chan_dump_tx_desc()
604 dev_dbg(dev, "control: 0x%08x\n", hw_desc->control); in xilinx_dpdma_chan_dump_tx_desc()
605 dev_dbg(dev, "desc_id: 0x%08x\n", hw_desc->desc_id); in xilinx_dpdma_chan_dump_tx_desc()
606 dev_dbg(dev, "xfer_size: 0x%08x\n", hw_desc->xfer_size); in xilinx_dpdma_chan_dump_tx_desc()
607 dev_dbg(dev, "hsize_stride: 0x%08x\n", hw_desc->hsize_stride); in xilinx_dpdma_chan_dump_tx_desc()
608 dev_dbg(dev, "timestamp_lsb: 0x%08x\n", hw_desc->timestamp_lsb); in xilinx_dpdma_chan_dump_tx_desc()
609 dev_dbg(dev, "timestamp_msb: 0x%08x\n", hw_desc->timestamp_msb); in xilinx_dpdma_chan_dump_tx_desc()
610 dev_dbg(dev, "addr_ext: 0x%08x\n", hw_desc->addr_ext); in xilinx_dpdma_chan_dump_tx_desc()
611 dev_dbg(dev, "next_desc: 0x%08x\n", hw_desc->next_desc); in xilinx_dpdma_chan_dump_tx_desc()
612 dev_dbg(dev, "src_addr: 0x%08x\n", hw_desc->src_addr); in xilinx_dpdma_chan_dump_tx_desc()
613 dev_dbg(dev, "addr_ext_23: 0x%08x\n", hw_desc->addr_ext_23); in xilinx_dpdma_chan_dump_tx_desc()
614 dev_dbg(dev, "addr_ext_45: 0x%08x\n", hw_desc->addr_ext_45); in xilinx_dpdma_chan_dump_tx_desc()
615 dev_dbg(dev, "src_addr2: 0x%08x\n", hw_desc->src_addr2); in xilinx_dpdma_chan_dump_tx_desc()
616 dev_dbg(dev, "src_addr3: 0x%08x\n", hw_desc->src_addr3); in xilinx_dpdma_chan_dump_tx_desc()
617 dev_dbg(dev, "src_addr4: 0x%08x\n", hw_desc->src_addr4); in xilinx_dpdma_chan_dump_tx_desc()
618 dev_dbg(dev, "src_addr5: 0x%08x\n", hw_desc->src_addr5); in xilinx_dpdma_chan_dump_tx_desc()
619 dev_dbg(dev, "crc: 0x%08x\n", hw_desc->crc); in xilinx_dpdma_chan_dump_tx_desc()
622 dev_dbg(dev, "------- TX descriptor dump end -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
626 * xilinx_dpdma_chan_alloc_tx_desc - Allocate a transaction descriptor
627 * @chan: DPDMA channel
642 INIT_LIST_HEAD(&tx_desc->descriptors); in xilinx_dpdma_chan_alloc_tx_desc()
643 tx_desc->chan = chan; in xilinx_dpdma_chan_alloc_tx_desc()
644 tx_desc->error = false; in xilinx_dpdma_chan_alloc_tx_desc()
650 * xilinx_dpdma_chan_free_tx_desc - Free a virtual DMA descriptor
665 list_for_each_entry_safe(sw_desc, next, &desc->descriptors, node) { in xilinx_dpdma_chan_free_tx_desc()
666 list_del(&sw_desc->node); in xilinx_dpdma_chan_free_tx_desc()
667 xilinx_dpdma_chan_free_sw_desc(desc->chan, sw_desc); in xilinx_dpdma_chan_free_tx_desc()
674 * xilinx_dpdma_chan_prep_cyclic - Prepare a cyclic dma descriptor
675 * @chan: DPDMA channel
704 dev_err(chan->xdev->dev, in xilinx_dpdma_chan_prep_cyclic()
714 xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, last, in xilinx_dpdma_chan_prep_cyclic()
716 hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_prep_cyclic()
717 hw_desc->xfer_size = period_len; in xilinx_dpdma_chan_prep_cyclic()
718 hw_desc->hsize_stride = in xilinx_dpdma_chan_prep_cyclic()
723 hw_desc->control = XILINX_DPDMA_DESC_CONTROL_PREEMBLE | in xilinx_dpdma_chan_prep_cyclic()
727 list_add_tail(&sw_desc->node, &tx_desc->descriptors); in xilinx_dpdma_chan_prep_cyclic()
733 sw_desc = list_first_entry(&tx_desc->descriptors, in xilinx_dpdma_chan_prep_cyclic()
735 last->hw.next_desc = lower_32_bits(sw_desc->dma_addr); in xilinx_dpdma_chan_prep_cyclic()
736 if (chan->xdev->ext_addr) in xilinx_dpdma_chan_prep_cyclic()
737 last->hw.addr_ext |= in xilinx_dpdma_chan_prep_cyclic()
739 upper_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_chan_prep_cyclic()
741 last->hw.control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; in xilinx_dpdma_chan_prep_cyclic()
743 return vchan_tx_prep(&chan->vchan, &tx_desc->vdesc, flags); in xilinx_dpdma_chan_prep_cyclic()
746 xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); in xilinx_dpdma_chan_prep_cyclic()
752 * xilinx_dpdma_chan_prep_interleaved_dma - Prepare an interleaved dma
754 * @chan: DPDMA channel
760 * Return: A DPDMA TX descriptor on success, or NULL.
769 size_t hsize = xt->sgl[0].size; in xilinx_dpdma_chan_prep_interleaved_dma()
770 size_t stride = hsize + xt->sgl[0].icg; in xilinx_dpdma_chan_prep_interleaved_dma()
772 if (!IS_ALIGNED(xt->src_start, XILINX_DPDMA_ALIGN_BYTES)) { in xilinx_dpdma_chan_prep_interleaved_dma()
773 dev_err(chan->xdev->dev, in xilinx_dpdma_chan_prep_interleaved_dma()
775 chan->id, XILINX_DPDMA_ALIGN_BYTES); in xilinx_dpdma_chan_prep_interleaved_dma()
785 xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); in xilinx_dpdma_chan_prep_interleaved_dma()
789 xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, sw_desc, in xilinx_dpdma_chan_prep_interleaved_dma()
790 &xt->src_start, 1); in xilinx_dpdma_chan_prep_interleaved_dma()
792 hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_prep_interleaved_dma()
794 hw_desc->xfer_size = hsize * xt->numf; in xilinx_dpdma_chan_prep_interleaved_dma()
795 hw_desc->hsize_stride = in xilinx_dpdma_chan_prep_interleaved_dma()
799 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_PREEMBLE; in xilinx_dpdma_chan_prep_interleaved_dma()
800 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_COMPLETE_INTR; in xilinx_dpdma_chan_prep_interleaved_dma()
801 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_IGNORE_DONE; in xilinx_dpdma_chan_prep_interleaved_dma()
802 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; in xilinx_dpdma_chan_prep_interleaved_dma()
804 list_add_tail(&sw_desc->node, &tx_desc->descriptors); in xilinx_dpdma_chan_prep_interleaved_dma()
809 /* -----------------------------------------------------------------------------
810 * DPDMA Channel Operations
814 * xilinx_dpdma_chan_enable - Enable the channel
815 * @chan: DPDMA channel
823 reg = (XILINX_DPDMA_INTR_CHAN_MASK << chan->id) in xilinx_dpdma_chan_enable()
825 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_enable()
826 reg = (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id) in xilinx_dpdma_chan_enable()
828 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_enable()
837 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, reg); in xilinx_dpdma_chan_enable()
841 * xilinx_dpdma_chan_disable - Disable the channel
842 * @chan: DPDMA channel
850 reg = XILINX_DPDMA_INTR_CHAN_MASK << chan->id; in xilinx_dpdma_chan_disable()
851 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_disable()
852 reg = XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id; in xilinx_dpdma_chan_disable()
853 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_disable()
855 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_ENABLE); in xilinx_dpdma_chan_disable()
859 * xilinx_dpdma_chan_pause - Pause the channel
860 * @chan: DPDMA channel
866 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_pause()
870 * xilinx_dpdma_chan_unpause - Unpause the channel
871 * @chan: DPDMA channel
877 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_unpause()
882 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_video_group_ready()
887 if (xdev->chan[i]->video_group && !xdev->chan[i]->running) in xilinx_dpdma_chan_video_group_ready()
890 if (xdev->chan[i]->video_group) in xilinx_dpdma_chan_video_group_ready()
898 * xilinx_dpdma_chan_queue_transfer - Queue the next transfer
899 * @chan: DPDMA channel
906 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_queue_transfer()
913 lockdep_assert_held(&chan->lock); in xilinx_dpdma_chan_queue_transfer()
915 if (chan->desc.pending) in xilinx_dpdma_chan_queue_transfer()
918 if (!chan->running) { in xilinx_dpdma_chan_queue_transfer()
921 chan->first_frame = true; in xilinx_dpdma_chan_queue_transfer()
922 chan->running = true; in xilinx_dpdma_chan_queue_transfer()
925 vdesc = vchan_next_desc(&chan->vchan); in xilinx_dpdma_chan_queue_transfer()
930 chan->desc.pending = desc; in xilinx_dpdma_chan_queue_transfer()
931 list_del(&desc->vdesc.node); in xilinx_dpdma_chan_queue_transfer()
937 list_for_each_entry(sw_desc, &desc->descriptors, node) in xilinx_dpdma_chan_queue_transfer()
938 sw_desc->hw.desc_id = desc->vdesc.tx.cookie in xilinx_dpdma_chan_queue_transfer()
941 sw_desc = list_first_entry(&desc->descriptors, in xilinx_dpdma_chan_queue_transfer()
943 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR, in xilinx_dpdma_chan_queue_transfer()
944 lower_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_chan_queue_transfer()
945 if (xdev->ext_addr) in xilinx_dpdma_chan_queue_transfer()
946 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE, in xilinx_dpdma_chan_queue_transfer()
948 upper_32_bits(sw_desc->dma_addr))); in xilinx_dpdma_chan_queue_transfer()
950 first_frame = chan->first_frame; in xilinx_dpdma_chan_queue_transfer()
951 chan->first_frame = false; in xilinx_dpdma_chan_queue_transfer()
953 if (chan->video_group) { in xilinx_dpdma_chan_queue_transfer()
962 channels = BIT(chan->id); in xilinx_dpdma_chan_queue_transfer()
970 dpdma_write(xdev->reg, XILINX_DPDMA_GBL, reg); in xilinx_dpdma_chan_queue_transfer()
974 * xilinx_dpdma_chan_ostand - Number of outstanding transactions
975 * @chan: DPDMA channel
984 dpdma_read(chan->reg, XILINX_DPDMA_CH_STATUS)); in xilinx_dpdma_chan_ostand()
988 * xilinx_dpdma_chan_notify_no_ostand - Notify no outstanding transaction event
989 * @chan: DPDMA channel
994 * should be re-enabled when this event is handled. If the channel status
998 * Return: 0 on success. On failure, -EWOULDBLOCK if there's still outstanding
1007 dev_dbg(chan->xdev->dev, in xilinx_dpdma_chan_notify_no_ostand()
1009 chan->id, cnt); in xilinx_dpdma_chan_notify_no_ostand()
1010 return -EWOULDBLOCK; in xilinx_dpdma_chan_notify_no_ostand()
1014 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_chan_notify_no_ostand()
1015 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_notify_no_ostand()
1016 wake_up(&chan->wait_to_stop); in xilinx_dpdma_chan_notify_no_ostand()
1022 * xilinx_dpdma_chan_wait_no_ostand - Wait for the no outstanding irq
1023 * @chan: DPDMA channel
1028 * Return: 0 on success. On failure, -ETIMEOUT for time out, or the error code
1036 ret = wait_event_interruptible_timeout(chan->wait_to_stop, in xilinx_dpdma_chan_wait_no_ostand()
1040 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_wait_no_ostand()
1041 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_wait_no_ostand()
1045 dev_err(chan->xdev->dev, "chan%u: not ready to stop: %d trans\n", in xilinx_dpdma_chan_wait_no_ostand()
1046 chan->id, xilinx_dpdma_chan_ostand(chan)); in xilinx_dpdma_chan_wait_no_ostand()
1049 return -ETIMEDOUT; in xilinx_dpdma_chan_wait_no_ostand()
1055 * xilinx_dpdma_chan_poll_no_ostand - Poll the outstanding transaction status
1056 * @chan: DPDMA channel
1062 * Return: 0 on success, or -ETIMEDOUT.
1072 } while (loop-- > 0 && cnt); in xilinx_dpdma_chan_poll_no_ostand()
1075 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_poll_no_ostand()
1076 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_poll_no_ostand()
1080 dev_err(chan->xdev->dev, "chan%u: not ready to stop: %d trans\n", in xilinx_dpdma_chan_poll_no_ostand()
1081 chan->id, xilinx_dpdma_chan_ostand(chan)); in xilinx_dpdma_chan_poll_no_ostand()
1083 return -ETIMEDOUT; in xilinx_dpdma_chan_poll_no_ostand()
1087 * xilinx_dpdma_chan_stop - Stop the channel
1088 * @chan: DPDMA channel
1093 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1104 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1106 chan->running = false; in xilinx_dpdma_chan_stop()
1107 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1113 * xilinx_dpdma_chan_done_irq - Handle hardware descriptor completion
1114 * @chan: DPDMA channel
1116 * Handle completion of the currently active descriptor (@chan->desc.active). As
1125 spin_lock(&chan->lock); in xilinx_dpdma_chan_done_irq()
1129 active = chan->desc.active; in xilinx_dpdma_chan_done_irq()
1131 vchan_cyclic_callback(&active->vdesc); in xilinx_dpdma_chan_done_irq()
1133 dev_warn(chan->xdev->dev, in xilinx_dpdma_chan_done_irq()
1135 chan->id); in xilinx_dpdma_chan_done_irq()
1137 spin_unlock(&chan->lock); in xilinx_dpdma_chan_done_irq()
1141 * xilinx_dpdma_chan_vsync_irq - Handle hardware descriptor scheduling
1142 * @chan: DPDMA channel
1154 spin_lock(&chan->lock); in xilinx_dpdma_chan_vsync_irq()
1156 pending = chan->desc.pending; in xilinx_dpdma_chan_vsync_irq()
1157 if (!chan->running || !pending) in xilinx_dpdma_chan_vsync_irq()
1160 desc_id = dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_ID) in xilinx_dpdma_chan_vsync_irq()
1164 sw_desc = list_first_entry(&pending->descriptors, in xilinx_dpdma_chan_vsync_irq()
1166 if (sw_desc->hw.desc_id != desc_id) { in xilinx_dpdma_chan_vsync_irq()
1167 dev_dbg(chan->xdev->dev, in xilinx_dpdma_chan_vsync_irq()
1169 chan->id, sw_desc->hw.desc_id, desc_id); in xilinx_dpdma_chan_vsync_irq()
1177 spin_lock(&chan->vchan.lock); in xilinx_dpdma_chan_vsync_irq()
1178 if (chan->desc.active) in xilinx_dpdma_chan_vsync_irq()
1179 vchan_cookie_complete(&chan->desc.active->vdesc); in xilinx_dpdma_chan_vsync_irq()
1180 chan->desc.active = pending; in xilinx_dpdma_chan_vsync_irq()
1181 chan->desc.pending = NULL; in xilinx_dpdma_chan_vsync_irq()
1184 spin_unlock(&chan->vchan.lock); in xilinx_dpdma_chan_vsync_irq()
1187 spin_unlock(&chan->lock); in xilinx_dpdma_chan_vsync_irq()
1191 * xilinx_dpdma_chan_err - Detect any channel error
1192 * @chan: DPDMA channel
1204 if (chan->running && in xilinx_dpdma_chan_err()
1205 ((isr & (XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id)) || in xilinx_dpdma_chan_err()
1206 (eisr & (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id)))) in xilinx_dpdma_chan_err()
1213 * xilinx_dpdma_chan_handle_err - DPDMA channel error handling
1214 * @chan: DPDMA channel
1223 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_handle_err()
1227 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1229 dev_dbg(xdev->dev, "chan%u: cur desc addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1230 chan->id, in xilinx_dpdma_chan_handle_err()
1231 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE), in xilinx_dpdma_chan_handle_err()
1232 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR)); in xilinx_dpdma_chan_handle_err()
1233 dev_dbg(xdev->dev, "chan%u: cur payload addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1234 chan->id, in xilinx_dpdma_chan_handle_err()
1235 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDRE), in xilinx_dpdma_chan_handle_err()
1236 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDR)); in xilinx_dpdma_chan_handle_err()
1239 chan->running = false; in xilinx_dpdma_chan_handle_err()
1241 if (!chan->desc.active) in xilinx_dpdma_chan_handle_err()
1244 active = chan->desc.active; in xilinx_dpdma_chan_handle_err()
1245 chan->desc.active = NULL; in xilinx_dpdma_chan_handle_err()
1249 if (active->error) in xilinx_dpdma_chan_handle_err()
1250 dev_dbg(xdev->dev, "chan%u: repeated error on desc\n", in xilinx_dpdma_chan_handle_err()
1251 chan->id); in xilinx_dpdma_chan_handle_err()
1254 if (!chan->desc.pending && in xilinx_dpdma_chan_handle_err()
1255 list_empty(&chan->vchan.desc_issued)) { in xilinx_dpdma_chan_handle_err()
1256 active->error = true; in xilinx_dpdma_chan_handle_err()
1257 list_add_tail(&active->vdesc.node, in xilinx_dpdma_chan_handle_err()
1258 &chan->vchan.desc_issued); in xilinx_dpdma_chan_handle_err()
1260 xilinx_dpdma_chan_free_tx_desc(&active->vdesc); in xilinx_dpdma_chan_handle_err()
1264 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1267 /* -----------------------------------------------------------------------------
1296 if (xt->dir != DMA_MEM_TO_DEV) in xilinx_dpdma_prep_interleaved_dma()
1299 if (!xt->numf || !xt->sgl[0].size) in xilinx_dpdma_prep_interleaved_dma()
1309 vchan_tx_prep(&chan->vchan, &desc->vdesc, flags | DMA_CTRL_ACK); in xilinx_dpdma_prep_interleaved_dma()
1311 return &desc->vdesc.tx; in xilinx_dpdma_prep_interleaved_dma()
1315 * xilinx_dpdma_alloc_chan_resources - Allocate resources for the channel
1320 * Return: 0 on success, or -ENOMEM if failed to allocate a pool.
1327 chan->desc_pool = dma_pool_create(dev_name(chan->xdev->dev), in xilinx_dpdma_alloc_chan_resources()
1328 chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1331 if (!chan->desc_pool) { in xilinx_dpdma_alloc_chan_resources()
1332 dev_err(chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1334 chan->id); in xilinx_dpdma_alloc_chan_resources()
1335 return -ENOMEM; in xilinx_dpdma_alloc_chan_resources()
1342 * xilinx_dpdma_free_chan_resources - Free all resources for the channel
1352 vchan_free_chan_resources(&chan->vchan); in xilinx_dpdma_free_chan_resources()
1354 dma_pool_destroy(chan->desc_pool); in xilinx_dpdma_free_chan_resources()
1355 chan->desc_pool = NULL; in xilinx_dpdma_free_chan_resources()
1363 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_issue_pending()
1364 spin_lock(&chan->vchan.lock); in xilinx_dpdma_issue_pending()
1365 if (vchan_issue_pending(&chan->vchan)) in xilinx_dpdma_issue_pending()
1367 spin_unlock(&chan->vchan.lock); in xilinx_dpdma_issue_pending()
1368 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_issue_pending()
1379 * The destination address doesn't need to be specified as the DPDMA is in xilinx_dpdma_config()
1382 * fixed both on the DPDMA side and on the DP controller side. in xilinx_dpdma_config()
1390 pconfig = config->peripheral_config; in xilinx_dpdma_config()
1391 if (WARN_ON(pconfig && config->peripheral_size != sizeof(*pconfig))) in xilinx_dpdma_config()
1392 return -EINVAL; in xilinx_dpdma_config()
1394 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_config()
1395 if (chan->id <= ZYNQMP_DPDMA_VIDEO2 && pconfig) in xilinx_dpdma_config()
1396 chan->video_group = pconfig->video_group; in xilinx_dpdma_config()
1397 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_config()
1417 * xilinx_dpdma_terminate_all - Terminate the channel and descriptors
1429 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1434 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_terminate_all()
1440 if (chan->video_group) { in xilinx_dpdma_terminate_all()
1442 if (xdev->chan[i]->video_group && in xilinx_dpdma_terminate_all()
1443 xdev->chan[i]->running) { in xilinx_dpdma_terminate_all()
1444 xilinx_dpdma_chan_pause(xdev->chan[i]); in xilinx_dpdma_terminate_all()
1445 xdev->chan[i]->video_group = false; in xilinx_dpdma_terminate_all()
1453 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1454 vchan_get_all_descriptors(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1455 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1457 vchan_dma_desc_free_list(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1463 * xilinx_dpdma_synchronize - Synchronize callback execution
1482 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1483 if (chan->desc.pending) { in xilinx_dpdma_synchronize()
1484 vchan_terminate_vdesc(&chan->desc.pending->vdesc); in xilinx_dpdma_synchronize()
1485 chan->desc.pending = NULL; in xilinx_dpdma_synchronize()
1487 if (chan->desc.active) { in xilinx_dpdma_synchronize()
1488 vchan_terminate_vdesc(&chan->desc.active->vdesc); in xilinx_dpdma_synchronize()
1489 chan->desc.active = NULL; in xilinx_dpdma_synchronize()
1491 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1493 vchan_synchronize(&chan->vchan); in xilinx_dpdma_synchronize()
1496 /* -----------------------------------------------------------------------------
1501 * xilinx_dpdma_err - Detect any global error
1517 * xilinx_dpdma_handle_err_irq - Handle DPDMA error interrupt
1518 * @xdev: DPDMA device
1523 * corresponding error interrupts, and those should be re-enabled once handling
1532 dev_dbg_ratelimited(xdev->dev, in xilinx_dpdma_handle_err_irq()
1537 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_handle_err_irq()
1539 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, in xilinx_dpdma_handle_err_irq()
1542 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_handle_err_irq()
1543 if (err || xilinx_dpdma_chan_err(xdev->chan[i], isr, eisr)) in xilinx_dpdma_handle_err_irq()
1544 tasklet_schedule(&xdev->chan[i]->err_task); in xilinx_dpdma_handle_err_irq()
1548 * xilinx_dpdma_enable_irq - Enable interrupts
1549 * @xdev: DPDMA device
1555 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_enable_irq()
1556 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_enable_irq()
1560 * xilinx_dpdma_disable_irq - Disable interrupts
1561 * @xdev: DPDMA device
1567 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_disable_irq()
1568 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_disable_irq()
1572 * xilinx_dpdma_chan_err_task - Per channel tasklet for error handling
1577 * re-enable channel error interrupts, and restart the channel if needed.
1582 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_err_task()
1590 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_err_task()
1591 XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1592 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, in xilinx_dpdma_chan_err_task()
1593 XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1595 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1596 spin_lock(&chan->vchan.lock); in xilinx_dpdma_chan_err_task()
1598 spin_unlock(&chan->vchan.lock); in xilinx_dpdma_chan_err_task()
1599 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1610 status = dpdma_read(xdev->reg, XILINX_DPDMA_ISR); in xilinx_dpdma_irq_handler()
1611 error = dpdma_read(xdev->reg, XILINX_DPDMA_EISR); in xilinx_dpdma_irq_handler()
1615 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, status); in xilinx_dpdma_irq_handler()
1616 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, error); in xilinx_dpdma_irq_handler()
1623 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in xilinx_dpdma_irq_handler()
1624 struct xilinx_dpdma_chan *chan = xdev->chan[i]; in xilinx_dpdma_irq_handler()
1633 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1634 xilinx_dpdma_chan_done_irq(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1639 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1640 xilinx_dpdma_chan_notify_no_ostand(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1650 /* -----------------------------------------------------------------------------
1659 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); in xilinx_dpdma_chan_init()
1661 return -ENOMEM; in xilinx_dpdma_chan_init()
1663 chan->id = chan_id; in xilinx_dpdma_chan_init()
1664 chan->reg = xdev->reg + XILINX_DPDMA_CH_BASE in xilinx_dpdma_chan_init()
1665 + XILINX_DPDMA_CH_OFFSET * chan->id; in xilinx_dpdma_chan_init()
1666 chan->running = false; in xilinx_dpdma_chan_init()
1667 chan->xdev = xdev; in xilinx_dpdma_chan_init()
1669 spin_lock_init(&chan->lock); in xilinx_dpdma_chan_init()
1670 init_waitqueue_head(&chan->wait_to_stop); in xilinx_dpdma_chan_init()
1672 tasklet_setup(&chan->err_task, xilinx_dpdma_chan_err_task); in xilinx_dpdma_chan_init()
1674 chan->vchan.desc_free = xilinx_dpdma_chan_free_tx_desc; in xilinx_dpdma_chan_init()
1675 vchan_init(&chan->vchan, &xdev->common); in xilinx_dpdma_chan_init()
1677 xdev->chan[chan->id] = chan; in xilinx_dpdma_chan_init()
1687 tasklet_kill(&chan->err_task); in xilinx_dpdma_chan_remove()
1688 list_del(&chan->vchan.chan.device_node); in xilinx_dpdma_chan_remove()
1694 struct xilinx_dpdma_device *xdev = ofdma->of_dma_data; in of_dma_xilinx_xlate()
1695 u32 chan_id = dma_spec->args[0]; in of_dma_xilinx_xlate()
1697 if (chan_id >= ARRAY_SIZE(xdev->chan)) in of_dma_xilinx_xlate()
1700 if (!xdev->chan[chan_id]) in of_dma_xilinx_xlate()
1703 return dma_get_slave_channel(&xdev->chan[chan_id]->vchan.chan); in of_dma_xilinx_xlate()
1715 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in dpdma_hw_init()
1716 reg = xdev->reg + XILINX_DPDMA_CH_BASE in dpdma_hw_init()
1722 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, XILINX_DPDMA_INTR_ALL); in dpdma_hw_init()
1723 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, XILINX_DPDMA_EINTR_ALL); in dpdma_hw_init()
1733 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); in xilinx_dpdma_probe()
1735 return -ENOMEM; in xilinx_dpdma_probe()
1737 xdev->dev = &pdev->dev; in xilinx_dpdma_probe()
1738 xdev->ext_addr = sizeof(dma_addr_t) > 4; in xilinx_dpdma_probe()
1740 INIT_LIST_HEAD(&xdev->common.channels); in xilinx_dpdma_probe()
1744 xdev->axi_clk = devm_clk_get(xdev->dev, "axi_clk"); in xilinx_dpdma_probe()
1745 if (IS_ERR(xdev->axi_clk)) in xilinx_dpdma_probe()
1746 return PTR_ERR(xdev->axi_clk); in xilinx_dpdma_probe()
1748 xdev->reg = devm_platform_ioremap_resource(pdev, 0); in xilinx_dpdma_probe()
1749 if (IS_ERR(xdev->reg)) in xilinx_dpdma_probe()
1750 return PTR_ERR(xdev->reg); in xilinx_dpdma_probe()
1754 xdev->irq = platform_get_irq(pdev, 0); in xilinx_dpdma_probe()
1755 if (xdev->irq < 0) in xilinx_dpdma_probe()
1756 return xdev->irq; in xilinx_dpdma_probe()
1758 ret = request_irq(xdev->irq, xilinx_dpdma_irq_handler, IRQF_SHARED, in xilinx_dpdma_probe()
1759 dev_name(xdev->dev), xdev); in xilinx_dpdma_probe()
1761 dev_err(xdev->dev, "failed to request IRQ\n"); in xilinx_dpdma_probe()
1765 ddev = &xdev->common; in xilinx_dpdma_probe()
1766 ddev->dev = &pdev->dev; in xilinx_dpdma_probe()
1768 dma_cap_set(DMA_SLAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1769 dma_cap_set(DMA_PRIVATE, ddev->cap_mask); in xilinx_dpdma_probe()
1770 dma_cap_set(DMA_CYCLIC, ddev->cap_mask); in xilinx_dpdma_probe()
1771 dma_cap_set(DMA_INTERLEAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1772 dma_cap_set(DMA_REPEAT, ddev->cap_mask); in xilinx_dpdma_probe()
1773 dma_cap_set(DMA_LOAD_EOT, ddev->cap_mask); in xilinx_dpdma_probe()
1774 ddev->copy_align = fls(XILINX_DPDMA_ALIGN_BYTES - 1); in xilinx_dpdma_probe()
1776 ddev->device_alloc_chan_resources = xilinx_dpdma_alloc_chan_resources; in xilinx_dpdma_probe()
1777 ddev->device_free_chan_resources = xilinx_dpdma_free_chan_resources; in xilinx_dpdma_probe()
1778 ddev->device_prep_dma_cyclic = xilinx_dpdma_prep_dma_cyclic; in xilinx_dpdma_probe()
1779 ddev->device_prep_interleaved_dma = xilinx_dpdma_prep_interleaved_dma; in xilinx_dpdma_probe()
1781 ddev->device_tx_status = dma_cookie_status; in xilinx_dpdma_probe()
1782 ddev->device_issue_pending = xilinx_dpdma_issue_pending; in xilinx_dpdma_probe()
1783 ddev->device_config = xilinx_dpdma_config; in xilinx_dpdma_probe()
1784 ddev->device_pause = xilinx_dpdma_pause; in xilinx_dpdma_probe()
1785 ddev->device_resume = xilinx_dpdma_resume; in xilinx_dpdma_probe()
1786 ddev->device_terminate_all = xilinx_dpdma_terminate_all; in xilinx_dpdma_probe()
1787 ddev->device_synchronize = xilinx_dpdma_synchronize; in xilinx_dpdma_probe()
1788 ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED); in xilinx_dpdma_probe()
1789 ddev->directions = BIT(DMA_MEM_TO_DEV); in xilinx_dpdma_probe()
1790 ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in xilinx_dpdma_probe()
1792 for (i = 0; i < ARRAY_SIZE(xdev->chan); ++i) { in xilinx_dpdma_probe()
1795 dev_err(xdev->dev, "failed to initialize channel %u\n", in xilinx_dpdma_probe()
1801 ret = clk_prepare_enable(xdev->axi_clk); in xilinx_dpdma_probe()
1803 dev_err(xdev->dev, "failed to enable the axi clock\n"); in xilinx_dpdma_probe()
1809 dev_err(xdev->dev, "failed to register the dma device\n"); in xilinx_dpdma_probe()
1813 ret = of_dma_controller_register(xdev->dev->of_node, in xilinx_dpdma_probe()
1816 dev_err(xdev->dev, "failed to register DMA to DT DMA helper\n"); in xilinx_dpdma_probe()
1824 dev_info(&pdev->dev, "Xilinx DPDMA engine is probed\n"); in xilinx_dpdma_probe()
1831 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_probe()
1833 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_probe()
1834 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_probe()
1836 free_irq(xdev->irq, xdev); in xilinx_dpdma_probe()
1847 free_irq(xdev->irq, xdev); in xilinx_dpdma_remove()
1850 of_dma_controller_free(pdev->dev.of_node); in xilinx_dpdma_remove()
1851 dma_async_device_unregister(&xdev->common); in xilinx_dpdma_remove()
1852 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_remove()
1854 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_remove()
1855 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_remove()
1859 { .compatible = "xlnx,zynqmp-dpdma",},
1868 .name = "xilinx-zynqmp-dpdma",
1876 MODULE_DESCRIPTION("Xilinx ZynqMP DPDMA driver");