Lines Matching +full:bam +full:- +full:v1
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
6 * QCOM BAM DMA engine driver
8 * QCOM BAM DMA blocks are distributed amongst a number of the on-chip
32 #include <linux/dma-mapping.h>
46 #include "../virt-dma.h"
201 /* BAM CTRL */
212 /* BAM REVISION */
234 /* BAM NUM PIPES */
242 /* BAM CNFG BITS */
342 #define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
343 #define BAM_FIFO_SIZE (SZ_32K - 8)
344 #define IS_BUSY(chan) (CIRC_SPACE(bchan->tail, bchan->head,\
404 * bam_addr - returns BAM register address
405 * @bdev: bam device
412 const struct reg_offset_data r = bdev->layout[reg]; in bam_addr()
414 return bdev->regs + r.base_offset + in bam_addr()
417 r.ee_mult * bdev->ee; in bam_addr()
421 * bam_reset() - reset and initialize BAM registers
422 * @bdev: bam device
428 /* s/w reset bam */ in bam_reset()
436 /* make sure previous stores are visible before enabling BAM */ in bam_reset()
439 /* enable bam */ in bam_reset()
454 /* unmask global bam interrupt */ in bam_reset()
459 * bam_reset_channel - Reset individual BAM DMA channel
460 * @bchan: bam channel
462 * This function resets a specific BAM channel
466 struct bam_device *bdev = bchan->bdev; in bam_reset_channel()
468 lockdep_assert_held(&bchan->vc.lock); in bam_reset_channel()
471 writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST)); in bam_reset_channel()
472 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST)); in bam_reset_channel()
474 /* don't allow cpu to reorder BAM register accesses done after this */ in bam_reset_channel()
478 bchan->initialized = 0; in bam_reset_channel()
482 * bam_chan_init_hw - Initialize channel hardware
483 * @bchan: bam channel
486 * This function resets and initializes the BAM channel
491 struct bam_device *bdev = bchan->bdev; in bam_chan_init_hw()
501 writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)), in bam_chan_init_hw()
502 bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR)); in bam_chan_init_hw()
504 bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES)); in bam_chan_init_hw()
508 bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); in bam_chan_init_hw()
512 val |= BIT(bchan->id); in bam_chan_init_hw()
523 writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL)); in bam_chan_init_hw()
525 bchan->initialized = 1; in bam_chan_init_hw()
528 bchan->head = 0; in bam_chan_init_hw()
529 bchan->tail = 0; in bam_chan_init_hw()
533 * bam_alloc_chan - Allocate channel resources for DMA channel.
541 struct bam_device *bdev = bchan->bdev; in bam_alloc_chan()
543 if (bchan->fifo_virt) in bam_alloc_chan()
547 bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE, in bam_alloc_chan()
548 &bchan->fifo_phys, GFP_KERNEL); in bam_alloc_chan()
550 if (!bchan->fifo_virt) { in bam_alloc_chan()
551 dev_err(bdev->dev, "Failed to allocate desc fifo\n"); in bam_alloc_chan()
552 return -ENOMEM; in bam_alloc_chan()
555 if (bdev->active_channels++ == 0 && bdev->powered_remotely) in bam_alloc_chan()
562 * bam_free_chan - Frees dma resources associated with specific channel
571 struct bam_device *bdev = bchan->bdev; in bam_free_chan()
576 ret = pm_runtime_get_sync(bdev->dev); in bam_free_chan()
582 if (!list_empty(&bchan->desc_list)) { in bam_free_chan()
583 dev_err(bchan->bdev->dev, "Cannot free busy channel\n"); in bam_free_chan()
587 spin_lock_irqsave(&bchan->vc.lock, flags); in bam_free_chan()
589 spin_unlock_irqrestore(&bchan->vc.lock, flags); in bam_free_chan()
591 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt, in bam_free_chan()
592 bchan->fifo_phys); in bam_free_chan()
593 bchan->fifo_virt = NULL; in bam_free_chan()
597 val &= ~BIT(bchan->id); in bam_free_chan()
601 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); in bam_free_chan()
603 if (--bdev->active_channels == 0 && bdev->powered_remotely) { in bam_free_chan()
604 /* s/w reset bam */ in bam_free_chan()
611 pm_runtime_mark_last_busy(bdev->dev); in bam_free_chan()
612 pm_runtime_put_autosuspend(bdev->dev); in bam_free_chan()
616 * bam_slave_config - set slave configuration for channel
629 spin_lock_irqsave(&bchan->vc.lock, flag); in bam_slave_config()
630 memcpy(&bchan->slave, cfg, sizeof(*cfg)); in bam_slave_config()
631 bchan->reconfigure = 1; in bam_slave_config()
632 spin_unlock_irqrestore(&bchan->vc.lock, flag); in bam_slave_config()
638 * bam_prep_slave_sg - Prep slave sg transaction
653 struct bam_device *bdev = bchan->bdev; in bam_prep_slave_sg()
662 dev_err(bdev->dev, "invalid dma direction\n"); in bam_prep_slave_sg()
678 async_desc->flags |= DESC_FLAG_NWD; in bam_prep_slave_sg()
681 async_desc->flags |= DESC_FLAG_EOT; in bam_prep_slave_sg()
683 async_desc->num_desc = num_alloc; in bam_prep_slave_sg()
684 async_desc->curr_desc = async_desc->desc; in bam_prep_slave_sg()
685 async_desc->dir = direction; in bam_prep_slave_sg()
688 desc = async_desc->desc; in bam_prep_slave_sg()
695 desc->flags |= cpu_to_le16(DESC_FLAG_CMD); in bam_prep_slave_sg()
697 desc->addr = cpu_to_le32(sg_dma_address(sg) + in bam_prep_slave_sg()
701 desc->size = cpu_to_le16(BAM_FIFO_SIZE); in bam_prep_slave_sg()
702 remainder -= BAM_FIFO_SIZE; in bam_prep_slave_sg()
705 desc->size = cpu_to_le16(remainder); in bam_prep_slave_sg()
709 async_desc->length += le16_to_cpu(desc->size); in bam_prep_slave_sg()
714 return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags); in bam_prep_slave_sg()
718 * bam_dma_terminate_all - terminate all transactions on a channel
719 * @chan: bam dma channel
733 spin_lock_irqsave(&bchan->vc.lock, flag); in bam_dma_terminate_all()
741 * the BAM will occur. There is a small window where this could happen in bam_dma_terminate_all()
744 * this first so that the BAM hardware doesn't cause memory corruption in bam_dma_terminate_all()
747 if (!list_empty(&bchan->desc_list)) { in bam_dma_terminate_all()
748 async_desc = list_first_entry(&bchan->desc_list, in bam_dma_terminate_all()
750 bam_chan_init_hw(bchan, async_desc->dir); in bam_dma_terminate_all()
754 &bchan->desc_list, desc_node) { in bam_dma_terminate_all()
755 list_add(&async_desc->vd.node, &bchan->vc.desc_issued); in bam_dma_terminate_all()
756 list_del(&async_desc->desc_node); in bam_dma_terminate_all()
759 vchan_get_all_descriptors(&bchan->vc, &head); in bam_dma_terminate_all()
760 spin_unlock_irqrestore(&bchan->vc.lock, flag); in bam_dma_terminate_all()
762 vchan_dma_desc_free_list(&bchan->vc, &head); in bam_dma_terminate_all()
768 * bam_pause - Pause DMA channel
775 struct bam_device *bdev = bchan->bdev; in bam_pause()
779 ret = pm_runtime_get_sync(bdev->dev); in bam_pause()
783 spin_lock_irqsave(&bchan->vc.lock, flag); in bam_pause()
784 writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT)); in bam_pause()
785 bchan->paused = 1; in bam_pause()
786 spin_unlock_irqrestore(&bchan->vc.lock, flag); in bam_pause()
787 pm_runtime_mark_last_busy(bdev->dev); in bam_pause()
788 pm_runtime_put_autosuspend(bdev->dev); in bam_pause()
794 * bam_resume - Resume DMA channel operations
801 struct bam_device *bdev = bchan->bdev; in bam_resume()
805 ret = pm_runtime_get_sync(bdev->dev); in bam_resume()
809 spin_lock_irqsave(&bchan->vc.lock, flag); in bam_resume()
810 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT)); in bam_resume()
811 bchan->paused = 0; in bam_resume()
812 spin_unlock_irqrestore(&bchan->vc.lock, flag); in bam_resume()
813 pm_runtime_mark_last_busy(bdev->dev); in bam_resume()
814 pm_runtime_put_autosuspend(bdev->dev); in bam_resume()
820 * process_channel_irqs - processes the channel interrupts
821 * @bdev: bam controller
838 for (i = 0; i < bdev->num_channels; i++) { in process_channel_irqs()
839 struct bam_chan *bchan = &bdev->channels[i]; in process_channel_irqs()
849 spin_lock_irqsave(&bchan->vc.lock, flags); in process_channel_irqs()
856 avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1); in process_channel_irqs()
858 if (offset < bchan->head) in process_channel_irqs()
859 avail--; in process_channel_irqs()
862 &bchan->desc_list, desc_node) { in process_channel_irqs()
864 if (avail < async_desc->xfer_len) in process_channel_irqs()
868 bchan->head += async_desc->xfer_len; in process_channel_irqs()
869 bchan->head %= MAX_DESCRIPTORS; in process_channel_irqs()
871 async_desc->num_desc -= async_desc->xfer_len; in process_channel_irqs()
872 async_desc->curr_desc += async_desc->xfer_len; in process_channel_irqs()
873 avail -= async_desc->xfer_len; in process_channel_irqs()
880 if (!async_desc->num_desc) { in process_channel_irqs()
881 vchan_cookie_complete(&async_desc->vd); in process_channel_irqs()
883 list_add(&async_desc->vd.node, in process_channel_irqs()
884 &bchan->vc.desc_issued); in process_channel_irqs()
886 list_del(&async_desc->desc_node); in process_channel_irqs()
889 spin_unlock_irqrestore(&bchan->vc.lock, flags); in process_channel_irqs()
896 * bam_dma_irq - irq handler for bam controller
900 * IRQ handler for the bam controller
912 tasklet_schedule(&bdev->task); in bam_dma_irq()
914 ret = pm_runtime_get_sync(bdev->dev); in bam_dma_irq()
922 * don't allow reorder of the various accesses to the BAM in bam_dma_irq()
930 pm_runtime_mark_last_busy(bdev->dev); in bam_dma_irq()
931 pm_runtime_put_autosuspend(bdev->dev); in bam_dma_irq()
937 * bam_tx_status - returns status of transaction
960 return bchan->paused ? DMA_PAUSED : ret; in bam_tx_status()
962 spin_lock_irqsave(&bchan->vc.lock, flags); in bam_tx_status()
963 vd = vchan_find_desc(&bchan->vc, cookie); in bam_tx_status()
965 residue = container_of(vd, struct bam_async_desc, vd)->length; in bam_tx_status()
967 list_for_each_entry(async_desc, &bchan->desc_list, desc_node) { in bam_tx_status()
968 if (async_desc->vd.tx.cookie != cookie) in bam_tx_status()
971 for (i = 0; i < async_desc->num_desc; i++) in bam_tx_status()
973 async_desc->curr_desc[i].size); in bam_tx_status()
977 spin_unlock_irqrestore(&bchan->vc.lock, flags); in bam_tx_status()
981 if (ret == DMA_IN_PROGRESS && bchan->paused) in bam_tx_status()
989 * @bchan: bam dma channel
995 struct bam_device *bdev = bchan->bdev; in bam_apply_new_config()
998 if (!bdev->controlled_remotely) { in bam_apply_new_config()
1000 maxburst = bchan->slave.src_maxburst; in bam_apply_new_config()
1002 maxburst = bchan->slave.dst_maxburst; in bam_apply_new_config()
1008 bchan->reconfigure = 0; in bam_apply_new_config()
1012 * bam_start_dma - start next transaction
1013 * @bchan: bam dma channel
1017 struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc); in bam_start_dma()
1018 struct bam_device *bdev = bchan->bdev; in bam_start_dma()
1021 struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt, in bam_start_dma()
1027 lockdep_assert_held(&bchan->vc.lock); in bam_start_dma()
1032 ret = pm_runtime_get_sync(bdev->dev); in bam_start_dma()
1037 list_del(&vd->node); in bam_start_dma()
1042 if (!bchan->initialized) in bam_start_dma()
1043 bam_chan_init_hw(bchan, async_desc->dir); in bam_start_dma()
1046 if (bchan->reconfigure) in bam_start_dma()
1047 bam_apply_new_config(bchan, async_desc->dir); in bam_start_dma()
1049 desc = async_desc->curr_desc; in bam_start_dma()
1050 avail = CIRC_SPACE(bchan->tail, bchan->head, in bam_start_dma()
1053 if (async_desc->num_desc > avail) in bam_start_dma()
1054 async_desc->xfer_len = avail; in bam_start_dma()
1056 async_desc->xfer_len = async_desc->num_desc; in bam_start_dma()
1059 if (async_desc->num_desc == async_desc->xfer_len) in bam_start_dma()
1060 desc[async_desc->xfer_len - 1].flags |= in bam_start_dma()
1061 cpu_to_le16(async_desc->flags); in bam_start_dma()
1063 vd = vchan_next_desc(&bchan->vc); in bam_start_dma()
1065 dmaengine_desc_get_callback(&async_desc->vd.tx, &cb); in bam_start_dma()
1069 * - FIFO is FULL. in bam_start_dma()
1070 * - No more descriptors to add. in bam_start_dma()
1071 * - If a callback completion was requested for this DESC, in bam_start_dma()
1072 * In this case, BAM will deliver the completion callback in bam_start_dma()
1075 if (((avail <= async_desc->xfer_len) || !vd || in bam_start_dma()
1077 !(async_desc->flags & DESC_FLAG_EOT)) in bam_start_dma()
1078 desc[async_desc->xfer_len - 1].flags |= in bam_start_dma()
1081 if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) { in bam_start_dma()
1082 u32 partial = MAX_DESCRIPTORS - bchan->tail; in bam_start_dma()
1084 memcpy(&fifo[bchan->tail], desc, in bam_start_dma()
1087 (async_desc->xfer_len - partial) * in bam_start_dma()
1090 memcpy(&fifo[bchan->tail], desc, in bam_start_dma()
1091 async_desc->xfer_len * in bam_start_dma()
1095 bchan->tail += async_desc->xfer_len; in bam_start_dma()
1096 bchan->tail %= MAX_DESCRIPTORS; in bam_start_dma()
1097 list_add_tail(&async_desc->desc_node, &bchan->desc_list); in bam_start_dma()
1102 writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw), in bam_start_dma()
1103 bam_addr(bdev, bchan->id, BAM_P_EVNT_REG)); in bam_start_dma()
1105 pm_runtime_mark_last_busy(bdev->dev); in bam_start_dma()
1106 pm_runtime_put_autosuspend(bdev->dev); in bam_start_dma()
1110 * dma_tasklet - DMA IRQ tasklet
1111 * @t: tasklet argument (bam controller structure)
1123 for (i = 0; i < bdev->num_channels; i++) { in dma_tasklet()
1124 bchan = &bdev->channels[i]; in dma_tasklet()
1125 spin_lock_irqsave(&bchan->vc.lock, flags); in dma_tasklet()
1127 if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan)) in dma_tasklet()
1129 spin_unlock_irqrestore(&bchan->vc.lock, flags); in dma_tasklet()
1135 * bam_issue_pending - starts pending transactions
1145 spin_lock_irqsave(&bchan->vc.lock, flags); in bam_issue_pending()
1148 if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan)) in bam_issue_pending()
1151 spin_unlock_irqrestore(&bchan->vc.lock, flags); in bam_issue_pending()
1155 * bam_dma_free_desc - free descriptor memory
1170 struct bam_device *bdev = container_of(of->of_dma_data, in bam_dma_xlate()
1174 if (dma_spec->args_count != 1) in bam_dma_xlate()
1177 request = dma_spec->args[0]; in bam_dma_xlate()
1178 if (request >= bdev->num_channels) in bam_dma_xlate()
1181 return dma_get_slave_channel(&(bdev->channels[request].vc.chan)); in bam_dma_xlate()
1186 * @bdev: bam device
1188 * Initialization helper for global bam registers
1195 if (!bdev->num_ees) { in bam_init()
1197 bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK; in bam_init()
1201 if (bdev->ee >= bdev->num_ees) in bam_init()
1202 return -EINVAL; in bam_init()
1204 if (!bdev->num_channels) { in bam_init()
1206 bdev->num_channels = val & BAM_NUM_PIPES_MASK; in bam_init()
1209 /* Reset BAM now if fully controlled locally */ in bam_init()
1210 if (!bdev->controlled_remotely && !bdev->powered_remotely) in bam_init()
1219 bchan->id = index; in bam_channel_init()
1220 bchan->bdev = bdev; in bam_channel_init()
1222 vchan_init(&bchan->vc, &bdev->common); in bam_channel_init()
1223 bchan->vc.desc_free = bam_dma_free_desc; in bam_channel_init()
1224 INIT_LIST_HEAD(&bchan->desc_list); in bam_channel_init()
1228 { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
1229 { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
1230 { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
1242 bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL); in bam_dma_probe()
1244 return -ENOMEM; in bam_dma_probe()
1246 bdev->dev = &pdev->dev; in bam_dma_probe()
1248 match = of_match_node(bam_of_match, pdev->dev.of_node); in bam_dma_probe()
1250 dev_err(&pdev->dev, "Unsupported BAM module\n"); in bam_dma_probe()
1251 return -ENODEV; in bam_dma_probe()
1254 bdev->layout = match->data; in bam_dma_probe()
1256 bdev->regs = devm_platform_ioremap_resource(pdev, 0); in bam_dma_probe()
1257 if (IS_ERR(bdev->regs)) in bam_dma_probe()
1258 return PTR_ERR(bdev->regs); in bam_dma_probe()
1260 bdev->irq = platform_get_irq(pdev, 0); in bam_dma_probe()
1261 if (bdev->irq < 0) in bam_dma_probe()
1262 return bdev->irq; in bam_dma_probe()
1264 ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee); in bam_dma_probe()
1266 dev_err(bdev->dev, "Execution environment unspecified\n"); in bam_dma_probe()
1270 bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node, in bam_dma_probe()
1271 "qcom,controlled-remotely"); in bam_dma_probe()
1272 bdev->powered_remotely = of_property_read_bool(pdev->dev.of_node, in bam_dma_probe()
1273 "qcom,powered-remotely"); in bam_dma_probe()
1275 if (bdev->controlled_remotely || bdev->powered_remotely) in bam_dma_probe()
1276 bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk"); in bam_dma_probe()
1278 bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); in bam_dma_probe()
1280 if (IS_ERR(bdev->bamclk)) in bam_dma_probe()
1281 return PTR_ERR(bdev->bamclk); in bam_dma_probe()
1283 if (!bdev->bamclk) { in bam_dma_probe()
1284 ret = of_property_read_u32(pdev->dev.of_node, "num-channels", in bam_dma_probe()
1285 &bdev->num_channels); in bam_dma_probe()
1287 dev_err(bdev->dev, "num-channels unspecified in dt\n"); in bam_dma_probe()
1289 ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees", in bam_dma_probe()
1290 &bdev->num_ees); in bam_dma_probe()
1292 dev_err(bdev->dev, "num-ees unspecified in dt\n"); in bam_dma_probe()
1295 ret = clk_prepare_enable(bdev->bamclk); in bam_dma_probe()
1297 dev_err(bdev->dev, "failed to prepare/enable clock\n"); in bam_dma_probe()
1305 tasklet_setup(&bdev->task, dma_tasklet); in bam_dma_probe()
1307 bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels, in bam_dma_probe()
1308 sizeof(*bdev->channels), GFP_KERNEL); in bam_dma_probe()
1310 if (!bdev->channels) { in bam_dma_probe()
1311 ret = -ENOMEM; in bam_dma_probe()
1316 INIT_LIST_HEAD(&bdev->common.channels); in bam_dma_probe()
1318 for (i = 0; i < bdev->num_channels; i++) in bam_dma_probe()
1319 bam_channel_init(bdev, &bdev->channels[i], i); in bam_dma_probe()
1321 ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq, in bam_dma_probe()
1327 bdev->common.dev = bdev->dev; in bam_dma_probe()
1328 dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE); in bam_dma_probe()
1333 dma_cap_zero(bdev->common.cap_mask); in bam_dma_probe()
1334 dma_cap_set(DMA_SLAVE, bdev->common.cap_mask); in bam_dma_probe()
1337 bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in bam_dma_probe()
1338 bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; in bam_dma_probe()
1339 bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; in bam_dma_probe()
1340 bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; in bam_dma_probe()
1341 bdev->common.device_alloc_chan_resources = bam_alloc_chan; in bam_dma_probe()
1342 bdev->common.device_free_chan_resources = bam_free_chan; in bam_dma_probe()
1343 bdev->common.device_prep_slave_sg = bam_prep_slave_sg; in bam_dma_probe()
1344 bdev->common.device_config = bam_slave_config; in bam_dma_probe()
1345 bdev->common.device_pause = bam_pause; in bam_dma_probe()
1346 bdev->common.device_resume = bam_resume; in bam_dma_probe()
1347 bdev->common.device_terminate_all = bam_dma_terminate_all; in bam_dma_probe()
1348 bdev->common.device_issue_pending = bam_issue_pending; in bam_dma_probe()
1349 bdev->common.device_tx_status = bam_tx_status; in bam_dma_probe()
1350 bdev->common.dev = bdev->dev; in bam_dma_probe()
1352 ret = dma_async_device_register(&bdev->common); in bam_dma_probe()
1354 dev_err(bdev->dev, "failed to register dma async device\n"); in bam_dma_probe()
1358 ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate, in bam_dma_probe()
1359 &bdev->common); in bam_dma_probe()
1363 pm_runtime_irq_safe(&pdev->dev); in bam_dma_probe()
1364 pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY); in bam_dma_probe()
1365 pm_runtime_use_autosuspend(&pdev->dev); in bam_dma_probe()
1366 pm_runtime_mark_last_busy(&pdev->dev); in bam_dma_probe()
1367 pm_runtime_set_active(&pdev->dev); in bam_dma_probe()
1368 pm_runtime_enable(&pdev->dev); in bam_dma_probe()
1373 dma_async_device_unregister(&bdev->common); in bam_dma_probe()
1375 for (i = 0; i < bdev->num_channels; i++) in bam_dma_probe()
1376 tasklet_kill(&bdev->channels[i].vc.task); in bam_dma_probe()
1378 tasklet_kill(&bdev->task); in bam_dma_probe()
1380 clk_disable_unprepare(bdev->bamclk); in bam_dma_probe()
1390 pm_runtime_force_suspend(&pdev->dev); in bam_dma_remove()
1392 of_dma_controller_free(pdev->dev.of_node); in bam_dma_remove()
1393 dma_async_device_unregister(&bdev->common); in bam_dma_remove()
1398 devm_free_irq(bdev->dev, bdev->irq, bdev); in bam_dma_remove()
1400 for (i = 0; i < bdev->num_channels; i++) { in bam_dma_remove()
1401 bam_dma_terminate_all(&bdev->channels[i].vc.chan); in bam_dma_remove()
1402 tasklet_kill(&bdev->channels[i].vc.task); in bam_dma_remove()
1404 if (!bdev->channels[i].fifo_virt) in bam_dma_remove()
1407 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, in bam_dma_remove()
1408 bdev->channels[i].fifo_virt, in bam_dma_remove()
1409 bdev->channels[i].fifo_phys); in bam_dma_remove()
1412 tasklet_kill(&bdev->task); in bam_dma_remove()
1414 clk_disable_unprepare(bdev->bamclk); in bam_dma_remove()
1421 clk_disable(bdev->bamclk); in bam_dma_runtime_suspend()
1431 ret = clk_enable(bdev->bamclk); in bam_dma_runtime_resume()
1445 clk_unprepare(bdev->bamclk); in bam_dma_suspend()
1455 ret = clk_prepare(bdev->bamclk); in bam_dma_resume()
1474 .name = "bam-dma-engine",
1483 MODULE_DESCRIPTION("QCOM BAM DMA engine driver");